[深度學習]-訓練和測試模型的常用代碼

數據轉換

1.numpy->tensor

data2 = torch.tensor(data1)

2.tensor->numpy

非訓練數據（訓練結束后的tensor）的轉換：
data2 = np.array(data1)
如果是訓練過程中需要轉換，則：
data2 = data1.cpu().data.numpy() 動態，會一起改變
data2 = data1.cpu().detach().numpy() 靜態

3.維度轉換

如果第一維，維度為1，可以去掉：（1，8，8）-> （8，8）
data2 = np.concatenate(datat1, axis=0)
如果包含維度為1的維度，可以去掉：（1，8，8，1）-> （8，8）
data2 = data1.squeeze()
轉換維度層數：（1，8，3）-> （8，1，3）
data2 = np.swapaxes(1, 0, 2)

---

加載數據集dataloader

from torch.utils.data import DataLoader
form 自己寫的dataset import Dataset

train_set = Dataset(train=True)
val_set = Dataset(train=False)

image_datasets = {
    'train': train_set, 'val': val_set
}

batch_size = 4

dataloaders = {
    'train': DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=2),
    'val': DataLoader(val_set, batch_size=batch_size, shuffle=False, num_workers=2)
}

dataset_sizes = {
    x: len(image_datasets[x]) for x in image_datasets.keys()
}
print(dataset_sizes)

for epoch in range(num_epochs):
	for phase in ['train', 'val']:
		if phase == 'train':
			# for param_group in optimizer.param_groups:
				# print("LR", param_group['lr'])
			model.train() 
		else:
			model.eval() 

以上適用於train一遍test一遍的情況

或者分別加載訓練和測試：

train_dataset = Dataset('train')
train_data_loader = torch.utils.data.DataLoader(train_dataset, batch_size=8, shuffle=True,
                                                    num_workers=2, collate_fn=collate_fn)

test_dataset = Dataset('eval')
test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=8, shuffle=False,
                                                   num_workers=2, collate_fn=collate_fn)

自己寫Dataset

from torch.utils.data import Dataset
import os
import cv2
import torch
import numpy as np


class Dataset(Dataset):
    def __init__(self,train):
        if train:
            self.datapath = {'image': '/home/myy/code/Final_Project/data_train.txt', 'target':'/home/myy/code/Final_Project/gt_train.txt'}
        else:
            self.datapath = {'image': '/home/myy/code/Final_Project/data_test.txt', 'target':'/home/myy/code/Final_Project/gt_test.txt'}
            # self.datapath = {'image': '/home/myy/code/Final_Project/test_small_data.txt', 'target':'/home/myy/code/Final_Project/test_small.txt'}
        self.image_list, self.target_list = self.read_txt(self.datapath)
    
# 此處可以依據需要自己定義一些函數
# 注意調用前要加上`self.`
# 比如以下兩個讀取數據的函數，read_txt、read_json就是自己定義的
    def read_txt(self,datapath):
        im =[]
        target_image = []
        print(datapath)
        with open(datapath['image'], 'r') as f:
            image_list = f.readlines()
        with open(datapath['target'], 'r') as f:
            target_list = f.readlines()
        return image_list, target_list

    def read_json(save_path, encoding='utf8'):
        jsondata = []
        with open(save_path, 'r', encoding=encoding) as f:
            content = f.read()
            content = json.loads(content)
            for key in content:
                jsondata.append(content[key])
            return jsondata

    def __getitem__(self, item):
        # 最核心的部分，經過處理，要返回輸入和gt

        return img, target

    def __len__(self):
		# 這可以根據具體情況修改，不寫也行
        return len(self.data)

---

模型訓練

保存模型

torch.save(model.state_dict(), model_path)

取最佳loss的權重

import copy
best_model_wts = copy.deepcopy(model.state_dict())

finetuning訓練指定層參數

trainable_vars = list(model.roi_heads.box_predictor.parameters())+ \
                 list(model.roi_heads.box_head.parameters())+ \
                 list(model.rpn.head.parameters())
                #  list(model.backbone.body.parameters())+ \
                #  list(model.backbone.fpn.parameters())+ \
 optimizer = torch.optim.SGD(trainable_vars , lr=1e-3, momentum=0.5)

模型測試

主要代碼

device = torch.device("cuda:0" if  torch.cuda.is_available() else "cpu")
print(device)
num_class = 1
model_path = '訓練模型保存路徑/保存名稱.pth'
model = 模型(num_class).to(device)
model.load_state_dict(torch.load(model_path))
model.eval()

模型測試完整流程

import cv2
from torchvision import transforms, datasets, models
from torch.utils.data import  DataLoader
import torch
import numpy as np
import os
from sklearn import metrics 
import matplotlib.pyplot as plt

device = torch.device("cuda:2" if torch.cuda.is_available() else "cpu")
print(device)
num_class = 3
model_path = 模型路徑
model = 模型(num_class).to(device)
model.load_state_dict(torch.load(model_path))
model.eval()   # Set model to evaluate mode

test_dataset = 數據集讀取(train=False)
test_loader = DataLoader(test_dataset, batch_size=1, shuffle=False, num_workers=2)

def turn(l):
    l = l.data.cpu().numpy()
    l = l.squeeze()
    l = np.swapaxes(l, 0, 2)
    l = np.swapaxes(l, 0, 1)
    return l

for inputs, labels in test_loader:
    model.to(device)
    inputs = inputs.to(device)
    labels = labels.to(device)

    pred = model(inputs)
    # pred = torch.relu(pred)
    pred = turn(pred)
    gt = turn(labels)

---

評價指標

混淆矩陣

以分割為例，經過.flatten()處理。

def acc(pred, gt):
    tp = 0
    tn = 0
    fp = 0
    fn = 0
    num = len(pred)
    for i in range(num):
        if pred[i] > 0 and gt[i] == 1:
            tp += 1
        if pred[i] > 0 and gt[i] == 0:
            fp += 1
        if pred[i] == 0 and gt[i] == 1:
            fn += 1
        if pred[i] == 0 and gt[i] == 0:
            tn += 1
    acc = (tp + tn) / num
    iou = tp / (tp + fp + fn)
    rec = tp / (tp + fn)
    pre = tp / (tp + fp)
    f1 = 2 * pre * rec / (pre + rec)
    print("mAcc is :{},  mIou is :{}, recall is :{}, precision is :{}, f1 is :{}".format(acc, iou, rec, pre, f1))

ROC曲線圖

def draw_roc(pred, gt, name):
    tpr, fpr, thresholds = metrics.roc_curve(gt, pred, pos_label=0)
    plt.figure
    plt.plot(fpr, tpr, label = name)
    plt.xlabel('FPR')
    plt.ylabel('TPR')
    plt.legend(loc = 'lower right')
    plt.title(name)
    plt.savefig('路徑/{}.png'.format(name))
    # plt.close()  如果有多個類別，不close()就會畫在一張圖上