線性回歸 pytorch實現
1.模擬回歸問題，生成訓練數據
import torch import torch.nn as nn import numpy as np # Hyper-parameters input_size = 10 output_size = 2 num_epochs = 750 learning_rate = 1e-5 np.random.seed(1) # Toy dataset x_train = np.random.randn(1000,10).astype(np.float32) W = np.random.randint(0,20,size = (10,2)) y_train = (x_train.dot(W)).astype(np.float32) x_train.shape,y_train.shape
2.用梯度下降的方法更新未知參數w1, 用隨機數初始化w1
w1 = np.random.randn(10,2)
for epoch in range(num_epochs): #forward pass y_pred = x_train.dot(w1) #compute loss loss = np.square(y_pred - y_train).sum() if (epoch+1) % 10 == 0: print ('Epoch [{}/{}], Loss: {:.4f}'.format(epoch+1, num_epochs, loss.item())) #backward pass grad_y_pred = 2.0*(y_pred - y_train) grad_w1 = x_train.T.dot(grad_y_pred) w1 -= grad_w1*learning_rate
3.輸出結果：
差不多700次左右loss就迭代到0了，我們對比w1和w可以看出它們已經非常接近了。
能否減少迭代次數呢？
我們減少參數量試試：
W是一個10x2的一個矩陣，我們可以分解成W1*W2，W1和W2為10x1,1*2的矩陣這樣的話我們的參數從20減少到了12，
減少了將近一半，可能在參數少的時候影響不大，先看看效果。
實現代碼：
import torch import torch.nn as nn import numpy as np # Hyper-parameters input_size = 10 output_size = 2 num_epochs = 700 learning_rate = 0.01 np.random.seed(1) # Toy dataset x_train = np.random.randn(1000,10).astype(np.float32) W = np.random.randint(0,20,size = (10,2)) y_train = (x_train.dot(W)).astype(np.float32) x_train.shape,y_train.shape w1 = np.random.randn(10,1) w2 = np.random.randn(1,2) for epoch in range(num_epochs): #forward pass h = x_train.dot(w1) #相當於隱藏層 y_pred = h.dot(w2) #compute loss loss = np.square(y_pred - y_train).sum() if (epoch+1) % 10 == 0: print ('Epoch [{}/{}], Loss: {:.4f}'.format(epoch+1, num_epochs, loss.item())) #backward pass grad_y_pred = 2.0*(y_pred - y_train) grad_w2 = h.T.dot(grad_y_pred) grad_h = grad_y_pred.dot(w2.T) grad_w1 = x_train.T.dot(grad_h) w1 -= grad_w1*learning_rate w2 -= grad_w2*learning_rate
結果如圖，如果學習率太小，loss可能會固定為一個值保持不變，如果學習率較大會產生這種nan的情況。
loss總是不能收斂到0。問題是什么？
我們將剛剛的操作反過來，W1和W2的維度設為10x5,5*2 ,參數量變成了60個比20個多了兩倍。
結果很好，loss很快收斂到0，大概在epoch = 420時候就收斂到0，比之前更快
對比W1*W2 和W:
可以看出差距很小，並且綜合上面，也可以看出MSE的loss可以很好的反映線性回歸擬合的好壞。
如果我們繼續增加參數量，會不會讓收斂速度更快呢。
會，
迭代次數變少，但是運算時間可能變大了，所以具體問題要具體分析得到合適參數量。
一開始建模的時候，參數越多越好，最后濃縮模型的時候再慢慢調整參數量。
4.用pytorch中的方法實現線性回歸
import torch import torch.nn as nn import numpy as np # Hyper-parameters input_size = 10 output_size = 2 num_epochs = 700 learning_rate = 1e-5 torch.manual_seed(1) # Toy dataset x_train = torch.randn(1000,input_size) W = torch.randint(0,20,size = (10,output_size),dtype = torch.float32) y_train = x_train.mm(W) w1 = torch.randn(input_size,5,requires_grad=True) w2 = torch.randn(5,output_size,requires_grad=True) for epoch in range(num_epochs): #forward pass h = x_train.mm(w1) #相當於隱藏層 y_pred = h.mm(w2) #compute loss loss = (y_pred - y_train).pow(2).sum() if (epoch+1) % 10 == 0: print ('Epoch [{}/{}], Loss: {:.4f}'.format(epoch+1, num_epochs, loss.item())) #backward pass loss.backward() #計算loss對w1 w2的導數 with torch.no_grad(): w1 -= learning_rate*w1.grad w2 -= learning_rate*w2.grad w1.grad.zero_() #清理緩存中的grad w2.grad.zero_()
5.用pytorch中的模型實現線性回歸：
import torch import torch.nn as nn import numpy as np # Hyper-parameters input_size = 10 output_size = 2 num_epochs = 700 learning_rate = 1e-5 torch.manual_seed(1) H = 5 # Toy dataset x_train = torch.randn(1000,input_size) W = torch.randint(0,20,size = (10,output_size),dtype = torch.float32) y_train = x_train.mm(W) model = torch.nn.Sequential( torch.nn.Linear(input_size,H,bias=False), torch.nn.Linear(H,output_size,bias=False), ) loss_fn = nn.MSELoss(reduction='sum') for epoch in range(num_epochs): #forward pass y_pred = model(x_train) #compute loss loss = loss_fn(y_pred,y_train) if (epoch+1) % 10 == 0: print ('Epoch [{}/{}], Loss: {:.4f}'.format(epoch+1, num_epochs, loss.item())) #backward pass model.zero_grad() loss.backward() #計算loss對w1 w2的導數 with torch.no_grad(): for param in model.parameters(): param -= learning_rate*param.grad
'''
也可以用optimizer來做參數更新
'''
6. 利用optim中的優化器來優化參數
import torch import torch.nn as nn import numpy as np # Hyper-parameters input_size = 10 output_size = 2 num_epochs = 700 learning_rate = 1e-1 torch.manual_seed(1) H = 5 # Toy dataset x_train = torch.randn(1000,input_size) W = torch.randint(0,20,size = (10,output_size),dtype = torch.float32) y_train = x_train.mm(W) model = torch.nn.Sequential( torch.nn.Linear(input_size,H,bias=False), torch.nn.Linear(H,output_size,bias=False), ) loss_fn = nn.MSELoss(reduction='sum') optimizer = torch.optim.Adam(model.parameters(),lr = learning_rate) for epoch in range(num_epochs): #forward pass y_pred = model(x_train) #compute loss loss = loss_fn(y_pred,y_train) if (epoch+1) % 10 == 0: print ('Epoch [{}/{}], Loss: {:.4f}'.format(epoch+1, num_epochs, loss.item())) #backward pass optimizer.zero_grad() loss.backward() #計算loss對w1 w2的導數 #update model parameters optimizer.step()
可以看出用pytorch的模型構建model非常的方便，但是模型內部是已經定義好的，用戶只能修改參數，而不能修改模型內部結構。
這需要我們自己定義一個模型：
7. DIY 自己的模型
import torch import torch.nn as nn import numpy as np # Hyper-parameters input_size = 10 output_size = 2 num_epochs = 700 learning_rate = 1e-5 torch.manual_seed(1) H = 5 # Toy dataset x_train = torch.randn(1000,input_size) W = torch.randint(0,20,size = (10,output_size),dtype = torch.float32) y_train = x_train.mm(W) class TwoLayerNet(torch.nn.Module): def __init__(self,input_size,H,output_size): super(TwoLayerNet, self).__init__() self.linear1 = torch.nn.Linear(input_size, H, bias = False) self.linear2 = torch.nn.Linear(H, output_size, bias = False) def forward(self,x): y_pred = self.linear2(self.linear1(x)) return y_pred model = TwoLayerNet(input_size, H, output_size) loss_fn = nn.MSELoss(reduction='sum') optimizer = torch.optim.SGD(model.parameters(),lr = learning_rate) for epoch in range(num_epochs): #forward pass y_pred = model(x_train) #compute loss loss = loss_fn(y_pred,y_train) if (epoch+1) % 10 == 0: print ('Epoch [{}/{}], Loss: {:.4f}'.format(epoch+1, num_epochs, loss.item())) #backward pass optimizer.zero_grad() loss.backward() #計算loss對w1 w2的導數 #update model parameters optimizer.step()
# Save the model checkpoint torch.save(model.state_dict(), 'model.ckpt')
模型參數最后可以保存下來。