一、簡化前饋網絡LeNet
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
|
import
torch as t
class
LeNet(t.nn.Module):
def
__init__(
self
):
super
(LeNet,
self
).__init__()
self
.features
=
t.nn.Sequential(
t.nn.Conv2d(
3
,
6
,
5
),
t.nn.ReLU(),
t.nn.MaxPool2d(
2
,
2
),
t.nn.Conv2d(
6
,
16
,
5
),
t.nn.ReLU(),
t.nn.MaxPool2d(
2
,
2
)
)
# 由於調整shape並不是一個class層，
# 所以在涉及這種操作（非nn.Module操作）需要拆分為多個模型
self
.classifiter
=
t.nn.Sequential(
t.nn.Linear(
16
*
5
*
5
,
120
),
t.nn.ReLU(),
t.nn.Linear(
120
,
84
),
t.nn.ReLU(),
t.nn.Linear(
84
,
10
)
)
def
forward(
self
, x):
x
=
self
.features(x)
x
=
x.view(
-
1
,
16
*
5
*
5
)
x
=
self
.classifiter(x)
return
x
net
=
LeNet()
|
二、優化器基本使用方法
- 建立優化器實例
- 循環：
- 清空梯度
- 向前傳播
- 計算Loss
- 反向傳播
- 更新參數
|
1
2
3
4
5
6
7
8
9
10
11
|
from
torch
import
optim
# 通常的step優化過程
optimizer
=
optim.SGD(params
=
net.parameters(), lr
=
1
)
optimizer.zero_grad()
# net.zero_grad()
input_
=
t.autograd.Variable(t.randn(
1
,
3
,
32
,
32
))
output
=
net(input_)
output.backward(output)
optimizer.step()
|
三、網絡模塊參數定制
為不同的子網絡參數不同的學習率，finetune常用，使分類器學習率參數更高，學習速度更快（理論上）。
1.經由構建網絡時划分好的模組進行學習率設定，
|
1
2
3
|
# # 直接對不同的網絡模塊制定不同學習率
optimizer
=
optim.SGD([{
'params'
: net.features.parameters()},
# 默認lr是1e-5
{
'params'
: net.classifiter.parameters(),
'lr'
:
1e
-
2
}], lr
=
1e
-
5
)
|
2.以網絡層對象為單位進行分組，並設定學習率
|
1
2
3
4
5
6
7
8
9
10
|
# # 以層為單位，為不同層指定不同的學習率
# ## 提取指定層對象
special_layers
=
t.nn.ModuleList([net.classifiter[
0
], net.classifiter[
3
]])
# ## 獲取指定層參數id
special_layers_params
=
list
(
map
(
id
, special_layers.parameters()))
print
(special_layers_params)
# ## 獲取非指定層的參數id
base_params
=
filter
(
lambda
p:
id
(p)
not
in
special_layers_params, net.parameters())
optimizer
=
t.optim.SGD([{
'params'
: base_params},
{
'params'
: special_layers.parameters(),
'lr'
:
0.01
}], lr
=
0.001
)
|
四、在訓練中動態的調整學習率
|
1
2
3
4
5
6
7
8
9
|
'''調整學習率'''
# 新建optimizer或者修改optimizer.params_groups對應的學習率
# # 新建optimizer更簡單也更推薦，optimizer十分輕量級，所以開銷很小
# # 但是新的優化器會初始化動量等狀態信息，這對於使用動量的優化器（momentum參數的sgd）可能會造成收斂中的震盪
# ## optimizer.param_groups:長度2的list，optimizer.param_groups[0]：長度6的字典
print
(optimizer.param_groups[
0
][
'lr'
])
old_lr
=
0.1
optimizer
=
optim.SGD([{
'params'
: net.features.parameters()},
{
'params'
: net.classifiter.parameters(),
'lr'
: old_lr
*
0.1
}], lr
=
1e
-
5
)
|
可以看到optimizer.param_groups結構，[{'params','lr', 'momentum', 'dampening', 'weight_decay', 'nesterov'},{……}]，集合了優化器的各項參數。
-
torch.optim的靈活使用
- 重寫sgd優化器
import torch from torch.optim.optimizer import Optimizer, required class SGD(Optimizer): def __init__(self, params, lr=required, momentum=0, dampening=0, weight_decay1=0, weight_decay2=0, nesterov=False): defaults = dict(lr=lr, momentum=momentum, dampening=dampening, weight_decay1=weight_decay1, weight_decay2=weight_decay2, nesterov=nesterov) if nesterov and (momentum <= 0 or dampening != 0): raise ValueError("Nesterov momentum requires a momentum and zero dampening") super(SGD, self).__init__(params, defaults) def __setstate__(self, state): super(SGD, self).__setstate__(state) for group in self.param_groups: group.setdefault('nesterov', False) def step(self, closure=None): """Performs a single optimization step. Arguments: closure (callable, optional): A closure that reevaluates the model and returns the loss. """ loss = None if closure is not None: loss = closure() for group in self.param_groups: weight_decay1 = group['weight_decay1'] weight_decay2 = group['weight_decay2'] momentum = group['momentum'] dampening = group['dampening'] nesterov = group['nesterov'] for p in group['params']: if p.grad is None: continue d_p = p.grad.data if weight_decay1 != 0: d_p.add_(weight_decay1, torch.sign(p.data)) if weight_decay2 != 0: d_p.add_(weight_decay2, p.data) if momentum != 0: param_state = self.state[p] if 'momentum_buffer' not in param_state: buf = param_state['momentum_buffer'] = torch.zeros_like(p.data) buf.mul_(momentum).add_(d_p) else: buf = param_state['momentum_buffer'] buf.mul_(momentum).add_(1 - dampening, d_p) if nesterov: d_p = d_p.add(momentum, buf) else: d_p = buf p.data.add_(-group['lr'], d_p) return loss