Pytorch LSTM/GRU更新h0, c0

LSTM隱層狀態h0, c0通常初始化為0，大部分情況下模型也能工作的很好。但是有時將h0, c0作為隨機值，或直接作為模型參數的一部分進行優化似乎更為合理。

這篇post給出了經驗證明：

Non-Zero Initial States for Recurrent Neural Networks

給出的經驗結果：

給出的結論是：（1）非零的初始狀態初始化能夠加速訓練並改善模型泛化性能，（2）將初始狀態作為模型參數去訓練要比具有零均值的噪聲初始化更有效， （3）如果選擇學習隱層初始狀態，添加噪聲並不能帶來額外的收益。

基本上，如果你的數據包括許多短序列，那么訓練初始狀態可以加速學習。相反，如果數據僅包含少量的長序列，那么可能沒有足夠的數據來有效地訓練初始狀態；在這種情況下，使用一個有噪聲的初始狀態可以加速學習。他們沒有提到的一個想法是如何恰當地確定隨機噪聲發生器的均值和std。此外，這篇文章Forecasting with Recurrent Neural Networks: 12 Tricks 中的Trick 4提出了一種基於反向傳播誤差的自適應方法，使初始狀態噪聲的大小根據反向傳播的誤差自適應變化。

 

 

 

 

 

 實際效果有待進一步驗證。

 

事實上，LSTM的隱藏層初始狀態h0, c0可以看做是模型的一部分參數，並在迭代中更新。這里給出pytorch中LSTM更新隱藏層初始狀態h0, c0的一種實現方法（來自知乎）。

作者：鄭華濱
鏈接：https://www.zhihu.com/question/270772480/answer/358198157
來源：知乎
著作權歸作者所有。商業轉載請聯系作者獲得授權，非商業轉載請注明出處。

import torch
import torch.nn as nn
from torch.autograd import Variable

class EasyLSTM(nn.LSTM):

    def __init__(self, *args, **kwargs):
        nn.LSTM.__init__(self, *args, **kwargs)
        self.num_direction = 1 + self.bidirectional
        state_size = (self.num_layers * self.num_direction, 1, self.hidden_size)
        self.init_h = nn.Parameter(torch.zeros(state_size))
        self.init_c = nn.Parameter(torch.zeros(state_size))

    def forward(self, rnn_input, prev_states = None):
        batch_size = rnn_input.size(1)
        if prev_states is None:
            state_size = (self.num_layers * self.num_direction, batch_size, self.hidden_size)
            init_h = self.init_h.expand(*state_size).contiguous()
            init_c = self.init_c.expand(*state_size).contiguous()
            prev_states = (init_h, init_c)
        rnn_output, states = nn.LSTM.forward(self, rnn_input, prev_states)
        return rnn_output, states

 

LSTM、GRU、LSTMCell、GRUCell ？

import torch
import torch.nn as nn
from torch.autograd import Variable

class EasyLSTM(nn.LSTM):

    def __init__(self, *args, **kwargs):
        nn.LSTM.__init__(self, *args, **kwargs)
        self.num_direction = 1 + self.bidirectional
        state_size = (self.num_layers * self.num_direction, 1, self.hidden_size)
        self.init_h = nn.Parameter(torch.zeros(state_size))
        self.init_c = nn.Parameter(torch.zeros(state_size))

    def forward(self, rnn_input, prev_states = None):
        batch_size = rnn_input.size(1)
        if prev_states is None:
            state_size = (self.num_layers * self.num_direction, batch_size, self.hidden_size)
            init_h = self.init_h.expand(*state_size).contiguous()
            init_c = self.init_c.expand(*state_size).contiguous()
            prev_states = (init_h, init_c)
        rnn_output, states = nn.LSTM.forward(self, rnn_input, prev_states)
        return rnn_output, states

class EasyGRU(nn.GRU):

    def __init__(self, *args, **kwargs):
        nn.GRU.__init__(self, *args, **kwargs)
        self.num_direction = 1 + self.bidirectional
        state_size = (self.num_layers * self.num_direction, 1, self.hidden_size)
        self.init_h = nn.Parameter(torch.zeros(state_size))

    def forward(self, rnn_input, prev_states = None):
        batch_size = rnn_input.size(1)
        if prev_states is None:
            state_size = (self.num_layers * self.num_direction, batch_size, self.hidden_size)
            init_h = self.init_h.expand(*state_size).contiguous()
            prev_states = init_h
        rnn_output, states = nn.GRU.forward(self, rnn_input, prev_states)
        return rnn_output, states


class EasyLSTMCell(nn.LSTMCell):

    def __init__(self, *args, **kwargs):
        nn.LSTMCell.__init__(self, *args, **kwargs)
        state_size = (1, self.hidden_size)
        self.init_h = nn.Parameter(torch.zeros(state_size))
        self.init_c = nn.Parameter(torch.zeros(state_size))

    def forward(self, rnn_input, prev_states=None):
        batch_size = rnn_input.size(0)
        if prev_states is None:
            state_size = (batch_size, self.hidden_size)
            init_h = self.init_h.expand(*state_size).contiguous()
            init_c = self.init_c.expand(*state_size).contiguous()
            prev_states = (init_h, init_c)
        h, c = nn.LSTMCell.forward(self, rnn_input, prev_states)
        return h, c


class EasyGRUCell(nn.GRUCell):

    def __init__(self, *args, **kwargs):
        nn.GRUCell.__init__(self, *args, **kwargs)
        state_size = (1, self.hidden_size)
        self.init_h = nn.Parameter(torch.zeros(state_size))

    def forward(self, rnn_input, prev_states=None):
        batch_size = rnn_input.size(0)
        if prev_states is None:
            state_size = (batch_size, self.hidden_size)
            init_h = self.init_h.expand(*state_size).contiguous()
            prev_states = init_h
        h = nn.GRUCell.forward(self, rnn_input, prev_states)
        return h

if __name__ == '__main__':

    lstm = EasyLSTM(10, 20, 2)
    input = torch.randn(5, 3, 10)
    h0 = torch.randn(2, 3, 20)
    c0 = torch.randn(2, 3, 20)
    output, (hn, cn) = lstm(input, (h0, c0))

    gru = EasyGRU(10, 20, 2)
    input = torch.randn(5, 3, 10)
    h0 = torch.randn(2, 3, 20)
    output, hn = gru(input, h0)

    lstmcell = EasyLSTMCell(10, 20)
    input = torch.randn(6, 3, 10)
    h = torch.randn(3, 20)
    c = torch.randn(3, 20)
    out = []
    for i in range(6):
        h, c = lstmcell(input[i], (h, c))
        out.append(h)

    grucell = EasyGRUCell(10, 20)
    input = torch.randn(6, 3, 10)
    h = torch.randn(3, 20)
    out = []
    for i in range(6):
        h = grucell(input[i], h)
        out.append(h)

 

 

參考：

Non-Zero Initial States for Recurrent Neural Networks

pytorch LSTM更新h0, c0

Best way to initialize LSTM state

https://danijar.com/tips-for-training-recurrent-neural-networks/

Initialization of first hidden state in LSTM and truncated BPTT