TensorFlow教程使用RNN生成唐詩

本教程轉載至：TensorFlow練習7: 基於RNN生成古詩詞

使用的數據集是全唐詩，首先提供一下數據集的下載鏈接：https://pan.baidu.com/s/13pNWfffr5HSN79WNb3Y0_w              提取碼：koss

RNN不像傳統的神經網絡-它們的輸出輸出是固定的，而RNN允許我們輸入輸出向量序列。RNN是為了對序列數據進行建模而產生的。本帖代碼移植自char-rnn，它是基於Torch的洋文模型，稍加修改即可應用於中文。char-rnn使用文本文件做為輸入、訓練RNN模型，然后使用它生成和訓練數據類似的文本。

下邊代碼有修改，以適應TensorFlow1.4和GPU平台

#coding=utf-8
import collections
import numpy as np 
import tensorflow as tf 
import io
import sys
import os
reload(sys)
sys.setdefaultencoding('utf-8')
#-------------------------------數據預處理---------------------------#
 
poetry_file ='poetry.txt'
 
# 詩集
poetrys = []
with io.open(poetry_file, "r", encoding='utf-8',) as f:
    for line in f:
        # print line
        try:
            title, content = line.strip().split(':')
            content = content.replace(' ','')
            if '_' in content or '(' in content or '（' in content or '《' in content or '[' in content:
                continue
            if len(content) < 5 or len(content) > 79:
                continue
            content = '[' + content + ']'
            poetrys.append(content)
        except Exception as e:
            pass

#按詩的字數排序
poetrys = sorted(poetrys,key=lambda line: len(line))
print(u"唐詩總數: ")
print(len(poetrys))
print(u"測試")

# 統計每個字出現次數
all_words = []
for poetry in poetrys:
    all_words += [word for word in poetry]
counter = collections.Counter(all_words)
count_pairs = sorted(counter.items(), key=lambda x: -x[1])
words, _ = zip(*count_pairs)

# 取前多少個常用字
words = words[:len(words)] + (' ',)
# 每個字映射為一個數字ID
word_num_map = dict(zip(words, range(len(words))))
# 把詩轉換為向量形式，參考TensorFlow練習1
to_num = lambda word: word_num_map.get(word, len(words))
poetrys_vector = [ list(map(to_num, poetry)) for poetry in poetrys]
#[[314, 3199, 367, 1556, 26, 179, 680, 0, 3199, 41, 506, 40, 151, 4, 98, 1],
#[339, 3, 133, 31, 302, 653, 512, 0, 37, 148, 294, 25, 54, 833, 3, 1, 965, 1315, 377, 1700, 562, 21, 37, 0, 2, 1253, 21, 36, 264, 877, 809, 1]
#....]

# 每次取64首詩進行訓練
batch_size = 64
n_chunk = len(poetrys_vector) // batch_size
x_batches = []
y_batches = []
for i in range(n_chunk):
    start_index = i * batch_size
    end_index = start_index + batch_size

    batches = poetrys_vector[start_index:end_index]
    length = max(map(len,batches))
    xdata = np.full((batch_size,length), word_num_map[' '], np.int32)
    for row in range(batch_size):
        xdata[row,:len(batches[row])] = batches[row]
    ydata = np.copy(xdata)
    ydata[:,:-1] = xdata[:,1:]
    """
    xdata             ydata
    [6,2,4,6,9]       [2,4,6,9,9]
    [1,4,2,8,5]       [4,2,8,5,5]
    """
    x_batches.append(xdata)
    y_batches.append(ydata)

#---------------------------------------RNN--------------------------------------#

input_data = tf.placeholder(tf.int32, [batch_size, None])
output_targets = tf.placeholder(tf.int32, [batch_size, None])
# 定義RNN
def neural_network(model='lstm', rnn_size=128, num_layers=2):
    if model == 'rnn':
        cell_fun = tf.nn.rnn_cell.BasicRNNCell
    elif model == 'gru':
        cell_fun = tf.nn.rnn_cell.GRUCell
    elif model == 'lstm':
        cell_fun = tf.nn.rnn_cell.BasicLSTMCell

    cell = cell_fun(rnn_size, state_is_tuple=True)
    cell = tf.nn.rnn_cell.MultiRNNCell([cell] * num_layers, state_is_tuple=True)

    initial_state = cell.zero_state(batch_size, tf.float32)

    with tf.variable_scope('rnnlm'):
        softmax_w = tf.get_variable("softmax_w", [rnn_size, len(words)+1])
        softmax_b = tf.get_variable("softmax_b", [len(words)+1])
        with tf.device("/gpu:0"):
            embedding = tf.get_variable("embedding", [len(words)+1, rnn_size])
            inputs = tf.nn.embedding_lookup(embedding, input_data)

    outputs, last_state = tf.nn.dynamic_rnn(cell, inputs, initial_state=initial_state, scope='rnnlm')
    output = tf.reshape(outputs,[-1, rnn_size])

    logits = tf.matmul(output, softmax_w) + softmax_b
    probs = tf.nn.softmax(logits)
    return logits, last_state, probs, cell, initial_state

ckpt_dir="./ckpt_dir"
if not os.path.exists(ckpt_dir):
    os.makedirs(ckpt_dir)

#訓練
def train_neural_network():
    logits, last_state, _, _, _ = neural_network()
    targets = tf.reshape(output_targets, [-1])
    loss = tf.contrib.legacy_seq2seq.sequence_loss_by_example([logits], [targets], [tf.ones_like(targets, dtype=tf.float32)], len(words))
    cost = tf.reduce_mean(loss)
    learning_rate = tf.Variable(0.0, trainable=False)
    tvars = tf.trainable_variables()
    grads, _ = tf.clip_by_global_norm(tf.gradients(cost, tvars), 5)
    optimizer = tf.train.AdamOptimizer(learning_rate)
    train_op = optimizer.apply_gradients(zip(grads, tvars))

    with tf.Session() as sess:
        sess.run(tf.initialize_all_variables())

        saver = tf.train.Saver(tf.all_variables())

        for epoch in range(295):
            sess.run(tf.assign(learning_rate, 0.002 * (0.97 ** epoch)))
            n = 0
            for batche in range(n_chunk):
                train_loss, _ , _ = sess.run([cost, last_state, train_op], feed_dict={input_data: x_batches[n], output_targets: y_batches[n]})
                n += 1
                print(epoch, batche, train_loss)
            if epoch % 7 == 0:
                saver.save(sess, ckpt_dir+'/poetry.module', global_step=epoch)

train_neural_network()

這里我只說自己對bug調試和調優的一些想法，具體代碼理解，請聯系作者本人。

首先是#coding=utf-8的問題，這里是告訴python環境，當前python腳本的文字編碼是utf-8,這里如果不調整的話，默認的ansii環境極有可能報告編碼錯誤。

之后是數據集的utf-8編碼問題，這里在encoding的時候，用了utf-8 的選項，但是卻沒有告訴python環境，字符集編碼是utf-8，會導致每次解析到的content和title都會報錯，最終處理完的數據集大小為0，設置sys的默認編碼可以解決。

同時，默認的open函數沒有encoding選項，這個是在io.open中的選項，這個地方需要修改。

還有一點是一些接口使用問題，比如saver.save現在需要一個parent directory

之后是預測的代碼

#coding=utf-8
import collections
import numpy as np
import tensorflow as tf
import io
import sys
import os
import pdb
import time
reload(sys)
sys.setdefaultencoding('utf-8')
#-------------------------------數據預處理---------------------------#
 
poetry_file ='poetry.txt'
 
# 詩集
poetrys = []
with io.open(poetry_file, "r", encoding='utf-8',) as f:
    for line in f:
        try:
            title, content = line.strip().split(':')
            content = content.replace(' ','')
            if '_' in content or '(' in content or '（' in content or '《' in content or '[' in content:
                continue
            if len(content) < 5 or len(content) > 79:
                continue
            content = '[' + content + ']'
            poetrys.append(content)
        except Exception as e: 
            pass
 
# 按詩的字數排序
poetrys = sorted(poetrys,key=lambda line: len(line))
print(u'唐詩總數: ', len(poetrys))
 
# 統計每個字出現次數
all_words = []
for poetry in poetrys:
    all_words += [word for word in poetry]
counter = collections.Counter(all_words)
count_pairs = sorted(counter.items(), key=lambda x: -x[1])
words, _ = zip(*count_pairs)
 
# 取前多少個常用字
words = words[:len(words)] + (' ',)
# 每個字映射為一個數字ID
word_num_map = dict(zip(words, range(len(words))))
# 把詩轉換為向量形式
to_num = lambda word: word_num_map.get(word, len(words))
poetrys_vector = [ list(map(to_num, poetry)) for poetry in poetrys]
#[[314, 3199, 367, 1556, 26, 179, 680, 0, 3199, 41, 506, 40, 151, 4, 98, 1],
#[339, 3, 133, 31, 302, 653, 512, 0, 37, 148, 294, 25, 54, 833, 3, 1, 965, 1315, 377, 1700, 562, 21, 37, 0, 2, 1253, 21, 36, 264, 877, 809, 1]
#....]
 
batch_size = 1
n_chunk = len(poetrys_vector) // batch_size
x_batches = []
y_batches = []
for i in range(n_chunk):
    start_index = i * batch_size
    end_index = start_index + batch_size
 
    batches = poetrys_vector[start_index:end_index]
    length = max(map(len,batches))
    xdata = np.full((batch_size,length), word_num_map[' '], np.int32)
    for row in range(batch_size):
        xdata[row,:len(batches[row])] = batches[row]
    ydata = np.copy(xdata)
    ydata[:,:-1] = xdata[:,1:]
    """
    xdata             ydata
    [6,2,4,6,9]       [2,4,6,9,9]
    [1,4,2,8,5]       [4,2,8,5,5]
    """
    x_batches.append(xdata)
    y_batches.append(ydata)
 
 
#---------------------------------------RNN--------------------------------------#
 
input_data = tf.placeholder(tf.int32, [batch_size, None])
output_targets = tf.placeholder(tf.int32, [batch_size, None])
# 定義RNN
def neural_network(model='lstm', rnn_size=128, num_layers=2):
    if model == 'rnn':
        cell_fun = tf.nn.rnn_cell.BasicRNNCell
    elif model == 'gru':
        cell_fun = tf.nn.rnn_cell.GRUCell
    elif model == 'lstm':
        cell_fun = tf.nn.rnn_cell.BasicLSTMCell
 
    cell = cell_fun(rnn_size, state_is_tuple=True)
    cell = tf.nn.rnn_cell.MultiRNNCell([cell] * num_layers, state_is_tuple=True)
 
    initial_state = cell.zero_state(batch_size, tf.float32)
 
    with tf.variable_scope('rnnlm'):
        softmax_w = tf.get_variable("softmax_w", [rnn_size, len(words)+1])
        softmax_b = tf.get_variable("softmax_b", [len(words)+1])
        with tf.device("/gpu:0"):
            embedding = tf.get_variable("embedding", [len(words)+1, rnn_size])
            inputs = tf.nn.embedding_lookup(embedding, input_data)
 
    outputs, last_state = tf.nn.dynamic_rnn(cell, inputs, initial_state=initial_state, scope='rnnlm')
    output = tf.reshape(outputs,[-1, rnn_size])
 
    logits = tf.matmul(output, softmax_w) + softmax_b
    probs = tf.nn.softmax(logits)
    return logits, last_state, probs, cell, initial_state
 
#-------------------------------生成古詩---------------------------------#
# 使用訓練完成的模型
 
def gen_poetry():
    def to_word(weights):
        t = np.cumsum(weights)
        s = np.sum(weights)
        sample = int(np.searchsorted(t, np.random.rand(1)*s))
        return words[sample]
 
    _, last_state, probs, cell, initial_state = neural_network()
 
    with tf.Session() as sess:
        sess.run(tf.initialize_all_variables())
 
        saver = tf.train.Saver(tf.all_variables())
        saver.restore(sess, './ckpt_dir/poetry.module-294')
 
        state_ = sess.run(cell.zero_state(1, tf.float32))
 
        x = np.array([list(map(word_num_map.get, '['))])
        [probs_, state_] = sess.run([probs, last_state], feed_dict={input_data: x, initial_state: state_})
        word = to_word(probs_)
        #word = words[np.argmax(probs_)]
        poem = ''
        while word != ']':
            poem += word
            x = np.zeros((1,1))
            x[0,0] = word_num_map[word]
            [probs_, state_] = sess.run([probs, last_state], feed_dict={input_data: x, initial_state: state_})
            word = to_word(probs_)
            #word = words[np.argmax(probs_)]
        return poem
 


def gen_poetry_with_head(head):
    def to_word(weights):
        t = np.cumsum(weights)
        s = np.sum(weights)
        sample = int(np.searchsorted(t, np.random.rand(1)*s))
        return words[sample]
 
    _, last_state, probs, cell, initial_state = neural_network()
 
    with tf.Session() as sess:
        sess.run(tf.initialize_all_variables())
 
        saver = tf.train.Saver(tf.all_variables())
        saver.restore(sess, './ckpt_dir/poetry.module-294')
 
        state_ = sess.run(cell.zero_state(1, tf.float32))
        poem = ''
        i = 0
        # print head
        # pdb.set_trace()
        for word in head:
            while word != '，' and word != '。':
                poem += word
                # print poem
                # print head
                # print word
                x = np.array([list(map(word_num_map.get, word))])
                [probs_, state_] = sess.run([probs, last_state], feed_dict={input_data: x, initial_state: state_})
                word = to_word(probs_)
                time.sleep(1)
            if i % 2 == 0:
                poem += '，'
            else:
                poem += '。'
            i += 1
        return poem

print(gen_poetry())
# print(gen_poetry_with_head(u'一二三四'))

這個藏頭詩的代碼用法有問題，不建議使用，我調了很久才調好，這次還是先列原作者的代碼，下次單獨說這塊的調整和調優問題。

結果：

有那么點意思，但仔細看問題還是很大，胡言亂語，模型的調優遠遠不行。