TensorFlow正則化方法tf.contrib.layers.l2_regularizer

在tensorflow里提供了計算L1、L2正則化的函數

tf.contrib.layers.l1_regularizer()
tf.contrib.layers.l2_regularizer()

設計一個簡易的網絡模型，實現了通過集合計算一個4層全連接神經網絡帶L2正則化損失函數的功能

import tensorflow as tf
import numpy as np

# 定義訓練輪次
training_steps = 30000

# 定義輸入的數據和對應的標簽並在 for 循環里進行填充
data = []
label = []

for i in range(200):
    x1 = np.random.uniform(-1, 1)
    x2 = np.random.uniform(0, 2)

    # 這里對 x1,x2 進行判斷，如果產生的點落在半徑為1的圓內，則label為0，否則為1
    if x1 ** 2 + x2 ** 2 <= 1:
        data.append([np.random.normal(x1, 0.1), np.random.normal(x2, 0.1)])
        label.append(0)
    else:
        data.append([np.random.normal(x1, 0.1), np.random.normal(x2, 0.1)])
        label.append(1)

# numpy 的 hstack() 函數用於在水平方向將元素堆起來

data = np.hstack(data).reshape(-1, 2)
label = np.hstack(label).reshape(-1, 1)


# 定義完成前向傳播的隱層
def hidden_layer(input_tensor, weight1, bias1, weight2, bias2, weight3, bias3):
    layer1 = tf.nn.relu(tf.matmul(input_tensor, weight1) + bias1)
    layer2 = tf.nn.relu(tf.matmul(layer1, weight2) + bias2)
    return tf.matmul(layer2, weight3) + bias3


xs = tf.placeholder(tf.float32, shape=(None, 2), name="x-input")
ys = tf.placeholder(tf.float32, shape=(None, 1), name="y-output")

# 定義權重參數和偏置參數
weight1 = tf.Variable(tf.truncated_normal([2, 10], stddev=0.1))
bias1 = tf.Variable(tf.constant(0.1, shape=[10]))
weight2 = tf.Variable(tf.truncated_normal([10, 10], stddev=0.1))
bias2 = tf.Variable(tf.constant(0.1, shape=[10]))
weight3 = tf.Variable(tf.truncated_normal([10, 1], stddev=0.1))
bias3 = tf.Variable(tf.constant(0.1, shape=[1]))

# 計算 data 數組長度
sample_size = len(data)

# 得到隱藏層前向傳播結果
y = hidden_layer(xs, weight1, bias1, weight2, bias2, weight3, bias3)

# 定義損失函數
error_loss = tf.reduce_sum(tf.pow(ys-y, 2))
tf.add_to_collection("losses", error_loss)

# 參數L2正則化
regularizer = tf.contrib.layers.l2_regularizer(0.01)
retularization = regularizer(weight1) + regularizer(weight2) + regularizer(weight3)
tf.add_to_collection("losses", retularization)

# get_collection函數獲取指定集合中的所有個體，這里是獲取所有損失值，並在 add_n() 函數中進行加和運算
loss = tf.add_n(tf.get_collection("losses"))

# 定義一個優化器
train_op = tf.train.AdamOptimizer(0.05).minimize(loss)

with tf.Session() as sess:
    init = tf.global_variables_initializer()
    sess.run(init)

    for i in range(training_steps):
        sess.run(train_op, feed_dict={xs: data, ys: label})

        # 每迭代 2000次 輸出一個loss值
        if i % 2000 == 0:
            loss_value = sess.run(loss, feed_dict={xs: data, ys: label})
            print("After %d steps, mse_loss: %f" %(i, loss_value))



# 運行結果：
After 0 steps, mse_loss: 51.364639
After 2000 steps, mse_loss: 7.050952
After 4000 steps, mse_loss: 4.775972
After 6000 steps, mse_loss: 4.787066
After 8000 steps, mse_loss: 4.931646
After 10000 steps, mse_loss: 4.702201
After 12000 steps, mse_loss: 4.578232
After 14000 steps, mse_loss: 4.605384
After 16000 steps, mse_loss: 5.032600
After 18000 steps, mse_loss: 4.586043
After 20000 steps, mse_loss: 4.606448
After 22000 steps, mse_loss: 4.518520
After 24000 steps, mse_loss: 4.620658
After 26000 steps, mse_loss: 4.713350
After 28000 steps, mse_loss: 4.740762