指數衰減學習率

 

#coding:utf-8
#設損失函數 loss=(w+1)^2,令w初值是常數10.反向傳播就是求最優w，即求最小loss對應的w值
#使用指數衰減學習率，在迭代初期得到較高的下降速度，可以在較小的訓練輪數下取得更有效收斂度

import tensorflow as tf

LEARNING_RATE_BASE = 0.1 #最初學習率
LEARNING_RATE_DECAY = 0.99 #學習率衰減率
LEARNING_RATE_STEP = 1 #喂入多少輪BATCH_SIZE后，更新一次學習率，一般設為：總樣本數/BATCH_SIZE

#運行了幾輪BATCH_SIZE的計數器，初值給0，設為不被訓練
global_step = tf.Variable(0, trainable=False)
#定義指數下降學習率
learning_rate = tf.train.exponential_decay(LEARNING_RATE_BASE,global_step,LEARNING_RATE_STEP, LEARNING_RATE_DECAY, staircase=True)
#定義待優化參數，初值給10S
w = tf.Variable(tf.constant(5,dtype=tf.float32))
#定義損失函數loss
loss = tf.square(w+1)
train_step = tf.train.GradientDescentOptimizer(learing_rate).minimize(loss, global_step=global_step)
#生成會話，訓練40輪
with tf.Session() as sess:
    init_op = tf.global_variables_initializer()
    sess.run(init_op)
    for i in range(40):
        sess.run(train_step)
        learing_rate_val = sess.run(learning_rate)
        global_step_val = sess.run(global_step)
        w_val = sess.run(w)
        loss_val = sess.run(loss)
        print "After %s steps: global_step is %f, w is %f, learning_rate is %f, loss is %f." % (i,global_step_val,w_val,learing_rate_val,loss_val)