对tensorflow 中的attention encoder-decoder模型调试分析

#-*-coding:utf8-*-

__author = "buyizhiyou"
__date = "2017-11-21"


import random, time, os, decoder
from PIL import Image
import numpy as np
import tensorflow as tf
import pdb
import decoder
import random

'''
在汉字ocr项目中，利用基于attention的encoder-decoder(seq2seq)模型进行端对端的训练
单步调试，追踪tensorflow 对 attention-seq2seq模型的实现方式
python 中seq2seq.py的接口:tf.nn.seq2seq.embedding_attention_seq2seq()
把用到的部分取出来单独调试
'''

batch_size = 16
dec_seq_len = 8#图片对应的汉字数8
enc_lstm_dim = 256
dec_lstm_dim = 512
vocab_size = 1002
embedding_size = 100
lr = 0.01
global_step = tf.Variable(0)

cnn = tf.truncated_normal([16,10,35,64],mean=0,stddev=1.0,dtype=tf.float32)#模拟初始化一个cnn提取特征后的图片
#(batch_size,height,width,channels)(16, 10, 35, 64)
true_labels = []
#随即生成batch中图片对应的序列，无需embedding
for i in range(batch_size):
    seq_label = []
    for j in range(dec_seq_len):
        seq_label.append(random.randint(0,1000))
    true_labels.append(seq_label)


#编码
def encoder(inp):#inp:shape=(16, 35, 64)
    #pdb.set_trace()
    enc_init_shape = [batch_size, enc_lstm_dim]#[16,256]
    with tf.variable_scope('encoder_rnn'):
        with tf.variable_scope('forward'):
            lstm_cell_fw = tf.nn.rnn_cell.LSTMCell(enc_lstm_dim)
            init_fw = tf.nn.rnn_cell.LSTMStateTuple(\
                                tf.get_variable("enc_fw_c", enc_init_shape),\
                                tf.get_variable("enc_fw_h", enc_init_shape)
                                )
        with tf.variable_scope('backward'):
            lstm_cell_bw = tf.nn.rnn_cell.LSTMCell(enc_lstm_dim)
            init_bw = tf.nn.rnn_cell.LSTMStateTuple(\
                                tf.get_variable("enc_bw_c", enc_init_shape),\
                                tf.get_variable("enc_bw_h", enc_init_shape)
                                )
        output, _ = tf.nn.bidirectional_dynamic_rnn(lstm_cell_fw, \
                                                    lstm_cell_bw, \
                                                    inp, \
                                                    sequence_length = tf.fill([batch_size],\
                                                    tf.shape(inp)[1]), #(35,35,35...,35,35,35)
                                                    initial_state_fw = init_fw, \
                                                    initial_state_bw = init_bw \
                                                    )#shape=(16, 35, 256)
    return tf.concat(2,output)##shape=(16, 35, 512)

encoder = tf.make_template('fun', encoder)
# shape is (batch size, rows, columns, features)
# swap axes so rows are first. map splits tensor on first axis, so encoder will be applied to tensors
# of shape (batch_size,time_steps,feat_size)
rows_first = tf.transpose(cnn,[1,0,2,3])#shape=(10, 16, 35, 64)
res = tf.map_fn(encoder, rows_first, dtype=tf.float32)#shape=(10, 16, 35, 512)
encoder_output = tf.transpose(res,[1,0,2,3])#shape=(16, 10, 35, 512)

dec_lstm_cell = tf.nn.rnn_cell.LSTMCell(dec_lstm_dim)
dec_init_shape = [batch_size, dec_lstm_dim]
dec_init_state = tf.nn.rnn_cell.LSTMStateTuple( tf.truncated_normal(dec_init_shape),\
                                                tf.truncated_normal(dec_init_shape) )

init_words = np.zeros([batch_size,1,vocab_size])#(16, 1, 1002)


#pdb.set_trace()
(output,state) = decoder.embedding_attention_decoder(dec_init_state,#[16, 512]第一个解码cell的state=[c,h]
                                                    tf.reshape(encoder_output,[batch_size, -1,2*enc_lstm_dim]),
                                                    #encoder输出reshape为 attention states作为attention模块的输入 shape=(16,350,512)
                                                    dec_lstm_cell,#lstm单元，作为解码层
                                                    vocab_size,#1002
                                                    dec_seq_len,#8
                                                    batch_size,#16
                                                    embedding_size,#100
                                                    feed_previous=True)#dec_seq_len = num_words　= time_steps
pdb.set_trace()
cross_entropy = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(output,true_labels))
learning_rate = tf.train.exponential_decay(lr, global_step, 50, 0.9)
train_step = tf.train.AdamOptimizer(learning_rate).minimize(cross_entropy,global_step=global_step)
correct_prediction = tf.equal(tf.to_int32(tf.argmax( output, 2)), true_labels)
                                               

decode.py

#-*-coding:utf8-*-


"""
截取自tensorflow seq2seq.py 文件
"""
import numpy as np
import tensorflow as tf
import pdb
from tensorflow.python import shape
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import control_flow_ops
from tensorflow.python.ops import embedding_ops
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import nn_ops
from tensorflow.python.ops import rnn
from tensorflow.python.ops import rnn_cell
from tensorflow.python.ops import variable_scope
from tensorflow.python.util import nest

linear = rnn_cell._linear    # pylint: disable=protected-access

def attention_decoder(initial_state,#(16, 512)
                      attention_states,#shape=(16, 350, 512)
                      cell,
                      vocab_size,#1002
                      time_steps,#num_words,8
                      batch_size,#16
                      output_size=None,#512
                      loop_function=None,
                      dtype=None,
                      scope=None):
    pdb.set_trace()
    if attention_states.get_shape()[2].value is None:#tf 张量　get_shape()方法获取size
        raise ValueError("Shape[2] of attention_states must be known: %s"
                                         % attention_states.get_shape())
    if output_size is None:
        output_size = cell.output_size#512

    with variable_scope.variable_scope(scope or "attention_decoder", dtype=dtype) as scope:
        dtype = scope.dtype

        attn_length = attention_states.get_shape()[1].value #350
        if attn_length is None:
            attn_length = shape(attention_states)[1]
        attn_size = attention_states.get_shape()[2].value#512

        # To calculate W1 * h_t we use a 1-by-1 convolution, need to reshape before.
        hidden = array_ops.reshape(attention_states, [-1, attn_length, 1, attn_size])#shape=(16, 350, 1, 512) 
        attention_vec_size = attn_size    # Size of query vectors for attention.   512
        k = variable_scope.get_variable("AttnW",[1, 1, attn_size, attention_vec_size])#shape=(1,1,512,512)
        hidden_features = nn_ops.conv2d(hidden, k, [1, 1, 1, 1], "SAME")#(16 ,350, 1, 512) w_1*h_j
        v = variable_scope.get_variable("AttnV", [attention_vec_size])


        def attention(query):
            #LSTMStateTuple(c= shape=(16, 512) dtype=float32>, h=< shape=16, 512) dtype=float32>)
            """Put attention masks on hidden using hidden_features and query."""
            if nest.is_sequence(query):    # If the query is a tuple, flatten it.
                query_list = nest.flatten(query) #[c,h]，第一个随即初始化，以后调用之前计算的
                for q in query_list:    # Check that ndims == 2 if specified.
                    ndims = q.get_shape().ndims
                    if ndims:
                        assert ndims == 2
                query = array_ops.concat(1, query_list)# shape=(16, 1024)
            with variable_scope.variable_scope("Attention_0"):
                y = linear(query, attention_vec_size, True)# shape=(16, 512) w_2*s_t
                y = array_ops.reshape(y, [-1, 1, 1, attention_vec_size]) # shape=(16, 1, 1, 512)
                s = math_ops.reduce_sum(
                        v * math_ops.tanh(hidden_features + y), [2, 3])  #!!!!!!!!!!!公式(3)shape=(16, 350)
                a = nn_ops.softmax(s)#  公式(2)shape=(16, 350)
                # Now calculate the attention-weighted vector d.
                d = math_ops.reduce_sum(
                        array_ops.reshape(a, [-1, attn_length, 1, 1]) * hidden,#公式(1)
                        [1, 2])#shape=(16, 512) 
                ds = array_ops.reshape(d, [-1, attn_size])#shape=(16, 512) #!!!!!!!!!!!!以上是attention　model中三个关键公式的实现
            return ds
        #pdb.set_trace()
        prev = array_ops.zeros([batch_size,output_size])# shape=(16, 512) cell层第一个cell启动计算所需输入，
                                                        #随机初始化，以后的cell调用之前的计算结果
        batch_attn_size = array_ops.pack([batch_size, attn_size]) #(2,?)
        attn = array_ops.zeros(batch_attn_size, dtype=dtype)#shape=(16, 512)
        attn.set_shape([None, attn_size])#(16,512)

        def cond(time_step, prev_o_t, prev_softmax_input, state_c, state_h, outputs2):
            return time_step < time_steps

        def body(time_step, prev_o_t, prev_softmax_input, state_c, state_h, outputs2):#prev_o_t=prev:shape=(16,512) 
                                                #outputs:shape=(16, ?, 1002) prev_softmax_input=init_word:shape=(16, 1002)
            state = tf.nn.rnn_cell.LSTMStateTuple(state_c,state_h)#第一次随机初始状态，之后调用之前的
            pdb.set_trace()
            with variable_scope.variable_scope("loop_function", reuse=True):
                inp = loop_function(prev_softmax_input, time_step)#shape=(16,100) inp用来做什么 作为每个cell单元从下而
                #来的输入？？而prev_o_t则为从左而来的输入？？而且Inp和上一个cell单元的softmax_input(最终进softmax之前的cell输出)有关（prev_softmax_input）

            input_size = inp.get_shape().with_rank(2)[1]#100
            if input_size.value is None:
                raise ValueError("Could not infer input size from input: %s" % inp.name)
            x = tf.concat(1,[inp,prev_o_t])#shape=(16, 612) 　这个地方inp ,prev_o_t = loop_function(softmax_output),output
            # Run the RNN.
            cell_output, state = cell(x, state)#decoder层512个lstm单元 cell_output:shape=(16, 512) state:shape=(16, 512)
            # Run the attention mechanism.
            attn = attention(state)#shape=(16, 512) attenion模块的输出,C_i

            with variable_scope.variable_scope("AttnOutputProjection"):
                output = math_ops.tanh(linear([cell_output, attn], output_size, False))#shape=(16, 512) y_i = f(C_i,S_i)
                with variable_scope.variable_scope("FinalSoftmax"):
                    softmax_input = linear(output,vocab_size,False)#shape=(16, 1002) #decoder层后加一层softmax??作为softmax_input

            new_outputs = tf.concat(1, [outputs2,tf.expand_dims(softmax_input,1)])#shape=(16, ?, 1002)[,...y_t-1,y_t,...]
            return (time_step + tf.constant(1, dtype=tf.int32),\
                            output, softmax_input, state.c, state.h, new_outputs)#既是输出，又是下一轮的输入

        time_step = tf.constant(0, dtype=tf.int32)
        shape_invariants = [time_step.get_shape(),\
                            prev.get_shape(),\
                            tf.TensorShape([batch_size, vocab_size]),\
                            tf.TensorShape([batch_size,512]),\
                            tf.TensorShape([batch_size,512]),\
                            tf.TensorShape([batch_size, None, vocab_size])]

# START keyword is 0
        init_word = np.zeros([batch_size, vocab_size])#shape=(16,1002)

        loop_vars = [time_step,\
                     prev,\
                     tf.constant(init_word, dtype=tf.float32),\
                     initial_state.c,initial_state.h,\
                     tf.zeros([batch_size,1,vocab_size])] 

        outputs = tf.while_loop(cond, body, loop_vars, shape_invariants)##shape=(16, ?, 1002)
        '''
        loop_vars = [...]
        while cond(*loop_vars):
            loop_vars = body(*loop_vars)   
        '''

    return outputs[-1][:,1:], tf.nn.rnn_cell.LSTMStateTuple(outputs[-3],outputs[-2])

def embedding_attention_decoder(initial_state,#shape=(16, 512)
                                attention_states,# shape=(16, 350, 512)
                                cell,#定义的lstm单元
                                num_symbols,#1002
                                time_steps,
                                batch_size,#16
                                embedding_size,#100
                                output_size=None,#512
                                output_projection=None,
                                feed_previous=False,#True
                                update_embedding_for_previous=True,
                                dtype=None,
                                scope=None):
    if output_size is None:
        output_size = cell.output_size#512
    if output_projection is not None:
        proj_biases = ops.convert_to_tensor(output_projection[1], dtype=dtype)
        proj_biases.get_shape().assert_is_compatible_with([num_symbols])

    with variable_scope.variable_scope(scope or "embedding_attention_decoder", dtype=dtype) as scope:
        embedding = variable_scope.get_variable("embedding",[num_symbols, embedding_size])
        loop_function = tf.nn.seq2seq._extract_argmax_and_embed(embedding, 
                          output_projection,update_embedding_for_previous) if feed_previous else None
                        #(16,1002)==>(16,100)找argmax,然后embedding
        return attention_decoder(
                initial_state,
                attention_states,
                cell,
                num_symbols,#1002
                time_steps,#8
                batch_size,
                output_size=output_size,#512
                loop_function=loop_function)

 

关于embedding接口:

测试如下:

#-*-coding:utf8-*-

__author = "buyizhiyou"
__date = "2017-11-21"

import tensorflow as tf
import numpy as np

'''
测试embedding接口
'''
embedding = tf.Variable(np.identity(5,dtype=np.int32))
inputs = tf.placeholder(dtype=tf.int32,shape=[None])
input_embedding = tf.nn.embedding_lookup(embedding,inputs)

with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())
    print(sess.run(embedding))
'''
[[1 0 0 0 0]
 [0 1 0 0 0]
 [0 0 1 0 0]
 [0 0 0 1 0]
 [0 0 0 0 1]]
'''
    print(sess.run(input_embedding,feed_dict={inputs:[1,2,3,0,3,2,1]}))
'''
[[0 1 0 0 0]
 [0 0 1 0 0]
 [0 0 0 1 0]
 [1 0 0 0 0]
 [0 0 0 1 0]
 [0 0 1 0 0]
 [0 1 0 0 0]]
'''