import sys
# import keras
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
from tensorflow.keras import layers
import os
import matplotlib.pyplot as plt
# 设置相关底层配置
physical_devices = tf.config.experimental.list_physical_devices('GPU')
assert len(physical_devices) > 0, "Not enough GPU hardware devices available"
tf.config.experimental.set_memory_growth(physical_devices[0], True)
def my_padding(x,size):
result = np.array([])
bat_size = x.shape[0]
for i in range(bat_size):
zero_mat = np.zeros((size - x.shape[1],x.shape[2]))
start_mat = np.ones((1,x.shape[2])) * 0.3
end_mat = np.ones((1,x.shape[2])) * 0.3
each_x = x[i]
each_x = np.vstack([start_mat,each_x])
pad_x = np.vstack([each_x,zero_mat])
pad_x = np.vstack([pad_x,end_mat])
pad_x = pad_x.reshape(1, size+2, x.shape[2])
if result.shape[0] == 0:
result = pad_x
else:
result = np.vstack([result,pad_x])
return result
# test_x = np.array([
# [[111, 112, 113],
# [121, 122, 123],
# ],
# [[211, 212, 213],
# [221, 222, 223],
# ],
# [[311, 312, 313],
# [321, 322, 323],
# ],
# ])
# a = my_padding(test_x,6)
# print(a.shape)
# print(a)
# sys.exit(2)
def get_angles(pos, i, d_model):
# 这里的i等价与上面公式中的2i和2i+1
angle_rates = 1 / np.power(10000, (2*(i // 2))/ np.float32(d_model))
return pos * angle_rates
def positional_encoding(position, d_model):
# print('position:',position)
angle_rads = get_angles(np.arange(position)[:, np.newaxis],
np.arange(d_model)[np.newaxis, :],
d_model)
# 第2i项使用sin
sines = np.sin(angle_rads[:, 0::2])
# 第2i+1项使用cos
cones = np.cos(angle_rads[:, 1::2])
pos_encoding = np.concatenate([sines, cones], axis=-1)
pos_encoding = pos_encoding[np.newaxis, ...]
return tf.cast(pos_encoding, dtype=tf.float32)
# pos_encoding = positional_encoding(50, 512)
# def create_padding_mark(seq):
# # 获取为0的padding项
# # print('seq:', seq)
# seq = tf.cast(tf.math.equal(seq, 0), tf.float32)
# # 扩充维度以便用于attention矩阵
# return seq[:, np.newaxis, np.newaxis, :] # (batch_size,1,1,seq_len)
# # print('seq_mask:',seq)
# return seq # (batch_size,seq_len)
def create_padding_mark(targets):
# 获取为0的padding项
# print('targets:',targets.shape)
zero_mask = np.max(targets,axis=-1)
zero_mask = tf.cast(tf.math.equal(zero_mask, 0), tf.float32)
zero_mask = tf.reshape(zero_mask,(-1,1))
one_mat = np.ones((targets.shape[0],targets.shape[0]))
# print('zero_mask:',zero_mask.shape,zero_mask)
result_mask1 = np.multiply(one_mat,zero_mask)
result_mask2 = np.multiply(one_mat, tf.transpose(zero_mask))
result_mask = tf.maximum(result_mask1,result_mask2)
# print('result_mask:', result_mask.shape, result_mask)
# sys.exit(2)
return result_mask # (batch_size,seq_len)
# mark 测试
# create_padding_mark([[1,2,0,0,3],[3,4,5,0,0],[0,0,0,0,0]])
# sys.exit(2)
def create_look_ahead_mark(size):
# print('size:',size)
# 1 - 对角线和取下三角的全部对角线（-1->全部）
# 这样就可以构造出每个时刻未预测token的掩码
mark = 1 - tf.linalg.band_part(tf.ones((size, size)), -1, 0) # (a, numLower, numUpper)
# print('mark.shapeL:',mark.shape)
# sys.exit(2)
# mark = np.reshape(mark,())
mark = tf.cast(mark,tf.float32)
return mark # (seq_len, seq_len)
# temp = create_look_ahead_mark(3)
# print(temp)
# sys.exit(2)
# 构建掩码
def create_mask(targets):
# loss_mark = np.max(targets,axis=-1)
# loss_mark = tf.cast(tf.math.equal(loss_mark, 0), tf.float32)
# loss_mark = tf.reshape(loss_mark,(-1,1))
# # print('loss_mark:',loss_mark,loss_mark.shape)
# decode_targets_padding_mask = create_padding_mark(targets)
# decode_targets_padding_mask = tf.cast(decode_targets_padding_mask,tf.float32)
# look_ahead 掩码， 掩掉未预测的词
look_ahead_mask = create_look_ahead_mark(targets.shape[0])
look_ahead_mask = tf.cast(look_ahead_mask , tf.float32)
# print('此粗话'k:', look_ahead_mask.shape,type(look_ahead_mask.shape))
# 合并解码层第一层掩码
# combine_mask = tf.maximum(decode_targets_padding_mask, look_ahead_mask)
# print('combine_m
# return combine_mask, loss_mark
return look_ahead_mask
# sys.exit(2)
def scaled_dot_product_attention(q, k, v, mask):
# query key 相乘获取匹配关系
matmul_qk = tf.matmul(q, k, transpose_b=True)
# 使用dk进行缩放
dk = tf.cast(tf.shape(k)[-1], tf.float32)
scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)
# 掩码
if mask is not None:
scaled_attention_logits += (mask * -1e9)
# 通过softmax获取attention权重
attention_weights = tf.nn.softmax(scaled_attention_logits, axis=-1)
# attention 乘上value
output = tf.matmul(attention_weights, v) # （.., seq_len_v, depth）
return output, attention_weights
# attention测试
# def print_out(q, k, v):
# temp_out, temp_att = scaled_dot_product_attention(
# q, k, v, None)
# print('attention weight:')
# print(temp_att)
# print('output:')
# print(temp_out)
# np.set_printoptions(suppress=True)
# temp_k = tf.constant([[10,0,0],
# [0,10,0],
# [0,0,10],
# [0,0,10]], dtype=tf.float32) # (4, 3)
#
# temp_v = tf.constant([[ 1,0],
# [ 10,0],
# [ 100,5],
# [1000,6]], dtype=tf.float32) # (4, 3)
# # 关注第2个key, 返回对应的value
# temp_q = tf.constant([[0,10,0]], dtype=tf.float32)
# print_out(temp_q, temp_k, temp_v)
# 构造mutil head attention层
class MutilHeadAttention(tf.keras.layers.Layer):
def __init__(self, d_model, num_heads):
super(MutilHeadAttention, self).__init__()
self.num_heads = num_heads
self.d_model = d_model
# d_model 必须可以正确分为各个头
assert d_model % num_heads == 0
# 分头后的维度
self.depth = d_model // num_heads
self.wq = tf.keras.layers.Dense(d_model)
self.wk = tf.keras.layers.Dense(d_model)
self.wv = tf.keras.layers.Dense(d_model)
self.dense = tf.keras.layers.Dense(d_model)
def split_heads(self, x, batch_size):
# 分头, 将头个数的维度 放到 seq_len 前面
x = tf.reshape(x, (batch_size, -1, self.num_heads, self.depth))
return tf.transpose(x, perm=[0, 2, 1, 3])
def call(self, v, k, q, mask):
batch_size = tf.shape(q)[0]
# 分头前的前向网络，获取q、k、v语义
# print('or_q:',q.shape)
q = self.wq(q) # (batch_size, seq_len, d_model)
# print('or_q:', q.shape)
# sys.exit(2)
k = self.wk(k)
v = self.wv(v)
# 分头
q = self.split_heads(q, batch_size) # (batch_size, num_heads, seq_len_q, depth)
k = self.split_heads(k, batch_size)
v = self.split_heads(v, batch_size)
# scaled_attention.shape == (batch_size, num_heads, seq_len_v, depth)
# attention_weights.shape == (batch_size, num_heads, seq_len_q, seq_len_k)
# 通过缩放点积注意力层
scaled_attention, attention_weights = scaled_dot_product_attention(
q, k, v, mask)
# 把多头维度后移
scaled_attention = tf.transpose(scaled_attention, [0, 2, 1, 3]) # (batch_size, seq_len_v, num_heads, depth)
# 合并多头
concat_attention = tf.reshape(scaled_attention,
(batch_size, -1, self.d_model))
# 全连接重塑
output = self.dense(concat_attention)
return output, attention_weights
# temp_mha = MutilHeadAttention(d_model=512, num_heads=8)
# # y = tf.random.uniform((1, 60, 512))
# y = tf.random.uniform((2, 60, 512))
# # y = tf.random.uniform((1, 2,60, 32))
# output, att = temp_mha(y, k=y, q=y, mask=None)
# print('x:{}'.format(y.shape))
# print("out:{},att:{}".format(output.shape, att.shape))
def point_wise_feed_forward_network(d_model, diff):
return tf.keras.Sequential([
tf.keras.layers.Dense(diff, activation='relu'),
tf.keras.layers.Dense(d_model)
])
class LayerNormalization(tf.keras.layers.Layer):
def __init__(self, epsilon=1e-6, **kwargs):
self.eps = epsilon
super(LayerNormalization, self).__init__(**kwargs)
def build(self, input_shape):
self.gamma = self.add_weight(name='gamma', shape=input_shape[-1:],
initializer=tf.ones_initializer(), trainable=True)
self.beta = self.add_weight(name='beta', shape=input_shape[-1:],
initializer=tf.zeros_initializer(), trainable=True)
super(LayerNormalization, self).build(input_shape)
def call(self, x):
mean = tf.keras.backend.mean(x, axis=-1, keepdims=True)
std = tf.keras.backend.std(x, axis=-1, keepdims=True)
return self.gamma * (x - mean) / (std + self.eps) + self.beta
def compute_output_shape(self, input_shape):
return input_shape
class EncoderLayer(tf.keras.layers.Layer):
def __init__(self, d_model, n_heads, ddf, dropout_rate=0.1):
super(EncoderLayer, self).__init__()
self.mha = MutilHeadAttention(d_model, n_heads) # return (output, attention_weights)
self.ffn = point_wise_feed_forward_network(d_model, ddf)
self.layernorm1 = LayerNormalization(epsilon=1e-6)
self.layernorm2 = LayerNormalization(epsilon=1e-6)
self.dropout1 = tf.keras.layers.Dropout(dropout_rate)
self.dropout2 = tf.keras.layers.Dropout(dropout_rate)
########################################################### 可配一个大残差链接 ##########################################################
def call(self, inputs, training, mask):
# 多头注意力网络
att_output, _ = self.mha(inputs, inputs, inputs, mask) # return (output, attention_weights)
att_output = self.dropout1(att_output, training=training)
out1 = self.layernorm1(inputs + att_output) # (batch_size, input_seq_len, d_model)
# 前向网络
ffn_output = self.ffn(out1)
ffn_output = self.dropout2(ffn_output, training=training)
out2 = self.layernorm2(out1 + ffn_output) # (batch_size, input_seq_len, d_model)
return out2
# sample_encoder_layer = EncoderLayer(512, 8, 2048) #(d_model, n_heads, ddf, dropout_rate=0.1)
# sample_encoder_layer_output = sample_encoder_layer(tf.random.uniform((64, 43, 512)), False, None)
# print('sample_encoder_layer_output.shape:',sample_encoder_layer_output.shape)
class DecoderLayer(tf.keras.layers.Layer):
def __init__(self, d_model, num_heads, dff, drop_rate=0.1):
super(DecoderLayer, self).__init__()
self.mha1 = MutilHeadAttention(d_model, num_heads)
self.mha2 = MutilHeadAttention(d_model, num_heads)
self.ffn = point_wise_feed_forward_network(d_model, dff)
self.layernorm1 = LayerNormalization(epsilon=1e-6)
self.layernorm2 = LayerNormalization(epsilon=1e-6)
self.layernorm3 = LayerNormalization(epsilon=1e-6)
self.dropout1 = layers.Dropout(drop_rate)
self.dropout2 = layers.Dropout(drop_rate)
self.dropout3 = layers.Dropout(drop_rate)
def call(self, inputs, encode_out, training,
look_ahead_mask, padding_mask):
# masked muti-head attention
att1, att_weight1 = self.mha1(inputs, inputs, inputs, look_ahead_mask) #(v, k, q)
att1 = self.dropout1(att1, training=training)
out1 = self.layernorm1(inputs + att1)
# muti-head attention
########################################################### 此处inputs 可换为out1 ##########################################################
# att2, att_weight2 = self.mha2(encode_out, encode_out, inputs, padding_mask) #(v, k, q)
att2, att_weight2 = self.mha2(encode_out, encode_out, out1, padding_mask) # (v, k, q)
att2 = self.dropout2(att2, training=training)
out2 = self.layernorm2(out1 + att2)
ffn_out = self.ffn(out2)
ffn_out = self.dropout3(ffn_out, training=training)
out3 = self.layernorm3(out2 + ffn_out)
return out3, att_weight1, att_weight2
# sample_encoder_layer = EncoderLayer(512, 8, 2048) #(d_model, n_heads, ddf, dropout_rate=0.1)
# sample_encoder_layer_output = sample_encoder_layer(tf.random.uniform((64, 43, 512)), False, None)
# sample_decoder_layer = DecoderLayer(512, 8, 2048) # (d_model, num_heads, dff, drop_rate=0.1)
# sample_decoder_layer_output, _, _ = sample_decoder_layer(
# tf.random.uniform((64, 50, 512)), sample_encoder_layer_output,False, None, None)
# print('sample_decoder_layer_output.shape:',sample_decoder_layer_output.shape)
class Encoder(layers.Layer):
# def __init__(self, n_layers, d_model, n_heads, ddf,
# input_vocab_size, max_seq_len, drop_rate=0.1):
def __init__(self, n_layers, d_model, n_heads, ddf, drop_rate=0.1):
super(Encoder, self).__init__()
self.n_layers = n_layers
self.d_model = d_model
# self.embedding = layers.Embedding(input_vocab_size, d_model)
self.embedding = layers.Dense(d_model,activation='relu')
# self.pos_embedding = positional_encoding(max_seq_len, d_model)
self.encode_layer = [EncoderLayer(d_model, n_heads, ddf, drop_rate)
for _ in range(n_layers)]
self.dropout = layers.Dropout(drop_rate)
def call(self, inputs, training, mark):
seq_len = inputs.shape[1]
word_emb = self.embedding(inputs)
word_emb *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))
# emb = word_emb + self.pos_embedding[:, :seq_len, :]
emb = word_emb + positional_encoding(seq_len, self.d_model)
x = self.dropout(emb, training=training)
for i in range(self.n_layers):
x = self.encode_layer[i](x, training, mark)
return x
# sample_encoder = Encoder(2, 512, 8, 1024, 180) # (n_layers, d_model, n_heads, ddf, max_seq_len, drop_rate=0.1)
# print('此处')
# sample_encoder_output = sample_encoder(tf.random.uniform((64, 180,120)),False, None)
# print('sample_encoder_output.shape:',sample_encoder_output.shape)
# sys.exit(2)
class Decoder(layers.Layer):
def __init__(self, n_layers, d_model, n_heads, ddf, drop_rate=0.1):
super(Decoder, self).__init__()
self.d_model = d_model
self.n_layers = n_layers
# self.embedding = layers.Embedding(target_vocab_size, d_model)
self.embedding = layers.Dense(d_model, activation='relu')
# self.pos_embedding = positional_encoding(max_seq_len, d_model)
self.decoder_layers = [DecoderLayer(d_model, n_heads, ddf, drop_rate)
for _ in range(n_layers)]
self.dropout = layers.Dropout(drop_rate)
def call(self, inputs, encoder_out, training,
look_ahead_mark, padding_mark):
# seq_len = tf.shape(inputs)[1]
seq_len = inputs.shape[1]
attention_weights = {}
h = self.embedding(inputs)
h *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))
# h += self.pos_embedding[:, :seq_len, :]
h += positional_encoding(seq_len, self.d_model)
h = self.dropout(h, training=training)
# print('--------------------\n',h, h.shape)
# 叠加解码层
for i in range(self.n_layers):
h, att_w1, att_w2 = self.decoder_layers[i](h, encoder_out,
training, look_ahead_mark,
padding_mark)
attention_weights['decoder_layer{}_att_w1'.format(i + 1)] = att_w1
attention_weights['decoder_layer{}_att_w2'.format(i + 1)] = att_w2
return h, attention_weights
# sample_encoder = Encoder(2, 512, 8, 1024) # (n_layers, d_model, n_heads, ddf, drop_rate=0.1)
# sample_encoder_output = sample_encoder(tf.random.uniform((64, 180,120)),False, None)
# # print('sample_encoder_output.shape:',sample_encoder_output.shape)
# sample_decoder = Decoder(2, 512,8,1024) # (n_layers, d_model, n_heads, ddf, drop_rate=0.1)
# sample_decoder_output, attn = sample_decoder(tf.random.uniform((64, 180,100)),sample_encoder_output, False, None, None)
# print(sample_decoder_output.shape, attn['decoder_layer1_att_w1'].shape)
# sys.exit(2)
class Transformer(tf.keras.Model):
def __init__(self, n_layers, d_model, n_heads, diff, target_vocab_size, drop_rate=0.1):
super(Transformer, self).__init__()
# self.bn1 = layers.BatchNormalization()
self.encoder = Encoder(n_layers, d_model, n_heads, diff,drop_rate)
self.decoder = Decoder(n_layers, d_model, n_heads, diff, drop_rate)
self.bn = tf.keras.layers.BatchNormalization()
self.final_layer = tf.keras.layers.Dense(target_vocab_size)
# self.final_layer = tf.keras.layers.Dense(target_vocab_size,activation='tanh')
# def call(self, inputs, targets, training, encode_padding_mask,
# look_ahead_mask, decode_padding_mask):
def call(self, inputs, targets, training,look_ahead_mask = None, encode_padding_mask = None,
decode_padding_mask = None):
# inputs = self.bn1(inputs)
encode_out = self.encoder(inputs, training, encode_padding_mask)
# print(encode_out.shape)
decode_out, att_weights = self.decoder(targets, encode_out, training,
look_ahead_mask, decode_padding_mask)
# print('decode_out.shape:',decode_out.shape)
decode_out = self.bn(decode_out)
final_out = self.final_layer(decode_out)
# final_out = self.final_layer(decode_out) *10
return final_out, att_weights
# sample_transformer = Transformer(n_layers=2, d_model=512, n_heads=8, diff=1024,target_vocab_size=20)
# temp_input = tf.random.uniform((64,180, 62))
# temp_target = tf.random.uniform((64, 180,26))
# fn_out, _ = sample_transformer(temp_input, temp_target, training=False,
# encode_padding_mask=None,
# look_ahead_mask=None,
# decode_padding_mask=None,
# )
# print('fn_out.shape:',fn_out.shape)
# @tf.function
global_num = 0
global_train_acc = 0
def train_step(inputs, targets):
# print("^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^6")
global global_num
global global_train_acc
global_num +=1
# tar_inp = targets[:, :-1,:6]
# tar_real = targets[:, 1:,:6]
tar_inp = targets[:, :-1, 5][:,:,np.newaxis]
tar_real = targets[:, 1:, 5][:,:,np.newaxis]
# print()
with tf.GradientTape() as tape:
predictions, _ = transformer(inputs, tar_inp,training = True,look_ahead_mask=look_mask,decode_padding_mask = None)
# print("pre:{}".format(predictions[:2,:5]))
# print("True:{}".format(tar_real[:2,:5]))
# predictions, _ = transformer(inputs, tar_inp, training=True, look_ahead_mask=None,decode_padding_mask=None)
# loss = loss_fun(tar_real, predictions,loss_mask)
loss = loss_fun(tar_real, predictions)
if global_num % 10 == 0:
acc = get_acc(predictions,tar_real)
global_train_acc = acc.numpy()
# print('train_acc:{:.2f}'.format(acc.numpy()))
# 求梯度
gradients = tape.gradient(loss, transformer.trainable_variables)
# 反向传播
optimizer.apply_gradients(zip(gradients, transformer.trainable_variables))
mse_loss = tf.reduce_mean(tf.losses.MSE(predictions,tar_real))
train_loss(mse_loss)
