NLP（二十二）使用LSTM进行语言建模以预测最优词
原文链接：http://www.one2know.cn/nlp22/
- 预处理
数据集使用Facebook上的BABI数据集
将文件提取成可训练的数据集，包括：文章 问题 答案
def get_data(infile):
stories,questions,answers = [],[],[]
story_text = []
fin = open(infile,'rb')
for line in fin:
line = line.decode('utf-8').strip()
lno,text = line.split(' ',1)
if '\t' in text:
question,answer,_ = text.split('\t')
stories.append(story_text)
questions.append(question)
answers.append(answer)
story_text = []
else:
story_text.append(text)
fin.close()
return stories,questions,answers
data_train = get_data('qa1_single-supporting-fact_train.txt')
data_test = get_data('qa1_single-supporting-fact_test.txt')
print('\nTrain observations:',len(data_train[0]),
'Test observations:',len(data_test[0]),'\n')
输出：
Train observations: 10000 Test observations: 1000
- 如何实现
1.预处理：创建字典并将文章，问题和答案映射到词表，进一步映射成向量形式
2.模型创建和验证：训练模型并在验证数据集上测试
3.预测结果：测试集测试数据的结果 - 代码
from __future__ import division,print_function
import collections
import itertools
import nltk
import numpy as np
import matplotlib.pyplot as plt
import os
import random
def get_data(infile):
stories,questions,answers = [],[],[]
story_text = []
fin = open(infile,'rb')
for line in fin:
line = line.decode('utf-8').strip()
lno,text = line.split(' ',1) # 去掉前面的数字标记
if '\t' in text: # 有制表符的是 问题 和 答案
question,answer,_ = text.split('\t')
stories.append(story_text)
questions.append(question)
answers.append(answer)
story_text = []
else: # 没制表符的是文章
story_text.append(text)
fin.close()
return stories,questions,answers
data_train = get_data('qa1_single-supporting-fact_train.txt')
data_test = get_data('qa1_single-supporting-fact_test.txt')
print('\nTrain observations:',len(data_train[0]),
'Test observations:',len(data_test[0]),'\n')
print(data_train[0][1],data_train[1][1],data_train[2][1])
# ['Daniel went back to the hallway.', 'Sandra moved to the garden.'] Where is Daniel? hallway
print(np.array(data_train).shape)
# (3, 10000)
dictnry = collections.Counter() # 返回列表元素出现次数的 字典，这里没有参数是一个空字典
for stories,questions,answers in [data_train,data_test]:
for story in stories:
for sent in story:
for word in nltk.word_tokenize(sent):
dictnry[word.lower()] += 1
for question in questions:
for word in nltk.word_tokenize(question):
dictnry[word.lower()] += 1
for answer in answers:
for word in nltk.word_tokenize(answer):
dictnry[word.lower()] += 1
word2indx = {w:(i+1) for i,(w,_) in enumerate(dictnry.most_common())} # 按词频排序
word2indx['PAD'] = 0
indx2word = {v:k for k,v in word2indx.items()}
vocab_size = len(word2indx) # 一共有22个不重复单词
print('vocabulary size:',len(word2indx))
story_maxlen = 0
question_maxlen = 0
for stories,questions,answers in [data_train,data_test]:
for story in stories:
story_len = 0
for sent in story:
swords = nltk.word_tokenize(sent)
story_len += len(swords)
if story_len > story_maxlen:
story_maxlen = story_len
for question in questions:
question_len = len(nltk.word_tokenize(question))
if question_len > question_maxlen:
question_maxlen = question_len
print('Story maximum length:',story_maxlen,'Question maximum length:',question_maxlen)
# 文章单词最大长度为14，问题中的单词最大长度为4，长度不够的补0，维度相同便于并向计算
from keras.layers import Input
from keras.layers.core import Activation,Dense,Dropout,Permute
from keras.layers.embeddings import Embedding
from keras.layers.merge import add,concatenate,dot
from keras.layers.recurrent import LSTM
from keras.models import Model
from keras.preprocessing.sequence import pad_sequences
from keras.utils import np_utils
def data_vectorization(data,word2indx,story_maxlen,question_maxlen): # 词 => 词向量
Xs,Xq,Y = [],[],[]
stories,questions,answers = data
for story,question,answer in zip(stories,questions,answers):
xs = [[word2indx[w.lower()] for w in nltk.word_tokenize(s)] for s in story] #
xs = list(itertools.chain.from_iterable(xs))
xq = [word2indx[w.lower()] for w in nltk.word_tokenize(question)]
Xs.append(xs)
Xq.append(xq)
Y.append(word2indx[answer.lower()])
return pad_sequences(Xs,maxlen=story_maxlen),pad_sequences(Xq,maxlen=question_maxlen),\
np_utils.to_categorical(Y,num_classes=len(word2indx))
Xstrain,Xqtrain,Ytrain = data_vectorization(data_train,word2indx,story_maxlen,question_maxlen)
Xstest,Xqtest,Ytest = data_vectorization(data_test,word2indx,story_maxlen,question_maxlen)
print('Train story',Xstrain.shape,'Train question',Xqtrain.shape,'Train answer',Ytrain.shape)
print('Test story',Xstest.shape,'Test question',Xqtest.shape,'Test answer',Ytest.shape)
# 超参数
EMBEDDING_SIZE = 128
LATENT_SIZE = 64
BATCH_SIZE = 64
NUM_EPOCHS = 40
# 输入层
story_input = Input(shape=(story_maxlen,))
question_input = Input(shape=(question_maxlen,))
# Story encoder embedding
# 将正整数（索引）转换为固定大小的密集向量。
# 例如，[[4]，[20]]->[[0.25，0.1]，[0.6，-0.2]] 此层只能用作模型中的第一层
story_encoder = Embedding(input_dim=vocab_size,output_dim=EMBEDDING_SIZE,input_length=story_maxlen)(story_input)
story_encoder = Dropout(0.2)(story_encoder)
# Question encoder embedding
question_encoder = Embedding(input_dim=vocab_size,output_dim=EMBEDDING_SIZE,input_length=question_maxlen)(question_input)
question_encoder = Dropout(0.3)(question_encoder)
# 返回两个张量的点积
match = dot([story_encoder,question_encoder],axes=[2,2])
# 将故事编码为问题的向量空间
story_encoder_c = Embedding(input_dim=vocab_size,output_dim=question_maxlen,input_length=story_maxlen)(story_input)
story_encoder_c = Dropout(0.3)(story_encoder_c)
# 结合两个向量 match和story_encoder_c
response = add([match,story_encoder_c])
response = Permute((2,1))(response)
# 结合两个向量 response和question_encoder
answer = concatenate([response, question_encoder], axis=-1)
answer = LSTM(LATENT_SIZE)(answer)
answer = Dropout(0.2)(answer)
answer = Dense(vocab_size)(answer)
output = Activation("softmax")(answer)
model = Model(inputs=[story_input, question_input], outputs=output)
model.compile(optimizer="adam", loss="categorical_crossentropy",metrics=["accuracy"])
print(model.summary())
# 模型训练
history = model.fit([Xstrain,Xqtrain],[Ytrain],batch_size=BATCH_SIZE,epochs=NUM_EPOCHS,
validation_data=([Xstest,Xqtest],[Ytest]))
# 画出准确率和损失函数
plt.title('Episodic Memory Q&A Accuracy')
plt.plot(history.history['acc'],color='g',label='train')
plt.plot(history.history['val_acc'],color='r',label='validation')
plt.legend(loc='best')
plt.show()
# get predictions of labels
ytest = np.argmax(Ytest, axis=1)
Ytest_ = model.predict([Xstest, Xqtest])
ytest_ = np.argmax(Ytest_, axis=1)
# 随机选择几个问题测试
NUM_DISPLAY = 10
for i in random.sample(range(Xstest.shape[0]),NUM_DISPLAY):
story = " ".join([indx2word[x] for x in Xstest[i].tolist() if x != 0])
question = " ".join([indx2word[x] for x in Xqtest[i].tolist()])
label = indx2word[ytest[i]]
prediction = indx2word[ytest_[i]]
print(story, question, label, prediction)
输出：
