6.1 处理文本数据
6.1.1 单词和字符的one-hot编码
(1)单词级的one-hot编码：
1 # 单词级的one-hot编码 2 import numpy as np 3 4 # 初始数据：每个样本是列表的一个元素(本例中的样本是一个句子，但也可以是一整篇文档) 5 samples = ['The cat sat on the mat.', 'The dog ate my homework.'] 6 7 # 构建数据中所有标记的索引 8 token_index = {} 9 for sample in samples: 10 # 利用split方法对样本进行分词，在实际应用中，还需要从样本中去掉标点和特殊符号。 11 for word in sample.split(): 12 if word not in token_index: 13 # 为每一个唯一的单词分配一个唯一的索引 14 token_index[word] = len(token_index) + 1 15 # 0号索引没有分配给任何单词 16 17 # 对样本进行分词。只考虑每个样本前max_length个单词 18 max_length = 10 19 20 # 将结果保存在result中 21 # result是一个3D张量，第一维(高)是样本个数，第二维(行)是某样本第几个单词， 22 # 第三维(列)是这个单词的向量表示 23 results = np.zeros((len(samples), max_length, max(token_index.values()) + 1)) 24 for i, sample in enumerate(samples): 25 for j, word in list(enumerate(sample.split()))[:max_length]: 26 index = token_index.get(word) 27 results[i, j, index] = 1.#将3D向量中出现的单词标记为1.
(2)字符级的one-hot编码：
1 import string 2 import numpy as np 3 4 samples = ['The cat sat on the mat.', 'The dog ate my homework.'] 5 characters = string.printable # 所有可打印的ASCII字符 6 token_index = dict(zip(characters, range(1, len(characters) + 1))) 7 8 max_length = 50 9 results = np.zeros((len(samples), max_length, max(token_index.values()) + 1)) 10 for i, sample in enumerate(samples): 11 for j, character in enumerate(sample[:max_length]): 12 index = token_index.get(character) 13 results[i, j, index] = 1.
(3)使用Keras实现单词级的one-hot编码：
1 from keras.preprocessing.text import Tokenizer 2 3 samples = ['The cat sat on the mat.', 'The dog ate my homework.'] 4 5 # 创建一个分词器，设置为只考虑前1000个最常见的单词 6 tokenizer = Tokenizer(num_words=1000) 7 # 构建单词索引 8 tokenizer.fit_on_texts(samples) 9 10 # 将字符串转换为整数索引组成的列表 11 sequences = tokenizer.texts_to_sequences(samples) 12 13 # 也可以直接得到one-hot二进制表示。这个分词器也支持除one-hot编码外的其他向量化模式 14 one_hot_results = tokenizer.texts_to_matrix(samples, mode='binary') 15 16 # 找回单词索引 17 word_index = tokenizer.word_index 18 print('Found %s unique tokens.' % len(word_index))
(4)使用散列技巧的单词级的one-hot编码：
1 import numpy as np 2 3 samples = ['The cat sat on the mat.', 'The dog ate my homework.'] 4 5 # 将单词保存为长度为1000的向量。如果单词数量接近1000个(或更多)， 6 # 那么会遇到很多散列冲突，这回降低这种编码方法的准确性 7 dimensionality = 1000 8 max_length = 10 9 10 results = np.zeros((len(samples), max_length, dimensionality)) 11 for i, sample in enumerate(samples): 12 for j, word in list(enumerate(sample.split()))[:max_length]: 13 # 将单词散列为0~1000范围内的一个随机整数索引 14 index = abs(hash(word)) % dimensionality 15 results[i, j, index] = 1.
6.1.2 使用词嵌入
(1)利用Embedding层学习词嵌入：
1 from keras.layers import Embedding 2 3 # Embedding层至少需要两个参数：标记的个数(这里是1000，即最大单词索引+1) 4 # 和嵌入的维度(这里是64) 5 embedding_layer = Embedding(1000, 64) 6 7 from keras.datasets import imdb 8 from keras import preprocessing 9 10 # 作为特征的单词个数 11 max_features = 10000 12 # 超出的单词会被截断 13 # (这些单词是最常见单词) 14 maxlen = 20 15 16 # 加载数据，整数列表 17 (x_train, y_train), (x_test, y_test) = imdb.load_data(num_words=max_features) 18 19 # 将整数列表转换为(samples, maxlen)的2D张量 20 x_train = preprocessing.sequence.pad_sequences(x_train, maxlen=maxlen) 21 x_test = preprocessing.sequence.pad_sequences(x_test, maxlen=maxlen) 22 23 24 from keras.models import Sequential 25 from keras.layers import Flatten, Dense 26 27 model = Sequential() 28 # 指定Embedding层的最大输入长度，以便后面将嵌入输入展平。 29 30 model.add(Embedding(max_features, 8, input_length=maxlen)) 31 # Embedding层激活的形状为(samples, maxlen, 8) 32 33 # 将3D的嵌入张量展平成形状(samples, maxlen * 8)的2D张量 34 model.add(Flatten()) 35 36 # 添加分类器 37 model.add(Dense(1, activation='sigmoid')) 38 model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['acc']) 39 # model.summary() 40 41 history = model.fit(x_train, y_train, 42 epochs=10, 43 batch_size=32, 44 validation_split=0.2)
(2)使用预训练的词嵌入：
1 import os 2 3 imdb_dir = '/home/ubuntu/data/aclImdb' 4 train_dir = os.path.join(imdb_dir, 'train') 5 6 labels = [] 7 texts = [] 8 9 for label_type in ['neg', 'pos']: 10 dir_name = os.path.join(train_dir, label_type) 11 for fname in os.listdir(dir_name): 12 if fname[-4:] == '.txt': 13 f = open(os.path.join(dir_name, fname)) 14 texts.append(f.read()) 15 f.close() 16 if label_type == 'neg': 17 labels.append(0) 18 else: 19 labels.append(1) 20 21 from keras.preprocessing.text import Tokenizer 22 from keras.preprocessing.sequence import pad_sequences 23 import numpy as np 24 25 maxlen = 100 # We will cut reviews after 100 words 26 training_samples = 200 # We will be training on 200 samples 27 validation_samples = 10000 # We will be validating on 10000 samples 28 max_words = 10000 # We will only consider the top 10,000 words in the dataset 29 30 tokenizer = Tokenizer(num_words=max_words) 31 tokenizer.fit_on_texts(texts) 32 sequences = tokenizer.texts_to_sequences(texts) 33 34 word_index = tokenizer.word_index 35 print('Found %s unique tokens.' % len(word_index)) 36 37 data = pad_sequences(sequences, maxlen=maxlen) 38 39 labels = np.asarray(labels) 40 print('Shape of data tensor:', data.shape) 41 print('Shape of label tensor:', labels.shape) 42 43 # Split the data into a training set and a validation set 44 # But first, shuffle the data, since we started from data 45 # where sample are ordered (all negative first, then all positive). 46 indices = np.arange(data.shape[0]) 47 np.random.shuffle(indices) 48 data = data[indices] 49 labels = labels[indices] 50 51 x_train = data[:training_samples] 52 y_train = labels[:training_samples] 53 x_val = data[training_samples: training_samples + validation_samples] 54 y_val = labels[training_samples: training_samples + validation_samples] 55 56 glove_dir = '/home/ubuntu/data/' 57 58 embeddings_index = {} 59 f = open(os.path.join(glove_dir, 'glove.6B.100d.txt')) 60 for line in f: 61 values = line.split() 62 word = values[0] 63 coefs = np.asarray(values[1:], dtype='float32') 64 embeddings_index[word] = coefs 65 f.close() 66 67 print('Found %s word vectors.' % len(embeddings_index)) 68 69 embedding_dim = 100 70 71 embedding_matrix = np.zeros((max_words, embedding_dim)) 72 for word, i in word_index.items(): 73 embedding_vector = embeddings_index.get(word) 74 if i < max_words: 75 if embedding_vector is not None: 76 # Words not found in embedding index will be all-zeros. 77 embedding_matrix[i] = embedding_vector 78 79 from keras.models import Sequential 80 from keras.layers import Embedding, Flatten, Dense 81 82 model = Sequential() 83 model.add(Embedding(max_words, embedding_dim, input_length=maxlen)) 84 model.add(Flatten()) 85 model.add(Dense(32, activation='relu')) 86 model.add(Dense(1, activation='sigmoid')) 87 model.summary() 88 89 model.layers[0].set_weights([embedding_matrix]) 90 model.layers[0].trainable = False 91 92 model.compile(optimizer='rmsprop', 93 loss='binary_crossentropy', 94 metrics=['acc']) 95 history = model.fit(x_train, y_train, 96 epochs=10, 97 batch_size=32, 98 validation_data=(x_val, y_val)) 99 model.save_weights('pre_trained_glove_model.h5') 100 101 import matplotlib.pyplot as plt 102 103 acc = history.history['acc'] 104 val_acc = history.history['val_acc'] 105 loss = history.history['loss'] 106 val_loss = history.history['val_loss'] 107 108 epochs = range(1, len(acc) + 1) 109 110 plt.plot(epochs, acc, 'bo', label='Training acc') 111 plt.plot(epochs, val_acc, 'b', label='Validation acc') 112 plt.title('Training and validation accuracy') 113 plt.legend() 114 115 plt.figure() 116 117 plt.plot(epochs, loss, 'bo', label='Training loss') 118 plt.plot(epochs, val_loss, 'b', label='Validation loss') 119 plt.title('Training and validation loss') 120 plt.legend() 121 122 plt.show() 123 124 from keras.models import Sequential 125 from keras.layers import Embedding, Flatten, Dense 126 127 model = Sequential() 128 model.add(Embedding(max_words, embedding_dim, input_length=maxlen)) 129 model.add(Flatten()) 130 model.add(Dense(32, activation='relu')) 131 model.add(Dense(1, activation='sigmoid')) 132 model.summary() 133 134 model.compile(optimizer='rmsprop', 135 loss='binary_crossentropy', 136 metrics=['acc']) 137 history = model.fit(x_train, y_train, 138 epochs=10, 139 batch_size=32, 140 validation_data=(x_val, y_val)) 141 142 acc = history.history['acc'] 143 val_acc = history.history['val_acc'] 144 loss = history.history['loss'] 145 val_loss = history.history['val_loss'] 146 147 epochs = range(1, len(acc) + 1) 148 149 plt.plot(epochs, acc, 'bo', label='Training acc') 150 plt.plot(epochs, val_acc, 'b', label='Validation acc') 151 plt.title('Training and validation accuracy') 152 plt.legend() 153 154 plt.figure() 155 156 plt.plot(epochs, loss, 'bo', label='Training loss') 157 plt.plot(epochs, val_loss, 'b', label='Validation loss') 158 plt.title('Training and validation loss') 159 plt.legend() 160 161 plt.show() 162 163 164 test_dir = os.path.join(imdb_dir, 'test') 165 166 labels = [] 167 texts = [] 168 169 for label_type in ['neg', 'pos']: 170 dir_name = os.path.join(test_dir, label_type) 171 for fname in sorted(os.listdir(dir_name)): 172 if fname[-4:] == '.txt': 173 f = open(os.path.join(dir_name, fname)) 174 texts.append(f.read()) 175 f.close() 176 if label_type == 'neg': 177 labels.append(0) 178 else: 179 labels.append(1) 180 181 sequences = tokenizer.texts_to_sequences(texts) 182 x_test = pad_sequences(sequences, maxlen=maxlen) 183 y_test = np.asarray(labels) 184 185 186 model.load_weights('pre_trained_glove_model.h5') 187 model.evaluate(x_test, y_test)
