Pytorch-LSTM+Attention文本分类

语料链接：https://pan.baidu.com/s/1aDIp3Hxw-Xuxcx-lQ_0w9A
提取码：hpg7

train.txt  pos/neg各500条，一共1000条（用于训练模型）
dev.txt    pos/neg各100条，一共200条（用于调参数）
test.txt    pos/neg各150条，一共300条（用于测试）

例如：下面是一个正面样本的例子。
<Polarity>1</Polarity>
<text>sit back in one of those comfortable chairs.</text>

1. 数据预处理

加载数据、创建vocabulary、创建iterator

import numpy as np
import torch
import torch.nn.functional as F 
from torchtext import data
import math
import time

SEED = 123
BATCH_SIZE = 128
LEARNING_RATE = 1e-3      # 学习率
EMBEDDING_DIM = 100       # 词向量维度

# 设置device
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

# 为CPU设置随机种子
torch.manual_seed(123)

# 两个Field对象定义字段的处理方法（文本字段、标签字段）
TEXT = data.Field(tokenize=lambda x: x.split(), lower=True)
LABEL = data.LabelField(dtype=torch.float)

# get_dataset: 返回Dataset所需的 text 和 label
def get_dataset(corpus_path, text_field, label_field):
    fields = [('text', text_field), ('label', label_field)]   # torchtext文本配对关系
    examples = []
    
    with open(corpus_path) as f:
        li = []
        while True:
            content = f.readline().replace('\n', '')
            if not content:     # 为空行，表示取完一次数据(一次的数据保存在li中)
                if not li:
                    break
                label = li[0][10]
                text = li[1][6:-7]
                examples.append(data.Example.fromlist([text, label], fields=fields))
                li = []
            else:
                li.append(content)
    return examples, fields

# 得到构建Dataset所需的examples 和 fields
train_examples, train_fileds = get_dataset('./corpus/trains.txt', TEXT, LABEL)
dev_examples, dev_fields = get_dataset('./corpus/dev.txt', TEXT, LABEL)
test_examples, test_fields = get_dataset('./corpus/tests.txt', TEXT, LABEL)

# 构建Dataset数据集
train_data = data.Dataset(train_examples, train_fileds)
dev_data = data.Dataset(dev_examples, dev_fields)
test_data = data.Dataset(test_examples, test_fields)
# for t in test_data:
#     print(t.text, t.label)

print('len of train data:', len(train_data))  # 1000
print('len of dev data:', len(dev_data))      # 200
print('len of test data:', len(test_data))    # 300

# 创建vocabulary
TEXT.build_vocab(train_data, max_size=5000, vectors='glove.6B.100d')
LABEL.build_vocab(train_data)
print(len(TEXT.vocab))         # 3287
print(TEXT.vocab.itos[:12])    # ['<unk>', '<pad>', 'the', 'and', 'a', 'to', 'is', 'was', 'i', 'of', 'for', 'in']
print(TEXT.vocab.stoi['love']) # 129
# print(TEXT.vocab.stoi)         # defaultdict {'<unk>': 0, '<pad>': 1, ....}

# 创建iterators, 每个iteration都会返回一个batch的example
train_iterator, dev_iterator, test_iterator = data.BucketIterator.splits(
                                            (train_data, dev_data, test_data), 
                                            batch_size=BATCH_SIZE,
                                            device=device,
                                            sort = False)

2. 定义模型

2.1 形式一：根据注意力机制的定义求解

class BiLSTM_Attention(nn.Module):
    
    def __init__(self, vocab_size, embedding_dim, hidden_dim, n_layers):
        
        super(BiLSTM_Attention, self).__init__()
        
        self.hidden_dim = hidden_dim
        self.n_layers = n_layers
        self.embedding = nn.Embedding(vocab_size, embedding_dim)
        self.rnn = nn.LSTM(input_size=embedding_dim, hidden_size=hidden_dim, num_layers=n_layers,
                           bidirectional=True, dropout=0.5)
       
        self.fc = nn.Linear(hidden_dim * 2, 1)
        self.dropout = nn.Dropout(0.5)  
        
    # x: [batch, seq_len, hidden_dim*2]
    # query : [batch, seq_len, hidden_dim * 2]
    # 软注意力机制 (key=value=x)
    def attention_net(self, x, query, mask=None): 
        
        d_k = query.size(-1)     # d_k为query的维度
       
        # query:[batch, seq_len, hidden_dim*2], x.t:[batch, hidden_dim*2, seq_len]
#         print("query: ", query.shape, x.transpose(1, 2).shape)  # torch.Size([128, 38, 128]) torch.Size([128, 128, 38])
        # 打分机制 scores: [batch, seq_len, seq_len]
        scores = torch.matmul(query, x.transpose(1, 2)) / math.sqrt(d_k)  
#         print("score: ", scores.shape)  # torch.Size([128, 38, 38])
        
        # 对最后一个维度 归一化得分
        alpha_n = F.softmax(scores, dim=-1) 
#         print("alpha_n: ", alpha_n.shape)    # torch.Size([128, 38, 38])
        # 对权重化的x求和
        # [batch, seq_len, seq_len]·[batch,seq_len, hidden_dim*2] = [batch,seq_len,hidden_dim*2] -> [batch, hidden_dim*2]
        context = torch.matmul(alpha_n, x).sum(1)
        
        return context, alpha_n
    
    def forward(self, x):
        # [seq_len, batch, embedding_dim]
        embedding = self.dropout(self.embedding(x)) 
        
        # output:[seq_len, batch, hidden_dim*2]
        # hidden/cell:[n_layers*2, batch, hidden_dim]
        output, (final_hidden_state, final_cell_state) = self.rnn(embedding)
        output = output.permute(1, 0, 2)  # [batch, seq_len, hidden_dim*2]
        
        query = self.dropout(output)
        # 加入attention机制
        attn_output, alpha_n = self.attention_net(output, query)
        
        logit = self.fc(attn_output)
        
        return logit

2.2 形式二

参考：https://blog.csdn.net/qsmx666/article/details/107118550

Attention公式：

图中的 \(W_w\) 和 \(u_w\) 对应了下面代码中的w_omega和u_omega，随机初始化而来，hit对应x。

class BiLSTM_Attention(nn.Module):

    def __init__(self, vocab_size, embedding_dim, hidden_dim, n_layers):

        super(BiLSTM_Attention, self).__init__()

        self.hidden_dim = hidden_dim
        self.n_layers = n_layers
        self.embedding = nn.Embedding(vocab_size, embedding_dim)        #单词数，嵌入向量维度
        self.rnn = nn.LSTM(embedding_dim, hidden_dim, num_layers=n_layers, bidirectional=True, dropout=0.5)
        self.fc = nn.Linear(hidden_dim * 2, 1)
        self.dropout = nn.Dropout(0.5)

        # 初始时间步和最终时间步的隐藏状态作为全连接层输入
        self.w_omega = nn.Parameter(torch.Tensor(hidden_dim * 2, hidden_dim * 2))
        self.u_omega = nn.Parameter(torch.Tensor(hidden_dim * 2, 1))

        nn.init.uniform_(self.w_omega, -0.1, 0.1)
        nn.init.uniform_(self.u_omega, -0.1, 0.1)


    def attention_net(self, x):       #x:[batch, seq_len, hidden_dim*2]

        u = torch.tanh(torch.matmul(x, self.w_omega))         #[batch, seq_len, hidden_dim*2]
        att = torch.matmul(u, self.u_omega)                   #[batch, seq_len, 1]
        att_score = F.softmax(att, dim=1)

        scored_x = x * att_score                              #[batch, seq_len, hidden_dim*2]

        context = torch.sum(scored_x, dim=1)                  #[batch, hidden_dim*2]
        return context


    def forward(self, x):
        embedding = self.dropout(self.embedding(x))       #[seq_len, batch, embedding_dim]

        # output: [seq_len, batch, hidden_dim*2]     hidden/cell: [n_layers*2, batch, hidden_dim]
        output, (final_hidden_state, final_cell_state) = self.rnn(embedding)
        output = output.permute(1, 0, 2)                  #[batch, seq_len, hidden_dim*2]

        attn_output = self.attention_net(output)
        logit = self.fc(attn_output)
        return logit

3. 训练、评估模型

常规套路：

计算准确率
训练函数
评估函数
打印模型表现
用保存的模型参数预测测试数据。

# 计算准确率
def binary_acc(preds, y):
    preds = torch.round(torch.sigmoid(preds))
    correct = torch.eq(preds, y).float()
    acc = correct.sum() / len(correct)
    return acc

# 训练函数
def train(rnn, iterator, optimizer, criteon):
    avg_loss = []
    avg_acc = []
    rnn.train()    # 表示进入训练模式
    
    for i, batch in enumerate(iterator):
        pred = rnn(batch.text).squeeze()   # [batch, 1] -> [batch]
        
        loss = criteon(pred, batch.label)
        acc = binary_acc(pred, batch.label).item()  # 计算每个batch的准确率
        
        avg_loss.append(loss.item())
        avg_acc.append(acc)
        
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        
    avg_acc = np.array(avg_acc).mean()
    avg_loss = np.array(avg_loss).mean()
    return avg_loss, avg_acc
    
# 评估函数
def evaluate(rnn, iterator, criteon):
    avg_loss = []
    avg_acc = []
    rnn.eval()    # 进入测试模式
    
    with torch.no_grad():
        for batch in iterator:
            pred = rnn(batch.text).squeeze()  # [batch, 1] -> [batch]
            
            loss = criteon(pred, batch.label)
            acc = binary_acc(pred, batch.label).item()
            
            avg_loss.append(loss.item())
            avg_acc.append(acc)
    
    avg_loss = np.array(avg_loss).mean()
    avg_acc = np.array(avg_acc).mean()
    return avg_loss, avg_acc

# 训练模型，并打印模型的表现
best_valid_acc = float('-inf')

for epoch in range(30):
    start_time = time.time()
    
    train_loss, train_acc = train(rnn, train_iterator, optimizer, criteon)
    dev_loss, dev_acc = evaluate(rnn, dev_iterator, criteon)
    
    end_time = time.time()
    
    epoch_mins, epoch_secs = divmod(end_time - start_time, 60)
    
    # 只要模型效果变好，就保存
    if dev_acc > best_valid_acc:
        best_valid_accs = dev_acc
        torch.save(rnn.state_dict(), 'wordavg-model.pt')
        
    print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs:.2f}s')
    print(f'\tTrain Loss: {train_loss:.3f} | Train Acc: {train_acc*100:.2f}%')
    print(f'\t Val. Loss: {dev_loss:.3f} |  Val. Acc: {dev_acc*100:.2f}%')

# 用保存的模型参数预测数据
rnn.load_state_dict(torch.load('wordavg-model.pt'))

test_loss, test_acc = evaluate(rnn, test_iterator, criteon)
print(f'Test. Loss: {test_loss:.3f} |  Test. Acc: {test_acc*100:.2f}%')

hidden_dim=64, n_layers=2的条件下：

当定义的模型部分只有LSTM时，准确率：81.16%

当使用2.1的Attention公式，准确率：82.46%

当使用2.2的Attention公式，准确率：81.49%

加入Attention机制，性能略有提升。

posted @ 2020-08-16 00:46 douzujun 阅读(12489) 评论(5) 编辑收藏举报

刷新页面返回顶部

douzi