Pytorch-torchtext的使用

使用torchtext的一般步骤https://www.cnblogs.com/cxq1126/p/13466998.html#_label9

1.使用torchtext默认支持的预训练词向量

默认情况下，会自动下载对应的预训练词向量文件到当前文件夹下的.vector_cache目录下，.vector_cache为默认的词向量文件和缓存文件的目录。

from torchtext.vocab import GloVe
from torchtext import data
TEXT = data.Field(sequential=True)

TEXT.build_vocab(train, vectors=GloVe(name='6B', dim=300))
TEXT.build_vocab(train, vectors="glove.6B.300d")

2.使用外部预训练好的词向量

从网站中（https://github.com/Embedding/Chinese-Word-Vectors）下载中文词向量sgns.wiki.word

通过name参数可以指定预训练词向量文件所在的目录；
默认情况下预训练词向量文件和缓存文件的目录位置都为当前目录下的 .vector_cache目录，虽然通过name参数指定了预训练词向量文件存在的目录，但是因为缓存文件的目录没有特殊指定，此时在当前目录下仍然需要存在 .vector_cache 目录。

if not os.path.exists('.vector_cache'):
    os.mkdir('.vector_cache')
vectors = Vectors(name='sgns.wiki.word')
TEXT.build_vocab(train_data, max_size=10000, vectors=vectors)

Embedding初始化还是一样

pretrained_embedding = TEXT.vocab.vectors
print('pretrained_embedding:', pretrained_embedding.shape)  #torch.Size([1727, 300])
model.src_embed[0].lut.weight.data.copy_(pretrained_embedding) 
print('Embedding初始化')

 参考https://blog.csdn.net/leo_95/article/details/87708267

3.篇章级文本分类，将每一篇文档按长度分三段保存，共用一份词表

textfield可以定义多个属性，text1，text2，text3。

from nltk.tokenize import word_tokenize
from torchtext import data as tdata
from torchtext.vocab import GloVe
from torchtext.vocab import Vectors

def read_data(data_path, text_field, label_field, split=3, overlap=0):
    fields = []
    for i in range(1, split+1):
        fields.append(('text'+str(i), text_field))
    fields.append(('label', label_field))

    examples = []

    with open(data_path) as csv_file:
        reader = csv.reader(csv_file, quotechar='"')
        for idx, line in enumerate(reader):
            text = ""
            for tx in line[1:]:
                text += tx              #tx就是一篇文档
                text += " "
                word_tokens = word_tokenize(text)
                len_text = len(word_tokens)
                document_encode = []
                for i in range(split):
                    len_true = int((len_text + overlap*(split-1)) / split)         #小文档的真实长度
                    len_rel = len_true - overlap
                    doc = word_tokens[i*len_rel : (i+1)*len_rel + overlap]
                    document_encode.append(doc)

            label = int(line[0])
            document_encode.append(label)
            examples.append(tdata.Example.fromlist(document_encode, fields))
    return examples, fields

def data_doc_iter(train_path, test_path, text_field, label_field, batch_size, embedding_dim=50):
    train_examples, train_fields = read_data(train_path, text_field, label_field)
    test_examples, test_fields = read_data(test_path, text_field, label_field)

    train_dataset = tdata.Dataset(train_examples, train_fields)
    test_dataset = tdata.Dataset(test_examples, test_fields)

    #构建词表
    text_field.build_vocab(train_dataset, vectors=GloVe(name='6B', dim=embedding_dim))        
    label_field.build_vocab(train_dataset)

    train_iter = tdata.Iterator(train_dataset, batch_size=batch_size, shuffle=False, sort=False, sort_within_batch=False, repeat=False)
    test_iter = tdata.Iterator(test_dataset, batch_size=batch_size, shuffle=False, sort=False, sort_within_batch=False, repeat=False)
    vocabulary = text_field.vocab
    return train_iter, test_iter, vocabulary

调用如下：

text_field = tdata.Field(tokenize=lambda x: word_tokenize(x), lower=True, fix_length=512, batch_first=True)       
label_field = tdata.LabelField(dtype=torch.int) 
train_iter, test_iter, vocabulary = data_doc_iter("./data/IMDB_new/train_shuffle.csv",  "./data/IMDB_new/test_new.csv", 
                                                        text_field, label_field, batch_size=8)

for batch in train_iter:
    print(batch.text1.shape)
    print(batch.text2.shape)
    print(batch.text3.shape)
    print(batch.label)

如果每个text的fix_length想设置得不一样，text_field可以不同

fields = [('text_en', text_field), ('text_ch', text_field), ('text', text_field2), ('label', label_field)]

4.篇章级文本分类，将每一篇文档按长度分三段保存，每一份文档的词表不同

即所有文档的第一份使用第一份词表，所有文档的第二份使用第二份词表，所有文档的第三份使用第三份词表。

每一份词表是从.vector_cache中的glove.6B.50d.txt中的词随机抽取一半当作新的词向量得到的，分别保存为cibiao1.txt，cibiao2.txt和cibiao3.txt。构建随机特征子空间。

from nltk.tokenize import word_tokenize
from torchtext import data as tdata
from torchtext.vocab import GloVe
from torchtext.vocab import Vectors

def read_split_data(data_path, text_fields, label_fields, split=3, overlap=0):
    
    field1, field2, field3 = [], [], []
    field1.append(('text', text_fields[0]))
    field1.append(('label', label_fields[0]))
    field2.append(('text', text_fields[1]))
    field2.append(('label', label_fields[1]))
    field3.append(('text', text_fields[2]))
    field3.append(('label', label_fields[2]))       

    examp1, examp2, examp3 = [], [], [], []

    with open(data_path) as csv_file:
        reader = csv.reader(csv_file, quotechar='"')
        for idx, line in enumerate(reader):
            text = ""
            for tx in line[1:]:
                text += tx              #tx就是一篇文档
                text += " "
                word_tokens = word_tokenize(text)
                len_text = len(word_tokens)
                document_encode = []
                for i in range(split):
                    len_true = int((len_text + overlap*(split-1)) / split)         #小文档的真实长度
                    len_rel = len_true - overlap
                    doc = word_tokens[i*len_rel : (i+1)*len_rel + overlap]
                    document_encode.append(doc)

            label = int(line[0])        
            
            doc1, doc2, doc3 = [], [], []
            doc1.append(document_encode[0])
            doc1.append(label)
            examp1.append(tdata.Example.fromlist(doc1, field1))

            doc2.append(document_encode[1])
            doc2.append(label)
            examp2.append(tdata.Example.fromlist(doc2, field2))

            doc3.append(document_encode[2])
            doc3.append(label)
            examp3.append(tdata.Example.fromlist(doc3, field3))

    return examp1, examp2, examp3, field1, field2, field3


def data_docsplit_iter(train_path, test_path, text_fields, label_fields, batch_size):
    train1_examp, train2_examp, train3_examp, field1, field2, field3 = read_split_data(train_path, text_fields, label_fields)
    test1_examp, test2_examp, test3_examp, tfield1, tfield2, tfield3 = read_split_data(test_path, text_fields, label_fields)

    #构建词表
    train1_data = tdata.Dataset(train1_examp, field1)
    train2_data = tdata.Dataset(train2_examp, field2)
    train3_data = tdata.Dataset(train3_examp, field3)

    vectors1 = Vectors(name='cibiao1.txt')
    vectors2 = Vectors(name='cibiao2.txt')
    vectors3 = Vectors(name='cibiao3.txt')

    text_fields[0].build_vocab(train1_data, vectors=vectors1)
    text_fields[1].build_vocab(train2_data, vectors=vectors2)
    text_fields[2].build_vocab(train3_data, vectors=vectors3)

    label_fields[0].build_vocab(train1_data)
    label_fields[1].build_vocab(train2_data)
    label_fields[2].build_vocab(train3_data)

    test1_data = tdata.Dataset(test1_examp, tfield1)
    test2_data = tdata.Dataset(test2_examp, tfield2)
    test3_data = tdata.Dataset(test3_examp, tfield3)
    
    train_iter1 = tdata.Iterator(train1_data, batch_size=batch_size, shuffle=False, sort=False, sort_within_batch=False, repeat=False)
    train_iter2 = tdata.Iterator(train2_data, batch_size=batch_size, shuffle=False, sort=False, sort_within_batch=False, repeat=False)
    train_iter3 = tdata.Iterator(train3_data, batch_size=batch_size, shuffle=False, sort=False, sort_within_batch=False, repeat=False)

    test_iter1 = tdata.Iterator(test1_data, batch_size=batch_size, shuffle=False, sort=False, sort_within_batch=False, repeat=False)
    test_iter2 = tdata.Iterator(test2_data, batch_size=batch_size, shuffle=False, sort=False, sort_within_batch=False, repeat=False)
    test_iter3 = tdata.Iterator(test3_data, batch_size=batch_size, shuffle=False, sort=False, sort_within_batch=False, repeat=False)
  
    vocabulary1, vocabulary2, vocabulary3 = text_fields[0].vocab, text_fields[1].vocab, text_fields[2].vocab
    return train_iter1, train_iter2, train_iter3, test_iter1, test_iter2, test_iter3, vocabulary1, vocabulary2, vocabulary3

调用如下：

SENTENCE_LIMIT_SIZE = 512
DATAPATH = './data/IMDB_new/'

text_fields, label_fields = [], []
for i in range(3):
    text_fields.append(tdata.Field(tokenize=lambda x: word_tokenize(x), lower=True, fix_length=SENTENCE_LIMIT_SIZE, batch_first=True))
    label_fields.append(tdata.LabelField(dtype=torch.int))

train_iter1, train_iter2, train_iter3, test_iter1, test_iter2, test_iter3, vocabulary1, vocabulary2, vocabulary3 = data_docsplit_iter(DATAPATH + "train_shuffle.csv", DATAPATH + "test_new.csv", 
                                                                                                            text_fields, label_fields, batch_size=4)        
print('vocabulary1.vectors.shape = ', vocabulary1.vectors)
print('vocabulary2.vectors.shape = ', vocabulary2.vectors.shape)
print('vocabulary3.vectors.shape = ', vocabulary3.vectors.shape)          
for i, batch in enumerate(zip(train_iter1, train_iter2, train_iter3)):  
    print(batch[0].text)
    print(batch[0].label)
    print(batch[1].text)
    print(batch[1].label)
    break