Pytorch-torchtext的使用

使用torchtext的一般步驟https://www.cnblogs.com/cxq1126/p/13466998.html#_label9

1.使用torchtext默認支持的預訓練詞向量

默認情況下，會自動下載對應的預訓練詞向量文件到當前文件夾下的.vector_cache目錄下，.vector_cache為默認的詞向量文件和緩存文件的目錄。

from torchtext.vocab import GloVe
from torchtext import data
TEXT = data.Field(sequential=True)

TEXT.build_vocab(train, vectors=GloVe(name='6B', dim=300))
TEXT.build_vocab(train, vectors="glove.6B.300d")

2.使用外部預訓練好的詞向量

從網站中（https://github.com/Embedding/Chinese-Word-Vectors）下載中文詞向量sgns.wiki.word

通過name參數可以指定預訓練詞向量文件所在的目錄；
默認情況下預訓練詞向量文件和緩存文件的目錄位置都為當前目錄下的 .vector_cache目錄，雖然通過name參數指定了預訓練詞向量文件存在的目錄，但是因為緩存文件的目錄沒有特殊指定，此時在當前目錄下仍然需要存在 .vector_cache 目錄。

if not os.path.exists('.vector_cache'):
    os.mkdir('.vector_cache')
vectors = Vectors(name='sgns.wiki.word')
TEXT.build_vocab(train_data, max_size=10000, vectors=vectors)

Embedding初始化還是一樣

pretrained_embedding = TEXT.vocab.vectors
print('pretrained_embedding:', pretrained_embedding.shape)  #torch.Size([1727, 300])
model.src_embed[0].lut.weight.data.copy_(pretrained_embedding) 
print('Embedding初始化')

 參考https://blog.csdn.net/leo_95/article/details/87708267

3.篇章級文本分類，將每一篇文檔按長度分三段保存，共用一份詞表

textfield可以定義多個屬性，text1，text2，text3。

from nltk.tokenize import word_tokenize
from torchtext import data as tdata
from torchtext.vocab import GloVe
from torchtext.vocab import Vectors

def read_data(data_path, text_field, label_field, split=3, overlap=0):
    fields = []
    for i in range(1, split+1):
        fields.append(('text'+str(i), text_field))
    fields.append(('label', label_field))

    examples = []

    with open(data_path) as csv_file:
        reader = csv.reader(csv_file, quotechar='"')
        for idx, line in enumerate(reader):
            text = ""
            for tx in line[1:]:
                text += tx              #tx就是一篇文檔
                text += " "
                word_tokens = word_tokenize(text)
                len_text = len(word_tokens)
                document_encode = []
                for i in range(split):
                    len_true = int((len_text + overlap*(split-1)) / split)         #小文檔的真實長度
                    len_rel = len_true - overlap
                    doc = word_tokens[i*len_rel : (i+1)*len_rel + overlap]
                    document_encode.append(doc)

            label = int(line[0])
            document_encode.append(label)
            examples.append(tdata.Example.fromlist(document_encode, fields))
    return examples, fields

def data_doc_iter(train_path, test_path, text_field, label_field, batch_size, embedding_dim=50):
    train_examples, train_fields = read_data(train_path, text_field, label_field)
    test_examples, test_fields = read_data(test_path, text_field, label_field)

    train_dataset = tdata.Dataset(train_examples, train_fields)
    test_dataset = tdata.Dataset(test_examples, test_fields)

    #構建詞表
    text_field.build_vocab(train_dataset, vectors=GloVe(name='6B', dim=embedding_dim))        
    label_field.build_vocab(train_dataset)

    train_iter = tdata.Iterator(train_dataset, batch_size=batch_size, shuffle=False, sort=False, sort_within_batch=False, repeat=False)
    test_iter = tdata.Iterator(test_dataset, batch_size=batch_size, shuffle=False, sort=False, sort_within_batch=False, repeat=False)
    vocabulary = text_field.vocab
    return train_iter, test_iter, vocabulary

調用如下：

text_field = tdata.Field(tokenize=lambda x: word_tokenize(x), lower=True, fix_length=512, batch_first=True)       
label_field = tdata.LabelField(dtype=torch.int) 
train_iter, test_iter, vocabulary = data_doc_iter("./data/IMDB_new/train_shuffle.csv",  "./data/IMDB_new/test_new.csv", 
                                                        text_field, label_field, batch_size=8)

for batch in train_iter:
    print(batch.text1.shape)
    print(batch.text2.shape)
    print(batch.text3.shape)
    print(batch.label)

如果每個text的fix_length想設置得不一樣，text_field可以不同

fields = [('text_en', text_field), ('text_ch', text_field), ('text', text_field2), ('label', label_field)]

4.篇章級文本分類，將每一篇文檔按長度分三段保存，每一份文檔的詞表不同

即所有文檔的第一份使用第一份詞表，所有文檔的第二份使用第二份詞表，所有文檔的第三份使用第三份詞表。

每一份詞表是從.vector_cache中的glove.6B.50d.txt中的詞隨機抽取一半當作新的詞向量得到的，分別保存為cibiao1.txt，cibiao2.txt和cibiao3.txt。構建隨機特征子空間。

from nltk.tokenize import word_tokenize
from torchtext import data as tdata
from torchtext.vocab import GloVe
from torchtext.vocab import Vectors

def read_split_data(data_path, text_fields, label_fields, split=3, overlap=0):
    
    field1, field2, field3 = [], [], []
    field1.append(('text', text_fields[0]))
    field1.append(('label', label_fields[0]))
    field2.append(('text', text_fields[1]))
    field2.append(('label', label_fields[1]))
    field3.append(('text', text_fields[2]))
    field3.append(('label', label_fields[2]))       

    examp1, examp2, examp3 = [], [], [], []

    with open(data_path) as csv_file:
        reader = csv.reader(csv_file, quotechar='"')
        for idx, line in enumerate(reader):
            text = ""
            for tx in line[1:]:
                text += tx              #tx就是一篇文檔
                text += " "
                word_tokens = word_tokenize(text)
                len_text = len(word_tokens)
                document_encode = []
                for i in range(split):
                    len_true = int((len_text + overlap*(split-1)) / split)         #小文檔的真實長度
                    len_rel = len_true - overlap
                    doc = word_tokens[i*len_rel : (i+1)*len_rel + overlap]
                    document_encode.append(doc)

            label = int(line[0])        
            
            doc1, doc2, doc3 = [], [], []
            doc1.append(document_encode[0])
            doc1.append(label)
            examp1.append(tdata.Example.fromlist(doc1, field1))

            doc2.append(document_encode[1])
            doc2.append(label)
            examp2.append(tdata.Example.fromlist(doc2, field2))

            doc3.append(document_encode[2])
            doc3.append(label)
            examp3.append(tdata.Example.fromlist(doc3, field3))

    return examp1, examp2, examp3, field1, field2, field3


def data_docsplit_iter(train_path, test_path, text_fields, label_fields, batch_size):
    train1_examp, train2_examp, train3_examp, field1, field2, field3 = read_split_data(train_path, text_fields, label_fields)
    test1_examp, test2_examp, test3_examp, tfield1, tfield2, tfield3 = read_split_data(test_path, text_fields, label_fields)

    #構建詞表
    train1_data = tdata.Dataset(train1_examp, field1)
    train2_data = tdata.Dataset(train2_examp, field2)
    train3_data = tdata.Dataset(train3_examp, field3)

    vectors1 = Vectors(name='cibiao1.txt')
    vectors2 = Vectors(name='cibiao2.txt')
    vectors3 = Vectors(name='cibiao3.txt')

    text_fields[0].build_vocab(train1_data, vectors=vectors1)
    text_fields[1].build_vocab(train2_data, vectors=vectors2)
    text_fields[2].build_vocab(train3_data, vectors=vectors3)

    label_fields[0].build_vocab(train1_data)
    label_fields[1].build_vocab(train2_data)
    label_fields[2].build_vocab(train3_data)

    test1_data = tdata.Dataset(test1_examp, tfield1)
    test2_data = tdata.Dataset(test2_examp, tfield2)
    test3_data = tdata.Dataset(test3_examp, tfield3)
    
    train_iter1 = tdata.Iterator(train1_data, batch_size=batch_size, shuffle=False, sort=False, sort_within_batch=False, repeat=False)
    train_iter2 = tdata.Iterator(train2_data, batch_size=batch_size, shuffle=False, sort=False, sort_within_batch=False, repeat=False)
    train_iter3 = tdata.Iterator(train3_data, batch_size=batch_size, shuffle=False, sort=False, sort_within_batch=False, repeat=False)

    test_iter1 = tdata.Iterator(test1_data, batch_size=batch_size, shuffle=False, sort=False, sort_within_batch=False, repeat=False)
    test_iter2 = tdata.Iterator(test2_data, batch_size=batch_size, shuffle=False, sort=False, sort_within_batch=False, repeat=False)
    test_iter3 = tdata.Iterator(test3_data, batch_size=batch_size, shuffle=False, sort=False, sort_within_batch=False, repeat=False)
  
    vocabulary1, vocabulary2, vocabulary3 = text_fields[0].vocab, text_fields[1].vocab, text_fields[2].vocab
    return train_iter1, train_iter2, train_iter3, test_iter1, test_iter2, test_iter3, vocabulary1, vocabulary2, vocabulary3

調用如下：

SENTENCE_LIMIT_SIZE = 512
DATAPATH = './data/IMDB_new/'

text_fields, label_fields = [], []
for i in range(3):
    text_fields.append(tdata.Field(tokenize=lambda x: word_tokenize(x), lower=True, fix_length=SENTENCE_LIMIT_SIZE, batch_first=True))
    label_fields.append(tdata.LabelField(dtype=torch.int))

train_iter1, train_iter2, train_iter3, test_iter1, test_iter2, test_iter3, vocabulary1, vocabulary2, vocabulary3 = data_docsplit_iter(DATAPATH + "train_shuffle.csv", DATAPATH + "test_new.csv", 
                                                                                                            text_fields, label_fields, batch_size=4)        
print('vocabulary1.vectors.shape = ', vocabulary1.vectors)
print('vocabulary2.vectors.shape = ', vocabulary2.vectors.shape)
print('vocabulary3.vectors.shape = ', vocabulary3.vectors.shape)          
for i, batch in enumerate(zip(train_iter1, train_iter2, train_iter3)):  
    print(batch[0].text)
    print(batch[0].label)
    print(batch[1].text)
    print(batch[1].label)
    break