Python Word2Vec训练和测试词向量

train_word2vec_model.py:

#!/usr/bin/env python # -*- coding: utf-8 -*-
 
import logging import os import sys import multiprocessing from gensim.models import Word2Vec from gensim.models.word2vec import LineSentence if __name__ == '__main__': program = os.path.basename(sys.argv[0]) logger = logging.getLogger(program) logging.basicConfig(format='%(asctime)s: %(levelname)s: %(message)s') logging.root.setLevel(level=logging.INFO) logger.info("running %s" % ' '.join(sys.argv)) # check and process input arguments
    if len(sys.argv) < 4: print(globals()['__doc__'] % locals()) sys.exit(1) inp, outp1, outp2 = sys.argv[1:4] model = Word2Vec(LineSentence(inp), size=800, window=10, min_count=5,sg=1,hs=1, workers=multiprocessing.cpu_count()) #window:skip-gram通常在10附近，CBOW通常在5附近
    #hs: 如果为1则会采用hierarchica softmax技巧。如果设置为0（defaut），则negative sampling会被使用。
    # trim unneeded model memory = use(much) less RAM
    # model.init_sims(replace=True)
 model.save(outp1) model.wv.save_word2vec_format(outp2, binary=False)

执行 "python train_word2vec_model.py v6_EN.txt v6_EN.model v6_EN.vector"即可训练词向量

train_word2vec_model.py为训练词向量的程序代码，v6_EN.txt是我训练的语料库的名称，v6_EN.model为我训练出来的词向量模型名称，v6_EN.vector为格式化保存词向量模型的文件（一般用不到，但执行语句必须包含这一项）

训练出的文件有五个：

中间3个.npy文件在load词向量模型时都必须和v6_EN.model放在同一文件夹下

In [1]: import gensim
 
In [2]: model = gensim.models.Word2Vec.load("v6_EN.model")
 
In [3]: result = model.most_similar("足球")
 
In [4]: for e in result:
    print e[0], e[1]
   ....:     
联赛 0.65538161993
甲级 0.653042972088
篮球 0.596754670143
俱乐部 0.587228953838
乙级 0.58406317234
足球队 0.556015253067
亚足联 0.530800580978
allsvenskan 0.52497625351
代表队 0.521494746208
甲组 0.51778960228

 test.py：

import gensim import numpy as np import xlwt model_EN = gensim.models.Word2Vec.load("../v6_EN_SG/v6_EN_SG_800.model") model_FR = gensim.models.Word2Vec.load("../v6_FR_SG/v6_FR_SG.model") workbook = xlwt.Workbook(encoding = 'utf-8') worksheet = workbook.add_sheet('Result') Thta = np.load("GT/ThtaEN-FR/Thta0.07/ThtaEN-FR0.07_7000.npy") test = np.load("GT/test1000EN-FR.npy") font1 = xlwt.Font() font1.height=0x00E8 font1.name = '宋体' style1 = xlwt.XFStyle() style1.font = font1 worksheet.write(0, 0, label = '英文测试单词', style = style1) worksheet.col(0).width = 3333 worksheet.write(0, 1, label = '预测的法语译文', style = style1) worksheet.col(1).width = 4000 worksheet.write(0, 2, label = '词典给出的法语译文', style = style1) worksheet.col(2).width = 4400 worksheet.write(0, 3, label = '对错', style = style1) worksheet.col(3).width = 4400 num = 0 true_Word=0.0
while num < 1000: word_EN = test[num][0] word_FR = test[num][1] vec_Test = model_EN.wv[word_EN] vec_Test.shape = (1,800) b = np.dot(vec_Test,Thta) b.shape = (200,) e = model_FR.wv.similar_by_vector(b, topn=5, restrict_vocab=None) print(e[0][0]) worksheet.write(num+1, 0, label = word_EN) worksheet.write(num+1, 1, label = [e[k][0]+'  ' for k in range(5)]) worksheet.write(num+1, 2, label = word_FR) for i in range(5): if e[i][0] == word_FR: worksheet.write(num+1, 3, label = '✔️') true_Word+=1
            break
        elif i == 4: worksheet.write(num+1, 3, label = '×') print('测试完成%d个单词'%(num+1)) num += 1 worksheet.write(num+1, 0, label = '正确率', style = style1) worksheet.write(num+1, 1, label = str(true_Word/num*100)+'%') print(str(true_Word/num*100)+'%') workbook.save('GT/test/testEN-FR/Thta0.07/EN-FR0.07@5_7000.xls')