git: https://github.com/linyi0604/MachineLearning
詞向量技術 Word2Vec
每個連續詞匯片段都會對后面有一定制約 稱為上下文context
找到句子之間語義層面的聯系
1 from sklearn.datasets import fetch_20newsgroups 2 from bs4 import BeautifulSoup 3 import nltk, re 4 from gensim.models import word2vec 5 6 # nltk.download('punkt') 7 8 9 ''' 10 詞向量技術 Word2Vec 11 每個連續詞匯片段都會對后面有一定制約 稱為上下文context 12 13 找到句子之間語義層面的聯系 14 15 ''' 16 17 # 聯網下載新聞數據 18 news = fetch_20newsgroups(subset="all") 19 x, y = news.data, news.target 20 21 # 定義一個函數 將每條新聞中的句子分離,並返回一個句子的列表 22 def news_to_sentences(news): 23 news_text = BeautifulSoup(news).get_text() 24 tokenizer = nltk.data.load("tokenizers/punkt/english.pickle") 25 raw_sentences = tokenizer.tokenize(news_text) 26 sentences = [] 27 for sent in raw_sentences: 28 temp = re.sub("[^a-zA-Z]", " ", sent.lower().strip()).split() 29 sentences.append(temp) 30 31 return sentences 32 33 # 將長新聞中的句子剝離出來用於訓練 34 sentences = [] 35 for i in x: 36 sentence_list = news_to_sentences(i) 37 sentences += sentence_list 38 39 40 # 配置詞向量的維度 41 num_features = 300 42 # 保證被考慮的詞匯的頻度 43 min_word_count = 20 44 # 並行計算使用cpu核心數量 45 num_workers = 2 46 # 定義訓練詞向量的上下文窗口大小 47 context = 5 48 downsapling = 1e-3 49 50 # 訓練詞向量模型 51 model = word2vec.Word2Vec(sentences, 52 workers=num_workers, 53 size=num_features, 54 min_count=min_word_count, 55 window=context, 56 sample=downsapling) 57 # 這個設定代表當前訓練好的詞向量為最終版, 也可以加速模型訓練的速度 58 model.init_sims(replace=True) 59 60 # 利用訓練好的模型 尋找文本中與college相關的十個詞匯 61 print(model.most_similar("college")) 62 ''' 63 [('wisconsin', 0.7664438486099243), 64 ('osteopathic', 0.7474539279937744), 65 ('madison', 0.7433826923370361), 66 ('univ', 0.7296794652938843), 67 ('melbourne', 0.7212647199630737), 68 ('walla', 0.7068545818328857), 69 ('maryland', 0.7038443088531494), 70 ('carnegie', 0.7038302421569824), 71 ('institute', 0.7003713846206665), 72 ('informatics', 0.6968873143196106)] 73 '''