LDA模型數據的可視化

"""
    執行lda2vec.ipnb中的代碼
    模型LDA
    功能：訓練好后模型數據的可視化
"""

from lda2vec import preprocess, Corpus
import matplotlib.pyplot as plt
import numpy as np
# %matplotlib inline
import pyLDAvis
try:
    import seaborn
except:
    pass
# 加載訓練好的主題-文檔模型，這里是查看數據使用。這里需要搞清楚數據的形式，還要去回看這個文件是怎么構成的
npz = np.load(open('D:/my_AI/lda2vec-master/examples/twenty_newsgroups/lda2vec/topics.pyldavis.npz', 'rb'))
# 數據
dat = {k: v for (k, v) in npz.iteritems()}
# 詞匯表變成list
dat['vocab'] = dat['vocab'].tolist()

#####################################
##  主題-詞匯
#####################################
# 主題個數為10
top_n = 10
# 主題對應10個最相關的詞
topic_to_topwords = {}
for j, topic_to_word in enumerate(dat['topic_term_dists']):
    top = np.argsort(topic_to_word)[::-1][:top_n]               # 概率從大到小的下標索引值
    msg = 'Topic %i '  % j
    # 通過list的下標獲取關鍵詞
    top_words = [dat['vocab'][i].strip()[:35] for i in top]
    # 數據拼接
    msg += ' '.join(top_words)
    print(msg)
    # 將數據保存到字典里面
    topic_to_topwords[j] = top_words

import warnings
warnings.filterwarnings('ignore')
prepared_data = pyLDAvis.prepare(dat['topic_term_dists'], dat['doc_topic_dists'],
                                 dat['doc_lengths'] * 1.0, dat['vocab'], dat['term_frequency'] * 1.0, mds='tsne')

from sklearn.datasets import fetch_20newsgroups
remove=('headers', 'footers', 'quotes')
texts = fetch_20newsgroups(subset='train', remove=remove).data


##############################################
##  選取一篇文章，確定該文章有哪些主題
##############################################

print(texts[1])
tt = dat['doc_topic_dists'][1]
msg = "{weight:02d}% in topic {topic_id:02d} which has top words {text:s}"
# 遍歷這20個主題，觀察一下它的權重，權重符合的跳出來
for topic_id, weight in enumerate(dat['doc_topic_dists'][1]):
    if weight > 0.01:
        # 權重符合要求，那么輸出該主題下的關聯詞匯
        text = ', '.join(topic_to_topwords[topic_id])
        print (msg.format(topic_id=topic_id, weight=int(weight * 100.0), text=text))

# plt.bar(np.arange(20), dat['doc_topic_dists'][1])

print(texts[51])
tt = texts[51]
msg = "{weight:02d}% in topic {topic_id:02d} which has top words {text:s}"
for topic_id, weight in enumerate(dat['doc_topic_dists'][51]):
    if weight > 0.01:
        text = ', '.join(topic_to_topwords[topic_id])
        print(msg.format(topic_id=topic_id, weight=int(weight * 100.0), text=text))


# plt.bar(np.arange(20), dat['doc_topic_dists'][51])