主题模型理解

 

先上代码

import nltk
from nltk.tokenize import word_tokenize

openfile = open('corpus.txt')
texts = openfile.readlines()
texts_tokenized = [[word.lower() for word in word_tokenize(text.decode('utf-8'))]  for text in texts]

print (texts_tokenized)


form nltk.corpus import stopwords
english_stopwords = stopwords.words('english')
texts_filterd_stopwords = [[word for word in text_tokenized if not word in english_stopwords] for text_tokenized in texts_tokenized]
english_punctuations = [',','.',':',';','?','(',')','[',']','&','!','*','@','#','$','%']

texts_filterd_punctuations = [[word for word in text_filterd_stopwords if not word in english_punctuations] for text_filterd_stopwords in texts_filterd_stopwords]
print texts_filterd_punctuations

# 词干化
from nltk.stem.lancaster import LancasterStemmer
st = LancasterStemmer()
texts_stemmed = [[st.stem(word) for word in text_filterd_punctuations] for text_filterd_punctuations in texts_filterd_punctuations]

# 写日志
from gensim import corpors,models,similarities
import logging
logging.basicConfig(format ='%(asctime)s : %(levelname)s : % (message)s', level = logging.INFO)


#1.抽取词袋
dictionary  = corpora.Dictionary(texts_stemmed)
print(dictionary.token2id)

# 2.文本向量化
corpus = [dictionary.doc2bow(text_stemmed) for text_stemmed in texts_stemmed]
print(corpus)

# 3.训练LDA模型
tfidf = models.TfidfModel(corpus)
corpus_tfidf = tfidf(corpus)
for text in corpus_tfidf:
    print(text)

lda = models.LdaModel(corpus_tfidf,id2word = dictionary,num_topics = 2)
corpus_lda = lda(corpus_tfidf)

for text in corpus_lda[0:6]:
    print(text)

# 4计算相似度
lda_index = similarities.MatrixSimilarity(corpus_lda)
sims = lda_index[corpus_lda[0]]
sort_sims = sorted(enumerate(sims),key = lambda item: -item[1])
print(sort_sims)

后面再补充说明，代码来自于 机器学习基础 吕云翔