转:Python 文本挖掘:使用gensim进行文本相似度计算
![Python 文本挖掘:使用gensim进行文本相似度计算 - rzcoding - Explore in Data](http://img1.ph.126.net/vlE1qOtmzVjic-rQ3GYX_g==/1470143803459906354.png)
# 读取txt 文档中的每条评论并用itertools 的yield 方法存储起来(比起把所有数据存在数组中,使用itertools 的内存效率高,具体原理请google)
class MyCorpus(object):
def __iter__(self):
for line in open(datapath):
yield line.split()
from gensim import corpora, models, similarities
# 以下是把评论通过gensim 转化为tf-idf 形式,程序具体解释参见52nlp的博客或gensim官方文档
Corp=MyCorpus()
dictionary = corpora.Dictionary(Corp)
corpus =[dictionary.doc2bow(text)for text inCorp]#把所有评论转化为词包(bag of words)
tfidf = models.TfidfModel(corpus)#使用tf-idf 模型得出该评论集的tf-idf 模型
corpus_tfidf = tfidf[corpus]#此处已经计算得出所有评论的tf-idf 值
#读取商品描述的txt 文档
q_file = open(querypath, 'r')
query = q_file.readline()
q_file.close()
vec_bow = dictionary.doc2bow(query.split())#把商品描述转为词包
vec_tfidf = tfidf[vec_bow]#直接使用上面得出的tf-idf 模型即可得出商品描述的tf-idf 值
index = similarities.MatrixSimilarity(corpus_tfidf)#把所有评论做成索引
sims = index[vec_tfidf]#利用索引计算每一条评论和商品描述之间的相似度
similarity = list(sims)#把相似度存储成数组,以便写入txt 文档
sim_file = open(storepath,'w')
for i in similarity:
sim_file.write(str(i)+'\n')#写入txt 时不要忘了编码
sim_file.close()
![Python 文本挖掘:使用gensim进行文本相似度计算 - rzcoding - Explore in Data](http://img0.ph.126.net/U_MiJmxq3Ql33gplQLODtg==/3143231065028185368.png)
#! /usr/bin/env python2.7
#coding=utf-8
import logging
from gensim import corpora, models, similarities
def similarity(datapath, querypath, storepath):
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)
classMyCorpus(object):
def __iter__(self):
for line in open(datapath):
yield line.split()
Corp=MyCorpus()
dictionary = corpora.Dictionary(Corp)
corpus =[dictionary.doc2bow(text)for text inCorp]
tfidf = models.TfidfModel(corpus)
corpus_tfidf = tfidf[corpus]
q_file = open(querypath,'r')
query = q_file.readline()
q_file.close()
vec_bow = dictionary.doc2bow(query.split())
vec_tfidf = tfidf[vec_bow]
index = similarities.MatrixSimilarity(corpus_tfidf)
sims = index[vec_tfidf]
similarity = list(sims)
sim_file = open(storepath,'w')
for i in similarity:
sim_file.write(str(i)+'\n')
sim_file.close()