word2vec_文本相似度
#提取关键词
#关键词向量化
#相似度计算
from jieba import analyse
import numpy
import gensim
# 实现给出任意字符串,获取字符串中某字符的位置以及出现的总次数
def get_char_pos(string,char):
chPos=[]
try:
chPos=list(((pos,char) for pos,val in enumerate(string) if(val == char)))
except:
pass
return chPos
#提取关键词
def keyword_extract(data,filename):
tfidf=analyse.extract_tags
keywords=tfidf(data)
return keywords
#将文档的每句话进行关键词提取,并将结果保存在txt文件中
def getKeywords(docpath,savepath):
with open(docpath,'r',encoding='utf-8') as docf,open(savepath,'w',encoding='utf-8') as outf:
for data in docf:#每句话
data=data[:len(data)-1]
keywords=keyword_extract(data,savepath)
for word in keywords:
outf.write(word+'/')
outf.write('\n ')
#利用训练好的词向量获取关键词的词向量
import codecs
def word2vec(file_name,model):
DataFile = codecs.open(file_name, "r",encoding='utf-8')
DataSet = DataFile.readlines()[:-1]
wordvec_size =100
word_vec_all = numpy.zeros(wordvec_size)
for data in DataSet:#
space_pos=get_char_pos(data,'/')
first_word=data[0:space_pos[0][0]]
if first_word in model:
word_vec_all=word_vec_all+model[first_word]
for i in range(len(space_pos)-2):
word=data[space_pos[i][0]:space_pos[i+1][0]]
try:
c=model[word]
except KeyError:
c=0
word_vec_all=word_vec_all+c
return word_vec_all
#词向量相似度计算代码:余弦
def simlarityCalu(vector1,vector2):
vector1Mod=numpy.sqrt(vector1.dot(vector1))
vector2Mod=numpy.sqrt(vector2.dot(vector2))
if vector2Mod!=0 and vector1Mod!=0:
simlarity=(vector1.dot(vector2))/(vector1Mod*vector2Mod)
else:
simlarity=0
return simlarity
if __name__=='__main__':
#下载模型
model=gensim.models.Word2Vec.load('zhiwiki_news.word2vec')
p1='P1.txt'
p2='P2.txt'
p1_keywords='P1_keyword.txt'
p2_keywords = 'P2_keyword.txt'
#获取关键词
getKeywords(p1,p1_keywords)
getKeywords(p2,p2_keywords)
p1_vec=word2vec(p1_keywords,model)
p2_vec=word2vec(p2_keywords,model)
#计算相似度
print(simlarityCalu(p1_vec,p2_vec))
posted on 2018-11-19 10:32 happygril3 阅读(4094) 评论(0) 编辑 收藏 举报