相似度分析
import re import jieba doc1 = '''曝甜瓜和篮网球员一起训练 新赛季有望加入篮网?:自从上赛季在火箭经历了失败后,安东尼一直无人问津,今年夏天关于他打不上球的话题也是被反复议论,根据最新消息,安东尼最近正和篮网球员一起训练。 ''' doc2 = '''周琦团队:感谢新疆耐心等待 新疆仍支持周琦留洋:昨天,男篮国手周琦重返新疆男篮的事尘埃落定。此后,周琦团队很快发表声明,新疆男篮仍会支持周琦挑战高水平联赛,而周琦之所以没有最终到欧洲去打球,是因为兼顾国家队备战,无法抽身去参加试训。 ''' doc3 = '''官宣!新疆宣布周琦回归 新赛季向总冠军发起冲击:北京时间8月13日,新疆篮球俱乐部官宣已经与周琦完成签约。周琦将重新身披新疆队战袍,征战CBA赛场。新赛季周琦与球队向总冠军发起冲击。 ''' doc4 = '''官宣!新疆宣布周琦回归 新赛季向总冠军发起冲击''' doc5 = '''欧联-武磊替补登场 西班牙人2-2总比分5-3进正赛_手机搜狐网,阵容方面,西班牙人主帅加耶戈排出4-3-3阵型,卡莱罗搭档路易斯-洛佩斯出任中卫,达德尔、罗卡、格拉内罗组成''' doc_complete = [doc1, doc2, doc3] # doc_clean = [clean(doc).split() for doc in doc_complete] def fenci(title): corpus = '' corpus = title stopwords = [line.strip() for line in open('../data/stopWord.txt', 'r', encoding='utf-8').readlines()] r4 = "\\【.*?】+|\\《.*?》+|\\#.*?#+|[0-9]+|[.!/_,$&%^*()<>+""'?@|:~{}#]+|[——!\\\,。=?、:“”‘’¥……()《》【】]" BIFEN_RE = re.compile(r'[0-9]+[:|:|-|-|+]\s{0,5}[0-9]+') doc = re.sub(BIFEN_RE, '比分', corpus) doc = re.sub(r'VS', '对战', doc, flags=re.IGNORECASE) doc = re.sub(r'8强', '八强', doc) doc = re.sub(r'4强', '四强', doc) doc = re.sub(r'16强', '十六强', doc) doc = re.sub(r4, ' ', doc) jieba.load_userdict("../data/user_dict.txt") seg = jieba.cut(doc) result = '' for word in seg: if word not in stopwords: result += word + ' ' return result doc_clean = [fenci(doc).split() for doc in doc_complete] # print(doc_clean) # LDA import gensim from gensim import corpora # Creating the term dictionary of our courpus, where every unique term is assigned an index. dictionary = corpora.Dictionary(doc_clean) # Converting list of documents (corpus) into Document Term Matrix using dictionary prepared above. doc_term_matrix = [dictionary.doc2bow(doc) for doc in doc_clean] Lda = gensim.models.ldamodel.LdaModel # Running and Trainign LDA model on the document term matrix. ldamodel = Lda(doc_term_matrix, num_topics=3, id2word = dictionary, passes=50) print(ldamodel.print_topics(num_topics=3, num_words=5)) # DF for doc in doc_clean: # dic = dict(zip(doc, [(i-i) for i in range(len(doc))])) # print(dic) dic = {} for word in doc: if dic. __contains__(word): dic[word] += 1 else: dic[word] = 1 print(dic) l = sorted(dic.items(), key=lambda d:d[1],reverse=True) print(l) import jieba import jieba.analyse keywords = jieba.analyse.extract_tags(doc3, topK=5, withWeight=True, allowPOS=('n', 'nr', 'ns')) # print(type(keywords)) # <class 'list'> for item in keywords: print(item[0], item[1]) print('****************') keywords = jieba.analyse.textrank(doc3, topK=5, withWeight=True, allowPOS=('n', 'nr', 'ns')) # type(keywords) # <class 'list'> for item in keywords: print(item[0], item[1])
from gensim.models import word2vec import jieba import re import numpy as np # 利用word2vec计算 model = word2vec.Word2Vec.load('../model/last.model') def fenci(title): corpus = '' corpus = title r4 = "\\【.*?】+|\\《.*?》+|\\#.*?#+|[.!/_,$&%^*()<>+""'?@|:~{}#]+|[——!\\\,。=?、:“”‘’¥……()《》【】]" BIFEN_RE = re.compile(r'[0-9]+[:|:|-|-|+]\s{0,5}[0-9]+') doc = re.sub(r4, ' ', corpus) doc = re.sub(BIFEN_RE, '比分', doc) doc = re.sub(r'VS', '对战', doc, flags=re.IGNORECASE) doc = re.sub(r'8强', '八强', doc) doc = re.sub(r'4强', '四强', doc) doc = re.sub(r'16强', '十六强', doc) jieba.load_userdict("../data/user_dict.txt") seg = jieba.cut(doc) result = '' for word in seg: result += word + ' ' return result sent1 = u'欧联-武磊替补登场 西班牙人2-2总比分5-3进正赛_手机搜狐网' sent2 = u'武磊替补登场' def getsentecevec(sentence): vec = np.zeros(100) # print(vec) count = 0 model = word2vec.Word2Vec.load('../model/last.model') for wor in fenci(sentence): try: vec += model.wv[wor] count += 1 except: # print(wor + 'not in voca') pass vec /= count return vec def xiangsidu(vec_a,vec_b): a = np.array(vec_a) b = np.array(vec_b) ret = np.sum(a * b) / (np.sqrt(np.sum(a ** 2)) * np.sqrt(np.sum(b ** 2))) print('分数:' + str(ret)) doc1 = u'欧联-武磊替补登场 西班牙人2-2总比分5-3进正赛_手机搜狐网' doc2 = u'西媒盛赞武磊:没有让人失望!跑位很出色 踢法灵活本报讯(记者王帆)' doc3 = u'西班牙人前瞻:武磊有望重回首发 伤员悉数归队_手机搜狐网' doc4 = u'表现平平!武磊仅获西媒评5分 0射门进攻端无贡献_手机搜狐网' doc = u'武磊替补登场' str5 = "篮球贡献获世界肯定 姚明将入选NBA名人堂 晚间体育新闻" str6 = "姚明已确定入选2016届名人堂 午间体育新闻" str7 = "篮球贡献获世界肯定 姚明将入选NBA名人堂" str8 = "美媒:篮球明星姚明将入选美国NBA名人堂" str9 = "姚明已确定入选2016届名人堂" str1 = "[冠军欧洲]特别企划:狭路相逢 球队交锋史" str2 = "[冠军欧洲]特别企划:逆转之态 欧冠小百科" str3 = "[冠军欧洲]特别企划:逆转之态 姚明小百科" doc_c = [str5, str6, str7, str8, str2, doc1,doc2,str3,doc4] vec_a = getsentecevec(str9) for d in doc_c: vec_b = getsentecevec(d) xiangsidu(vec_a, vec_b) # sent1 = u'军训服蹲下崩线' # sent2 = u'军训服蹲下就崩线' # sent1 = u'吴亦凡女友身份 ' # sent2 = u'吴亦凡女友疑曝光'