互联网时代的社会语言学:基于SNS的文本数据挖掘(python)
![](https://images.cnblogs.com/OutliningIndicators/ContractedBlock.gif)
# -*- coding=utf-8 -*- import feedparser import re import collections import math import sys reload(sys) sys.setdefaultencoding("utf8") def info_entropy(words): result = 0 total = sum([val for _, val in words.iteritems()]) for word, cnt in words.iteritems(): p = float(cnt) / total result -= p * math.log(p) return result max_word_len = 5 #字个数阈值 entropy_threshold = 1 #商阈值 num_threshold = 3; #词个数阈值 con_threshold = 100; #粘贴度阈值,越大,则是词的概率越大 txtin = u'luxunzawen.txt' txtout = u'luxunzawen_out.txt' f = open(txtin,'r').read().decode('utf-8') bef_sentences = [] bef_sentences = re.split("\W+|[a-zA-Z0-9]+|\s+|\n+", f, 0, re.UNICODE) sentences = [one for one in bef_sentences if len(one)>1] print r"sentences:finish" freq = collections.Counter() #计数器容器 for sentence in sentences: if sentence: l = len(sentence) wl = min(l, max_word_len) for i in xrange(1, wl + 1): for j in xrange(0, l - i + 1): freq[sentence[j:j + i]] += 1 total = sum([val for _, val in freq.iteritems()]) ps = collections.defaultdict(int) for word, val in freq.iteritems(): ps[word] = float(val) / total #每个词的词频 words = set() for word, word_p in ps.items(): if len(word) > 1: p = 0 for i in xrange(1, len(word)): t = ps[word[0:i]] * ps[word[i:]] p = max(p, t) if freq[word] >= num_threshold and word_p / p > con_threshold: #词频大于3,粘合度越大越好 words.add(word) final_words = set() for word in words: lf = rf = True left_words = collections.Counter() right_words = collections.Counter() pattern = re.compile(word.join(['.?', '.?'])) for sentence in sentences: l = pattern.findall(sentence) flag_l = flag_r = False if l: if l[0][0] != word[0]: left_words[l[0][0]] += 1 flag_l = True else: lf = False or flag_l if l[0][-1] != word[-1]: right_words[l[0][-1]] += 1 #有右汉字 flag_r = True else: rf = False or flag_r left_info_entropy = info_entropy(left_words) right_info_entropy = info_entropy(right_words) if lf and len(left_words) > 0 and left_info_entropy < entropy_threshold: continue if rf and len(right_words) > 0 and right_info_entropy < entropy_threshold: continue final_words.add(word) words_list = list(final_words) words_list.sort(cmp = lambda x, y: cmp(freq[y], freq[x])) #for word in words_list: # print word.encode('gb2312'), freq[word] fout = open(txtout, 'w') for word in words_list: result = word+u" "+str(freq[word])+"\n" fout.write(result) print('Done')
C#版本间这里