使用pymmseg进行中文分词
python分词模块,基于mmseg算法编写,核心代码c++,提供python接口。
最近在做BUG质量分析的时候,要提取BUG关键词,正好用到的pymmseg这分词模块,分词的效率还不错。
Code Example:
#-*- coding: utf-8 -*-
from pymmseg import mmseg
import os
import sys
def CWS_pymmseg(shortdeslist,wordlist):
if os.path.isfile(shortdeslist):
mmseg.dict_load_defaults()
sd = open(shortdeslist,'r')
word = open(wordlist,'w')
for bugdes in sd.readlines():
algor = mmseg.Algorithm(bugdes)
wlist = []
for tok in algor:
wlist.append(tok.text + "\n")
word.writelines(wlist)
sd.close()
word.close()
print "CWSeg_pymmseg is OK ! %s ==> %s" % (shortdeslist,wordlist)
else:
print "ERROR : The file ,shortdeslist doesn't exist!"
if __name__ == '__main__':
if len(sys.argv) == 3:
CWS_pymmseg(sys.argv[1],sys.argv[2])
else:
print "Usage: python CWS_pymmseg.py [shortdeslist] [wordlist]"