作者：finallyliuyu 转载使用等请注明出处
功能：根据：档集合大小，特征词数目，交叉验证折数的不同需求，快速生成Libsvm格式数据
数据Demo请见：新闻文本分类libsvm格式数据
此模块的各个子模块的功能我就不详细介绍了，与此系列博客K-means文本聚类系列（已经完成） 一曲同工。
下面开始给出各个子模块的代码：
# -*- coding: cp936 -*-
########################################################################3
#
#自动建立文件夹
#
########################################################################
def CreateFolders(path):
import os
curpath=os.getcwd()
os.mkdir(path)
os.chdir(path)
os.mkdir('class1')
os.mkdir('class2')
os.chdir(curpath)
def CreateAssist(toCalInfoGain):
import os
if toCalInfoGain==0:
folders=[r'D:\TextCategorization\corpus',r'D:\TextCategorization\testing',r'D:\TextCategorization\training',r'D:\TextCategorization\segmented',r'D:\TextCategorization\tsegmented']
for myfolder in folders:
CreateFolders(myfolder)
os.mkdir(r'D:\TextCategorization\VITdata')
os.mkdir(r'D:\TextCategorization\data')
os.mkdir(r'D:\TextCategorization\VITdata\data')
os.mkdir(r'D:\TextCategorization\data\data')
print 'finish,congratulations'
if __name__=="__main__":
import os
os.mkdir(r'D:\TextCategorization')
CreateAssist()
文档集分割：多少篇文章归为训练集，多少篇文章归为测试集：注意参数N为整个文档集合（包括测试集和训练集）的每一类的文章数目。这里设置两个类别有相等的文章数目集整个文档集合的文章数目为2N
# -*- coding: cp936 -*-
#此模块用于对文本语料库进行预处理
###################################################
#origidir:原语料库目录如E:\新闻语料\EntireTrainingSet\ClassFile\C000024
#destidir:目标语料库目录D:\corpus\class1
#N：需要移动的文本的个数
#####################################################
def CorpusFormation(origidir,destidir,N):
paths=[]
import os,shutil
for i in range(0,N):#产生待移动的文件路径
paths.append(origidir+'\\'+str(i)+'.txt')
for mypath in paths:
shutil.copy(mypath,destidir)
#print 'finsh%s'%mypath
#####################################################
#移动语料库
#####################################################
def MoveCorpus(N,toCalInfoGain):
if toCalInfoGain==0:
originaldirs=[r'E:\新闻语料\EntireTrainingSet\ClassFile\C000024',r'E:\新闻语料\EntireTrainingSet\ClassFile\C000013']
destinationdirs=[r'D:\TextCategorization\corpus\class1',r'D:\TextCategorization\corpus\class2']
for i in range(0,2):
CorpusFormation(originaldirs[i],destinationdirs[i],N)
print 'finish'
#####################################################################
#origidir:原语料库目录，如D:\corpus\class1
#destdir1:目标目录：如D:\training\class1
#destidir2:目标目录：如D:\testing\class1
#Vfold:几折交叉验证 count:已经进行了几次实验count=0,1,2,3
#N语料库的总规模
#####################################################################
def CorpusPartition(origidir,destdir1,destdir2,count,N,vfold=5):
import os,shutil
step=N/vfold
paths=[]
for i in range(0+count*step,step+count*step):
paths.append(origidir+'\\'+str(i)+'.txt')
for mypath in paths:
shutil.move(mypath,destdir1)
#print 'finsh%s'%mypath
paths=[]
pathstemp=os.listdir(origidir)
for m in pathstemp:
paths.append(origidir+os.sep+m)
for mypath in paths:
shutil.move(mypath,destdir2)
#print 'finish%s'%mypath
#print 'finalfinish congratulations!'
def moveAccordingPartition(N,count,toCalInfoGain):
if toCalInfoGain==0:
originaldirs=[r'D:\TextCategorization\corpus\class1',r'D:\TextCategorization\corpus\class2']
destidirs1=[r'D:\TextCategorization\training\class1',r'D:\TextCategorization\training\class2']
destidirs2=[r'D:\TextCategorization\testing\class1',r'D:\TextCategorization\testing\class2']
for i in range(0,2):
CorpusPartition(originaldirs[i],destidirs1[i],destidirs2[i],count,N)
print '第%s finish'%i
if __name__=="__main__":
#MoveCorpus(500)
N=500
count=0
moveAccordingPartition(N,count)
# -*- coding: cp936 -*-
#此模块用于建立词袋子模型
def BagOfWordsConstruction(root,toCalInfoGain):
if toCalInfoGain==0:
import cPickle as mypickle
file_dest=file(r'D:\TextCategorization\VITdata\vocabularystatistics.dat','w')
rawVSMMatrix=TrainingFileProcess(root)
vocabularystatistics={}
templist=[]
for rawVSM in rawVSMMatrix:
templist=templist+rawVSM
wordscollection=list(set(templist))
for word in wordscollection:
index=0
for rawVSM in rawVSMMatrix:
count=rawVSM.count(word)
if count>0 :
if vocabularystatistics.has_key(word)==False:
vocabularystatistics[word]=[]
vocabularystatistics[word].append((index,count))
else:
vocabularystatistics[word].append((index,count))
index=index+1
mypickle.dump(vocabularystatistics,file_dest)
print len(vocabularystatistics)
file_dest.close()
print 'BagOfWordsConstructionFinish'
############################################################################################
#将文章内容变成词集合
def FilePreProcess(rawtext):
import re
listresult=rawtext.split("|")
finalrawVSM=[]
stopwordlist=FilterNoiseWord(r'C:\Python26\SVM\stopwords.txt')
for m in listresult:
if m!=''and m not in stopwordlist and re.search('\xa3[\xa1-\xfe]',m)==None:
finalrawVSM.append(m)
return finalrawVSM
#################################################################################################
#训练集文档预处理
def TrainingFileProcess(root):
from SVM import DataManager
import cPickle as mypickle
import os
rawVSMMatrix=[]#存放整个文档集
dm=DataManager.DataManager(root)
subdir=dm.GetSubDir()
for sub in subdir:
dm.SetFilePathsFromsubDir(root+os.sep+sub)
filepaths=dm.GetFilePaths()
for path in filepaths:
myfile=file(root+os.sep+sub+os.sep+path)
rawtext=myfile.read()
myfile.close()
rawVSM=FilePreProcess(rawtext)
rawVSMMatrix.append(rawVSM)
return rawVSMMatrix
####################################################################################
#生成停用词列表
def FilterNoiseWord(stopword_file_name):
import re
f=file(stopword_file_name)
stopword=f.read()
f.close()
stopwordlist=re.split('\n',stopword)
return stopwordlist
if __name__=="__main__":
BagOfWordsConstruction(r'D:\TextCategorization\segmented')
#fid=file(r'D:\3011.txt')
#rawtext=fid.read()
#fid.close()
#FilePreProcess(rawtext)
class IG:
'''
此模块用于计算信息增益
'''
#######################################################################################
def __init__(self,n_size):
''' 类的构造函数，初始化 类的数据成员变量
keys保存Term的值
labelOneNums保存对应的term出现在类1中的文章的篇数
labelTwoNums保存对应的term出现在类2中的文字的篇数
'''
import cPickle as mypickle
mydict=mypickle.load(file(r'D:\TextCategorization\VITData\vocabularystatistics.dat'))
self.mykeys=[]
self.labelOneNums=[]
self.labelTwoNums=[]
self.probs=[]
self.conProbs=[]
#self.informationgain=[]
for key ,value in mydict.iteritems():
self.mykeys.append(key)
class1_count=0 #某个term属于类别一的次数
class2_count=0#某个term 属于类别二的次数
for val in value:
if val[0]<n_size/2:#该文章标号属于类别1
class1_count=class1_count+val[1]
#class1_count=class1_count+1
else:#该文章标号属于类别2
class2_count=class2_count+val[1]
#class2_count=class2_count+1
self.labelOneNums.append(class1_count)
self.labelTwoNums.append(class2_count)
#测试代码
#fid=file('1.txt','a')
#for m in self.labelOneNums:
#print>>fid,m
#fid.flush()
#fid.close()
#print len([m for m in self.labelOneNums if m>0])
#print len(self.labelTwoNums)
#print len(self.mykeys)
###################################################################################3
#def GetConditionProbabilityBaseC(self,n_size,termcount):
#conditionPtxC=float(termcount+1)/(n_size/2+len(self.mykeys))
#return conditionPtxC
def GetConditionProbabilityBaseC(self,index,termcount):
'''
计算P(t|C)
'''
if index==1:
conditionPtxC=float(termcount+1)/(len(self.mykeys)+sum(self.labelOneNums))
else:
conditionPtxC=float(termcount+1)/(len(self.mykeys)+sum(self.labelTwoNums))
return conditionPtxC
def GetTermProbability(self,n_size):
'''
计算每个term的先验概率
'''
#sumtotal=sum(self.labelOneNums)+sum(self.labelTwoNums)
for i in range(0,len(self.mykeys)):
prob=0.5*self.GetConditionProbabilityBaseC(1,self.labelOneNums[i])+0.5*self.GetConditionProbabilityBaseC(2,self.labelTwoNums[i])
self.probs.append(prob)
#测试代码
#fid=file('prob.txt','a')
#for m in self.probs:
#print>>fid,m
#fid.flush()
#fid.close()
###################################################################################
def GetCategoryProbConditionTerm(self,n_size):
'''
保存在一个词出现与否的前提下，文章属于某个类的概率
'''
for i in range(0,len(self.mykeys)):
# conprob1:出现term t 其属于类别1的概率
conprob1=self.GetConditionProbabilityBaseC(1,self.labelOneNums[i])*0.5/self.probs[i]
# conprob2:出现term t 其属于类别2的概率
conprob2=self.GetConditionProbabilityBaseC(2,self.labelTwoNums[i])*0.5/self.probs[i]
#nonconprob1:不出现term t的条件下,属于类别 1的概率
nonconprob1=1-conprob1
#nonconprob2:不出现term t的条件下,属于类别 2的概率
nonconprob2=1-conprob2
self.conProbs.append((conprob1,conprob2,nonconprob1,nonconprob2))
#测试代码
#fid=file('conprob.txt','a')
#for m in self.conProbs:
#print>>fid,m
#fid.flush()
#fid.close()
########################################################################################
def CalInformationGain(self,n_size):
'''
计算每个单词的信息增益
'''
import math
import cPickle as mypickle
temp=0#辅助计算变量
nontemp=0#辅助计算变量
self.GetTermProbability(n_size)
self.GetCategoryProbConditionTerm(n_size)
infoGain={}
for i in range(0,len(self.mykeys)):
temp=0#辅助计算变量
nontemp=0#辅助计算变量
conprob1=self.conProbs[i][0]
conprob2=self.conProbs[i][1]
nonconprob1=self.conProbs[i][2]
nonconprob2=self.conProbs[i][3]
if conprob1!=0:
temp=temp+conprob1*math.log(conprob1,2)
if conprob2!=0:
temp=temp+conprob2*math.log(conprob2,2)
if nonconprob1!=0:
nontemp=nontemp+nonconprob1*math.log(nonconprob1,2)
if nonconprob2!=0:
nontemp=nontemp+nonconprob2+math.log(nonconprob2,2)
igval=2+self.probs[i]*temp+(1-self.probs[i])*nontemp
infoGain[self.mykeys[i]]=igval
#infoGain.sort(key=lambda d:d[1],reverse=True)
infoGainResult=sorted(infoGain.iteritems(),key=lambda infoGain:infoGain[1],reverse=True)
print '共计算了%s个词的IG值' %len(infoGainResult)
#for m in infoGainResult:
#print '%s,%s'%(m[0],m[1])
fid=file(r'D:\TextCategorization\VITData\infoGain.dat','w')
mypickle.dump(infoGainResult,fid)
fid.close()
if __name__=="__main__":
MyIG=IG(200)
MyIG.CalInformationGain(200)
-*- coding: cp936 -*-
'''
此模块根据信息增益选择特征词
'''
###########################################################################
def featureSelectionIG(N,flag,n_size):
'''
更新数据库，并返回特征词集合
#flag=0表示infoGain没有被计算
'''
from SVM import InformationGain
import cPickle as mypickle
if flag==0:
MyIG=InformationGain.IG(n_size)
MyIG.CalInformationGain(n_size)
featurewords=[]
infoGainResult=mypickle.load(file(r'D:\TextCategorization\VITdata\infoGain.dat'))
print 'infoGainResult的长度%s'%len(infoGainResult)
#N=1000#确定特征维数。
infoGainfinal=infoGainResult[0:N]
print 'infoGainfinal的长度%s' %len(infoGainfinal)
featurewords=[m[0] for m in infoGainfinal]
print '共有%s个特征词'%len(featurewords)
return featurewords
#####################################################################
if __name__=="__main__":
featurewords=featureSelectionIG(1000,0,200)
import cPickle as mypickle
fid=file(r'D:\TextCategorization\VITData\data\keywords.dat','w')
mypickle.dump(featurewords,fid)
fid.close()
'''
此模块用于形成文档向量模型
'''
################################################################
def FormatVSM(sub,root,keywordsaddress):
'''
对文档集建立文档向量模型，储存在一个二维list中
'''
from SVM import DataManager
import cPickle as mypickle
import re
import os
#root=r'D:\tsegmented'
keywords=mypickle.load(file(keywordsaddress))
dm=DataManager.DataManager(root)#读数据专家
VSMMatrix=[]
dm.SetFilePathsFromsubDir(root+os.sep+sub)
filepaths=dm.GetFilePaths()
for path in filepaths:
myfile=file(root+os.sep+sub+os.sep+path)
rawtext=myfile.read()
myfile.close()
textwordslist=FilePreProcess(rawtext)
VSM=[]
for i in range(0,len(keywords)):
count=textwordslist.count(keywords[i])
VSM.append((i+1,count))
VSMMatrix.append(VSM)
return VSMMatrix
####################################################################
def LibSVMFormat(dest,root,keywordsaddress):
'''
形成VSM
'''
fid=file(dest,'a')
VSMMatrix=FormatVSM('class1',root,keywordsaddress)
for VSM in VSMMatrix:
s='1'
for elem in VSM:
if elem[1]!=0:
s=s+' \t'+str(elem[0])+':'+str(elem[1])
s=s+' \t\n'
fid.write(s)
VSMMatrix=FormatVSM('class2',root,keywordsaddress)
for VSM in VSMMatrix:
s='0'
for elem in VSM:
if elem[1]!=0:
s=s+' \t'+str(elem[0])+':'+str(elem[1])
s=s+' \t\n'
fid.write(s)
#print 'finish'
fid.close()
print 'functionfinish'
#############################################################33
def FilePreProcess(rawtext):
listresult=rawtext.split("|")
return listresult
###################################################################
if __name__=="__main__":
root1=r'D:\TextCategorization\segmented'
root2=r'D:\TextCategorization\tsegmented'
print 'begin.....'
LibSVMFormat(r'D:\TextCategorization\data\train.libsvm',root1,keywordsaddress)
print '训练语料库转化完毕'
LibSVMFormat(r'D:\TextCategorization\data\test.libsvm',root2,keywordsaddress)
print '测试语料库转化完毕'
文本预处理的主程序模块，该模块调用上面的各个子模块完成“根据：档集合大小，特征词数目，交叉验证折数的不同需求，快速生成Libsvm格式数据”的功能。# -*- coding: cp936 -*-
#coding gb2312
from SVM import FoldersCreation
import os
##############################################################################################
#参数设计
corpus_size=[1500]
#N: half of total corpus size
vfold=5 #vfold: 循环验证的次数
featureDimensions=[10,20,30,40,50,60,70,80,90,100,110,120,130,140,150] #featureDimension:VSM模型特征维度
toCalInfoGain=0#是否计算词袋子模型中的词集合的信息增益=1则不计算
times=[2]
#count_done_research_times=0#已经进行了几次试验
# N，count_done_research 为CorpusPartition.moveAccordingPartition的参数
#featureDimension,toCalInfoGain 2*N/vfold 为FeatureSelectionModel.featureSelectionIG
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
for count_done_research_times in times:
for N in corpus_size:
print '目前文档集规模为%s'%N
print '目前在该规模文档集上面已经进行了%s次实验'%count_done_research_times
for featureDimension in featureDimensions:
if featureDimension>10:
toCalInfoGain=1
print '目前处理的特征维数是%s'%featureDimension
##############创建文件夹########################################################################
if toCalInfoGain==0:
os.mkdir(r'D:\TextCategorization')
FoldersCreation.CreateAssist(toCalInfoGain)
print '创建文件夹模块运行结束'
print '***************************************************************************'
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
################处理文档集合，对文档集合进行划分，区分测试集合和训练集合###############################
from SVM import CorpusPartition
CorpusPartition.MoveCorpus(N,toCalInfoGain)
CorpusPartition.moveAccordingPartition(N,count_done_research_times,toCalInfoGain)
print '分割文本集模块运行结束'
print '*******************************************************************'
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
#########################文档集合分词##########################################################
from SVM import DataManager
from ctypes import *
import os
import cPickle as p
import re
if toCalInfoGain==0:
roots=[r'D:\TextCategorization\training',r'D:\TextCategorization\testing']
rootfinals=[r'D:\TextCategorization\segmented',r'D:\TextCategorization\tsegmented']
for i in range(0,2):
dm=DataManager.DataManager(roots[i])
subdir=dm.GetSubDir()
filepathstotalsrc=[]
for sub in subdir:
dm.SetFilePathsFromsubDir(roots[i]+os.sep+sub)
filepaths=dm.GetFilePaths()
filepathsassist=[sub+os.sep+path for path in filepaths ]
filepathstotalsrc=filepathstotalsrc+filepathsassist
for path in filepathstotalsrc:
myfile=file(roots[i]+os.sep+path)
s=myfile.read()
myfile.close()
dll=cdll.LoadLibrary("ICTCLAS30.dll")
dll.ICTCLAS_Init(c_char_p("."))
bSuccess = dll.ICTCLAS_ParagraphProcess(c_char_p(s),0)
segmented=c_char_p(bSuccess).value
segmentedtmp=re.sub("\s+",'|',segmented,0)
segmentedfinal=re.sub('\xa1\xa1','',segmentedtmp)
fid=file(rootfinals[i]+os.sep+path,'w')
fid.write(segmentedfinal)
fid.close()
dll.ICTCLAS_Exit()
#print 'finalfinish congratulations!'
print '文档集分词模块运行结束'
print '**********************************************************************'
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
##################建立词袋子模型######################################################################
from SVM import BagOfWordsConstruction
BagOfWordsConstruction.BagOfWordsConstruction(r'D:\TextCategorization\segmented',toCalInfoGain)
print '建立词袋子模型模块运行结束'
print '***********************************************************************************'
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
#######################特征词选择##################################################################
from SVM import FeatureSelectionModel
featurewords=FeatureSelectionModel.featureSelectionIG(featureDimension,toCalInfoGain,2*N/vfold)#feature
import cPickle as mypickle
fid=file(r'D:\TextCategorization\VITData\data\keywords.dat','w')
mypickle.dump(featurewords,fid)
fid.close()
print '特征词选择模块运行结束'
print '*******************************************************************************************'
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~#
#######################文档向量模型建立模块##############################################################
from SVM import VSMformation
import shutil
root1=r'D:\TextCategorization\segmented'
root2=r'D:\TextCategorization\tsegmented'
keywordsaddress=r'D:\TextCategorization\VITData\data\keywords.dat'
print 'begin.....'
VSMformation.LibSVMFormat(r'D:\TextCategorization\data\data\train.libsvm',root1,keywordsaddress)
print '训练语料库转化完毕'
VSMformation.LibSVMFormat(r'D:\TextCategorization\data\data\test.libsvm',root2,keywordsaddress)
print '测试语料库转化完毕'
print '文档向量模型建立模块运行结束'
print '批处理完毕，congratulations!'
os.chdir(r'C:\\Python26')
os.chdir('D:\\TextCategorization')
new_dir='TextCategorization_'+str(count_done_research_times)+'_'+str(N)+'_'+str(featureDimension)
os.mkdir(new_dir)
os.chdir(new_dir)
os.mkdir('data')
os.chdir(r'C:\\Python26')
print os.getcwd()
shutil.move(r'D:\TextCategorization\VITdata\data\keywords.dat','D:\\TextCategorization\\'+new_dir+'\\data')
shutil.move(r'D:\TextCategorization\data\data\train.libsvm','D:\\TextCategorization\\'+new_dir+'\\data')
shutil.move(r'D:\TextCategorization\data\data\test.libsvm','D:\\TextCategorization\\'+new_dir+'\\data')
print'恭喜，文件夹重命名完毕'
print '###########################finish##################################'
os.chdir('D:\\')
print os.getcwd()
if os.path.isdir('TextCategorization'):
os.rename('TextCategorization',str(count_done_research_times)+'_'+str(N)+'_rfinish')
os.chdir(r'C:\Python26')
toCalInfoGain=0
print str(count_done_research_times)+'_'+str(N)+'finish'
