#!/usr/bin/env Python
# coding:utf-8
#improt依赖包
# import sys
# reload(sys)
# sys.setdefaultencoding('utf-8')
import chardet
from gensim import utils
from gensim.models.doc2vec import LabeledSentence
from gensim.models import Doc2Vec
import numpy
from random import shuffle
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import confusion_matrix
import sklearn.metrics as metrics
# Doc2vec需要以LabeledLineSentece对象作为输入，所以需要构建一个类将文本转化为LabeledLineStentece对象
class LabeledLineSentence(object):
def __init__(self, sources):
self.sources = sources
flipped = {}
# make sure that keys are unique
for key, value in sources.items():
if value not in flipped:
flipped[value] = [key]
else:
raise Exception('Non-unique prefix encountered')
def __iter__(self):
for source, prefix in self.sources.items():
with utils.smart_open(source) as fin:
for item_no, line in enumerate(fin):
yield LabeledSentence(utils.to_unicode(line).split(), [prefix + '_%s' % item_no])
def to_array(self):
self.sentences = []
for source, prefix in self.sources.items():
with utils.smart_open(source) as fin:
for item_no, line in enumerate(fin):
print chardet.detect(line)
line=line.decode("GB2312",'ignore').encode("utf-8")
print chardet.detect(line)
self.sentences.append(LabeledSentence(utils.to_unicode(line).split(), [prefix + '_%s' % item_no]))
# self.sentences.append(LabeledSentence(utils.to_utf8(line).split(), [prefix + '_%s' % item_no]))
return self.sentences
def sentences_perm(self):
shuffle(self.sentences)
return self.sentences
#将文本数据以以下方式导入到Doc2vec中
# sources = {u'/Volumes/Macintosh HD/Users/RayChou/Downloads/情感分析训练语料/neg_train.txt':'TRAIN_NEG',
# u'/Volumes/Macintosh HD/Users/RayChou/Downloads/情感分析训练语料/pos_train.txt':'TRAIN_POS'
# ,u'/Volumes/Macintosh HD/Users/RayChou/Downloads/情感分析训练语料/uns_train.txt':'TRAIN_UNS',
# u'/Volumes/Macintosh HD/Users/RayChou/Downloads/情感分析训练语料/uns_test.txt':'TEST_UNS'}
sources = {\
'./yuliao/fYuliao0.txt':'TRAIN_0',
'./yuliao/fYuliao1.txt':'TRAIN_1',
'./yuliao/fYuliao2.txt':'TRAIN_2',
'./yuliao/fYuliao3.txt':'TRAIN_3',
'./yuliao/fYuliao4.txt':'TRAIN_4',
'./yuliao/fYuliao5.txt':'TRAIN_5',\
}
sentences = LabeledLineSentence(sources)
#构建Doc2vec模型
model = Doc2Vec(min_count=1, window=15, size=100, sample=1e-4, negative=5, workers=8)
model.build_vocab(sentences.to_array())
#训练Doc2vec模型（本例迭代次数为10，如果时间允许，可以迭代更多的次数）
for epoch in range(2):
model.train(sentences.sentences_perm())
model.save("model.txt")
# model=Doc2Vec.load("model.txt")
#将训练好的句子向量装进array里面，后文作为分类器的输入
train_arrays = numpy.zeros((5000, 100))
train_labels = numpy.zeros(5000)
test_arrays = []
true_labels=[]
train_data=[]
train_lb=[]
for i in range(5000):
if(i<=645):
prefix_train_0 = 'TRAIN_0_' + str(i)
train_arrays[i] = model.docvecs[prefix_train_0]
train_labels[i] = 0
elif(i>645 and i<=4249):
j=i-646
prefix_train_1 = 'TRAIN_1_' + str(j)
train_arrays[i]=model.docvecs[prefix_train_1]
train_labels[i]=1
elif(i>4249 and i<=4800):
j=i-4250
prefix_train_2 = 'TRAIN_2_' + str(j)
train_arrays[i]=model.docvecs[prefix_train_2]
train_labels[i]=2
elif(i>4800 and i<=4965):
j=i-4801
prefix_train_3 = 'TRAIN_3_' + str(j)
train_arrays[i]=model.docvecs[prefix_train_3]
train_labels[i]=3
elif(i>4965 and i<=4994):
j=i-4966
prefix_train_4 = 'TRAIN_4_' + str(j)
train_arrays[i]=model.docvecs[prefix_train_4]
train_labels[i]=4
else:
j=i-4995
prefix_train_5 = 'TRAIN_5_' + str(j)
train_arrays[i]=model.docvecs[prefix_train_5]
train_labels[i]=5
#载入测试集数据
a=open("./yuliao/fYuliao0_test.txt")
b=open("./yuliao/fYuliao1_test.txt")
c=open("./yuliao/fYuliao2_test.txt")
d=open("./yuliao/fYuliao3_test.txt")
e=open("./yuliao/fYuliao4_test.txt")
f=open("./yuliao/fYuliao5_test.txt")
test_content1=a.readlines()
test_content2=b.readlines()
test_content3=c.readlines()
test_content4=d.readlines()
test_content5=e.readlines()
test_content6=f.readlines()
g=open("./yuliao/fYuliao0_test.txt")
test_content7=g.readline()
inferred_docvec=model.infer_vector(test_content7)
print model.docvecs.most_similar([inferred_docvec], topn=3)
for i in test_content1:
test_arrays.append(model.infer_vector(i))
true_labels.append(0)
for i in test_content2:
test_arrays.append(model.infer_vector(i))
true_labels.append(1)
for i in test_content3:
test_arrays.append(model.infer_vector(i))
true_labels.append(2)
for i in test_content4:
test_arrays.append(model.infer_vector(i))
true_labels.append(3)
for i in test_content5:
test_arrays.append(model.infer_vector(i))
true_labels.append(4)
for i in test_content6:
test_arrays.append(model.infer_vector(i))
true_labels.append(5)
#构建逻辑回归分类器
classifier = LogisticRegression(class_weight={0:0.38,1:0.62})
classifier.fit(train_arrays, train_labels)
# 构建随机森林分类器
'''
from sklearn.ensemble import RandomForestClassifier
RF = RandomForestClassifier(n_estimators=1200,max_depth=14,class_weight={0:0.3,1:0.7})
RF.fit(train_arrays, train_labels)
'''
#构建GBDT分类器
'''
from sklearn.ensemble import GradientBoostingClassifier
GBDT = GradientBoostingClassifier(n_estimators=1000,max_depth=14)
GBDT.fit(train_arrays, train_labels)
'''
#对Test数据进行预测
test_labels_LR=[]
# test_labels_RF=[]
# test_labels_GBDT=[]
for i in range(len(test_arrays)):
test_labels_LR.append(classifier.predict(test_arrays[i]))
'''
test_labels_RF.append(RF.predict(test_arrays[i]))
test_labels_GBDT.append(GBDT.predict(test_arrays[i]))
'''
#打印各个模型的准确率和召回率
print("LR:")
test_labels_LR1 = []
count = 0
for i in range(len(test_labels_LR)):
if (test_labels_LR[i][0] == true_labels[i]):
count +=1
print count
'''
print("RF:")
print(metrics.accuracy_score(test_labels_RF,true_labels))
print(confusion_matrix(test_labels_RF,true_labels))
print("GBDT:")
print(metrics.accuracy_score(test_labels_GBDT,true_labels))
print(confusion_matrix(test_labels_GBDT,true_labels))
'''
