python文本分类
前面博客里面从谣言百科中爬取到了所有类别（10类）的新闻并以文本的形式存储。
现在对这些数据进行分类，上代码：
# -*- coding: utf-8 -*- """ Created on Fri Mar 9 14:18:49 2018 @author: Administrator """ import os import time import random import jieba import nltk import sklearn from sklearn.naive_bayes import MultinomialNB import numpy as np import pylab as pl import matplotlib.pyplot as plt def MakeWordsSet(words_file): words_set = set() with open(words_file, 'r', encoding='UTF-8') as fp: for line in fp.readlines(): word = line.strip() if len(word)>0 and word not in words_set: # 去重 words_set.add(word) return words_set def TextProcessing(folder_path, test_size=0.2): folder_list = os.listdir(folder_path)#获取文件夹下所有子文件夹 data_list = []#获取文本数据 class_list = []#获取类别数据 # 所有类别进行循环 for folder in folder_list: new_folder_path = os.path.join(folder_path, folder)#获得子文件夹路径 files = os.listdir(new_folder_path)#获得子文件夹下所有文件 # 类内循环 j = 0 for file in files: if j > 410: # 每类text样本数最多不超过这个 break with open(os.path.join(new_folder_path, file), 'r', encoding='UTF-8') as fp: raw = fp.read() # print raw ## -------------------------------------------------------------------------------- ## jieba分词 word_cut = jieba.cut(raw, cut_all=False) # 精确模式，返回的结构是一个可迭代的genertor word_list = list(word_cut) # genertor转化为list，每个词unicode格式 ## -------------------------------------------------------------------------------- data_list.append(word_list) class_list.append(folder) j += 1 ## 划分训练集和测试集 # train_data_list, test_data_list, train_class_list, test_class_list = sklearn.cross_validation.train_test_split(data_list, class_list, test_size=test_size) data_class_list = list(zip(data_list, class_list))#zip函数:接受2个序列作为参数，返回tuple列表 random.shuffle(data_class_list)#shuffle() 将序列的所有元素随机排序。 index = int(len(data_class_list)*test_size)+1#数据总量*0.2来划分训练集和测试集 train_list = data_class_list[index:]#训练集为后0.8的数据 test_list = data_class_list[:index]#测试集为前0.2的数据 train_data_list, train_class_list = zip(*train_list)#训练数据集 test_data_list, test_class_list = zip(*test_list)#测试数据集 # 统计词频放入all_words_dict all_words_dict = {} for word_list in train_data_list: for word in word_list: if word in all_words_dict: all_words_dict[word] += 1 else: all_words_dict[word] = 1 # key函数利用词频进行降序排序 all_words_tuple_list = sorted(all_words_dict.items(), key=lambda f:f[1], reverse=True) # 内建函数sorted参数需为list all_words_list = list(zip(*all_words_tuple_list))[0] return all_words_list, train_data_list, test_data_list, train_class_list, test_class_list def words_dict(all_words_list, deleteN, stopwords_set=set()):# 选取特征词:不全为数字，不是停留子，长度在1到5之间 feature_words = [] n = 1 for t in range(deleteN, len(all_words_list), 1): if n > 1000: # feature_words的维度1000 break # print all_words_list[t] if not all_words_list[t].isdigit() and all_words_list[t] not in stopwords_set and 1<len(all_words_list[t])<5:# isdigit() 方法检测字符串是否只由数字组成。 feature_words.append(all_words_list[t]) n += 1 return feature_words def TextFeatures(train_data_list, test_data_list, feature_words, flag='nltk'): def text_features(text, feature_words): text_words = set(text) ## ----------------------------------------------------------------------------------- if flag == 'nltk': ## nltk特征 dict features = {word:1 if word in text_words else 0 for word in feature_words} elif flag == 'sklearn': ## sklearn特征 list features = [1 if word in text_words else 0 for word in feature_words] else: features = [] ## ----------------------------------------------------------------------------------- return features train_feature_list = [text_features(text, feature_words) for text in train_data_list] test_feature_list = [text_features(text, feature_words) for text in test_data_list] return train_feature_list, test_feature_list def TextClassifier(train_feature_list, test_feature_list, train_class_list, test_class_list, flag='nltk'): ## ----------------------------------------------------------------------------------- if flag == 'nltk': ## nltk分类器 train_flist = zip(train_feature_list, train_class_list) test_flist = zip(test_feature_list, test_class_list) classifier = nltk.classify.NaiveBayesClassifier.train(train_flist) test_accuracy = nltk.classify.accuracy(classifier, test_flist) elif flag == 'sklearn': ## sklearn分类器 classifier = MultinomialNB().fit(train_feature_list, train_class_list) test_accuracy = classifier.score(test_feature_list, test_class_list) else: test_accuracy = [] return test_accuracy if __name__ == '__main__': print("start") ## 文本预处理 folder_path = 'F:\\test\\demo' all_words_list, train_data_list, test_data_list, train_class_list, test_class_list = TextProcessing(folder_path, test_size=0.2) # 生成stopwords_set stopwords_file = 'F:\\test\\stopword.txt' stopwords_set = MakeWordsSet(stopwords_file) ## 文本特征提取和分类 # flag = 'nltk' flag = 'sklearn' deleteNs = range(0, 1000, 20) test_accuracy_list = [] for deleteN in deleteNs: # feature_words = words_dict(all_words_list, deleteN) feature_words = words_dict(all_words_list, deleteN, stopwords_set)#特征词; train_feature_list, test_feature_list = TextFeatures(train_data_list, test_data_list, feature_words, flag)#获得训练集以及测试数据集; test_accuracy = TextClassifier(train_feature_list, test_feature_list, train_class_list, test_class_list, flag) test_accuracy_list.append(test_accuracy) print(test_accuracy_list) # 结果评价 plt.figure() plt.plot(deleteNs, test_accuracy_list) plt.title('Relationship of deleteNs and test_accuracy') plt.xlabel('deleteNs') plt.ylabel('test_accuracy') plt.savefig('result.png') print("finished")
运行完分类完成！
