期末大作业

一、boston房价预测

from sklearn.datasets import load_boston
boston = load_boston() #读取波士顿数据集

from sklearn.model_selection import train_test_split

x_train, x_test, y_train, y_test = train_test_split(boston.data,boston.target,test_size=0.3)
0
print(x_train.shape,y_train.shape)

from sklearn.linear_model import LinearRegression #导线性回归模型包
m = LinearRegression()
m.fit(x_train,y_train)#建立模型，fit是一种方法
w = m.coef_
b = m.intercept_
print("系数是：",w,"截距是：",b)

from sklearn.metrics import regression 
y_predict = m.predict(x_test)  #测试集用来做预测
# 计算模型的预测指标(模型预测的房价与真实房价的误差)
print("预测的均方误差：", regression.mean_squared_error(y_test,y_predict))
print("预测的平均绝对误差：", regression.mean_absolute_error(y_test,y_predict))
# 打印模型的分数
print("模型的分数：",m.score(x_test, y_test))#检测模型好坏

from sklearn.preprocessing import PolynomialFeatures 
poly2 = PolynomialFeatures(degree=2) #调用多项式方法
x_poly_train = poly2.fit_transform(x_train) #多项式化训练集
x_poly_test = poly2.transform(x_test) #多项式化测试集

mp = LinearRegression()
mp.fit(x_poly_train, y_train)# 建立多项式回归模型

y_predict2 = mp.predict(x_poly_test)# 多项式化测试集做预测
# 计算模型的预测指标(模型预测的房价与真实房价的误差)
print("预测的均方误差：", regression.mean_squared_error(y_test,y_predict2))
print("预测的平均绝对误差：", regression.mean_absolute_error(y_test,y_predict2))
print("模型的分数：",mp.score(x_poly_test, y_test))#检测模型好坏

总结：比较线性模型与非线性模型的分数，可知非线性模型的性能比线性模型的性能好，因为非线性模型的参数更多，误差更少。

 

二、中文文本分类

import os
path = 'C:\\Users\\AAA\\Desktop\\0369' #获取文件夹

for root,dirs,files in os.walk(path): 
    print(root)  #当前目录路径
    print(dirs)  #当前路径下所有子目录
    print(files) #当前路径下所有非目录子文件

import jieba  #导入jieba库

file_path = r'F:\\PyCharm\\stopsCN.txt'#读取停用词
stopwords = open(file_path,'r',encoding='utf-8').read() 

def processing(tokens):
    tokens = "".join([char for char in tokens if char.isalpha()])# 去掉符号
    tokens = [token for token in jieba.cut(tokens,cut_all=True) if len(token) >=2]# 结巴分词
    tokens = " ".join([token for token in tokens if token not in stopwords])# 去掉停用词
    return tokens

tokenList=[]   #处理后的文本列表
targetList=[]  #处理后的标签列表
#tokenList[0:2] #查看前两个文件内容
#len(targetList) #查看标签个数

# 用os.walk获取需要的变量，并拼接文件路径再打开每一个文件
for root,dirs,files in os.walk(path):# 用os.walk获取路径，目录，文件
    for f in files:#遍历文件
        filePath = os.path.join(root,f)
        with open(filePath, encoding='utf-8') as f:
            content = f.read()
            # 获取新闻类别标签，并处理该新闻
        target = filePath.split('\\')[-2]
        targetList.append(target) #把标签追加记录与targetList
        tokenList.append(processing(content)) #把文本内容追加记录与tokenList

for root,dirs,files in os.walk(path):    # 用os.walk获取路径，目录，文件
    for f in files:                      #遍历文件
        filePath = os.path.join(root,f)
        print(filePath)#得到文件路径

#导包
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import GaussianNB,MultinomialNB
from sklearn.model_selection import cross_val_score
from sklearn.metrics import classification_report
x_train,x_test,y_train,y_test = train_test_split(tokenList,targetList,test_size=0.2,stratify=targetList)
#划分训练集和测试集
vectorizer = TfidfVectorizer() #TfidfVectorizer的方式建立特征向量
X_train = vectorizer.fit_transform(x_train)
X_test = vectorizer.transform(x_test)

mnb = MultinomialNB() #贝叶斯预测种类
module = mnb.fit(X_train, y_train)

y_predict = module.predict(X_test)#进行预测
scores=cross_val_score(mnb,X_test,y_test,cv=5)# 输出模型精确度
print("Accuracy:%.3f"%scores.mean())
print("classification_report:\n",classification_report(y_predict,y_test))# 输出模型评估报告

import collections
# 统计测试集和预测集的各类新闻个数
testCount = collections.Counter(y_test)
predCount = collections.Counter(y_predict)
print('实际：',testCount,'\n', '预测', predCount)