期末大作业
一、boston房价预测
#1、导入数据集 from sklearn.datasets import load_boston boston=load_boston() x=boston.data y=boston.target print(x.shape) print(y.shape) #2、划分数据集 from sklearn.model_selection import train_test_split from sklearn.linear_model import LinearRegression #将数据集划分成75%的训练集和25%的测试集 x_train,x_test,y_train,y_test=train_test_split(x,y,test_size=0.25) #查看数据集的shape print("train dataset's shape is :",x_train.shape) print("test dataset's shape is :",x_test.shape) #3、多元线性回归模型,建立13个变量与房价之间的预测模型,并检测模型好坏。 from sklearn.linear_model import LinearRegression import matplotlib.pyplot as plt linR=LinearRegression() linR.fit(x_train,y_train) print("系数:",linR.coef_,"\n截距:",linR.intercept_) from sklearn.metrics import regression y_predict=linR.predict(x_test) # 计算模型的预测指标 print("预测的均方误差:",regression.mean_squared_error(y_test,y_predict)) print("预测的平均绝对误差:",regression.mean_absolute_error(y_test,y_predict)) print("模型的分数:",linR.score(x_test,y_test)) #4. 多项式回归模型:建立13个变量与房价之间的预测模型,并检测模型好坏。 from sklearn.preprocessing import PolynomialFeatures poly=PolynomialFeatures(degree=2) x_poly_train=poly.fit_transform(x_train) x_poly_test=poly.transform(x_test) #建立模型 lrp=LinearRegression() lrp.fit(x_poly_train,y_train) #预测 y_predict1=lrp.predict(x_poly_test) #检测模型好坏 print("预测的均方误差:",regression.mean_squared_error(y_test,y_predict1)) print("预测的平均绝对误差:",regression.mean_absolute_error(y_test,y_predict1)) print("模型的分数:",lrp.score(x_poly_test,y_test))
多元线性回归模型
多项式回归模型
5. 比较线性模型与非线性模型的性能,并说明原因
从预测的均方误差看,多项式回归模型的均方误差比多元线性回归模型小,从预测的平均绝对误差看,多项式回归模型的平均绝对误差
比多元线性回归模型小。说明多项式回归模型更好,更贴近样本点的分布。
二、中文文本分类
import os import numpy as np import sys from datetime import datetime import gc path = 'F:\\python大作业\\0369'
# 导入结巴库 import jieba # 导入停用词 with open(r'F:\python大作业\stopsCN.txt', encoding='utf-8') as f: stopwords = f.read().split('\n')
stopwords[0:10]
def processing(tokens): # 去掉非字母汉字的字符 tokens = "".join([char for char in tokens if char.isalpha()]) # 结巴分词 tokens = [token for token in jieba.cut(tokens,cut_all=True) if len(token) >=2] # 去掉停用词 tokens = " ".join([token for token in tokens if token not in stopwords]) return tokens
tokenList = []#处理好的新闻 targetList = []#新闻的标签 # 用os.walk获取需要的变量,并拼接文件路径再打开每一个文件 for root,dirs,files in os.walk(path):#根据根目录里的文件夹名找到对应的新闻类别的文件 for f in files: filePath = os.path.join(root,f) with open(filePath, encoding='utf-8') as f: content = f.read() # 获取新闻类别标签,并处理该新闻 target = filePath.split('\\')[-2] targetList.append(target) tokenList.append(processing(content))
processing(content)
# 划分训练集测试集 from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.model_selection import train_test_split from sklearn.naive_bayes import GaussianNB,MultinomialNB from sklearn.model_selection import cross_val_score from sklearn.metrics import classification_report #stratify=targetList划分处理好的新闻 x_train,x_test,y_train,y_test = train_test_split(tokenList,targetList,test_size=0.2,stratify=targetList) # 转化为特征向量 vectorizer = TfidfVectorizer() X_train = vectorizer.fit_transform(x_train) X_test = vectorizer.transform(x_test) # 建立模型 mnb = MultinomialNB() module = mnb.fit(X_train, y_train)
#进行预测 y_predict = module.predict(X_test) # 输出模型精确度 scores=cross_val_score(mnb,X_test,y_test,cv=5) print("Accuracy:%.3f"%scores.mean()) # 输出模型评估报告 print("classification_report:\n",classification_report(y_predict,y_test)) # 将预测结果和实际结果进行对比 import collections import matplotlib.pyplot as plt from pylab import mpl mpl.rcParams['font.sans-serif'] = ['FangSong'] # 指定默认字体 mpl.rcParams['axes.unicode_minus'] = False # 解决图像无法显示中文的问题
#precision表示预测的精确度,recall表示实际的精确度,f1-score=2recallprecision/(recall+precision)
# 统计测试集和预测集的各类新闻个数 testCount = collections.Counter(y_test) predCount = collections.Counter(y_predict) print('实际:',testCount,'\n', '预测', predCount)
# 建立标签列表,实际结果列表,预测结果列表, nameList = list(testCount.keys()) testList = list(testCount.values()) predictList = list(predCount.values()) x = list(range(len(nameList))) print("新闻类别:",nameList,'\n',"实际:",testList,'\n',"预测:",predictList)