大作业
#多元线性回归模型
from sklearn.datasets import load_boston
#波士顿房价数据集
data = load_boston()
from sklearn.model_selection import train_test_split x_train,x_test,y_train,y_test = train_test_split(data.data,data.target,test_size=0.3) print(x_train.shape,y_train.shape)
data.target[0:20]
data.keys()
data.feature_names
len(data.target)
len(x_train)
#线性回归模型:建立13个变量与房价之间的预测模型,并检测模型好
#建立多元线性回归模型 from sklearn.linear_model import LinearRegression mlr = LinearRegression() mlr.fit(x_train,y_train) print('系数',mlr.coef_,"\n截距",mlr.intercept_)
![](https://img2018.cnblogs.com/blog/1482929/201812/1482929-20181226184609348-353617829.png)
#检测模型的好坏
from sklearn.metrics import regression y_predict = mlr.predict(x_test) #计算模型的预测指标 print('线性回归模型') print("预测的均方误差:",regression.mean_squared_error(y_test,y_predict)) print("预测的平均绝对误差:",regression.mean_absolute_error(y_test,y_predict)) print("模型的分数:",mlr.score(x_test,y_test))
#多项式回归模型:建立13个变量与房价之间的预测模型,并检测模型好坏
from sklearn.preprocessing import PolynomialFeatures
#多项式化 poly2 =PolynomialFeatures(degree=2) x_poly_train = poly2.fit_transform(x_train) x_poly_test = poly2.transform(x_test)
#建立模型 mlrp=LinearRegression() mlrp.fit(x_poly_train,y_train)
#预测 y_predict2 = mlrp.predict(x_poly_test) #计算模型的预测指标 print("多项式回归模型") print("预测的均方误差:",regression.mean_squared_error(y_test,y_predict2)) print("预测平均绝对误差:",regression.mean_absolute_error(y_test,y_predict2)) print("模型的分数:",mlrp.score(x_poly_test,y_test))
#比较线性模型与非线性模型的性能,并说明原因
多项式模型是一条平滑的曲线,更贴近样本点的分布,并且误差比线性小,所以非线性模型的性能比线性的性能较好
import os import numpy as np import sys from datetime import datetime import gc path = 'F:\\147'
import jieba with open(r'E:\\stopsCN.txt',encoding='utf-8') as f: stopwords = f.read().split('\n')
def processing(tokens): #去掉非字母汉字的字符 tokens = "".join([char for char in tokens if char.isalpha()]) #结巴分词 tokens = [token for token in jieba.cut(tokens,cut_all=True) if len(token) >=2] #去掉停用词 tokens = " ".join([token for token in tokens if token not in stopwords]) return tokens
tokenList = [] targetList = [] #用os.walk获取需要的变量,并拼接文件路径打开每一个文件 for root,dirs,files in os.walk(path): for f in files: filePath = os.path.join(root,f) with open(filePath,encoding='utf-8') as f: content = f.read()
#获取新闻类别标签,并处理该新闻 target = filePath.split('\\')[-2] targetList.append(target) tokenList.append(processing(content))
# 划分训练集测试集并建立特征向量,为建立模型做准备 # 划分训练集测试集
from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.model_selection import train_test_split from sklearn.naive_bayes import GaussianNB,MultinomialNB from sklearn.model_selection import cross_val_score from sklearn.metrics import classification_report
x_train,x_test,y_train,y_test = train_test_split(tokenList,targetList,test_size=0.3,stratify=targetList)
#选择TfidfVectorizer建立特征向量
vectorizer = TfidfVectorizer() X_train = vectorizer.fit_transform(x_train) X_test = vectorizer.transform(x_test)
#用多项式朴素贝叶斯建立模型
from sklearn.naive_bayes import MultinomialNB mnb = MultinomialNB() module = mnb.fit(X_train,y_train)
#预测
y_predict = module.predict(X_test)
#输出模型的精确度 scores=cross_val_score(mnb,X_test,y_test,cv=5) print("Accuracy:%.3f"%scores.mean())
#输出模型评估报告 print("classification_report:\n",classification_report(y_predict,y_test))
targetList.append(target) print(targetList[0:10]) tokenList.append(processing(content)) tokenList[0:10]
#统计测试集和预测集的各类新闻个数
import collections testCount = collections.Counter(y_test) predCount = collections.Counter(y_predict) print('实际:',testCount,'\n','预测',predCount)