汽车数据分析
背景介绍:
我们采用的数据是:根据汽车的多种细节,如:车门数量,后备箱大小,维修成本等,来确定汽车质量。
最后的分类目的是把车辆的质量分为4中类型,不达标,达标,良好,优秀。点击: [ 数据集下载
第一步:读取数据
我们的数据存放在一个txt文件中。
def read_data(path):
data = []
with open(path) as trainData:
lines_data = trainData.readlines()
for line in lines_data:
line_ = line.strip().split(',')
line = line.strip('\n').split(',')
data.append(line)
return data
if __name__ == '__main__':
path = './data/car.data.txt'
data = read_data(path)
data = np.array(data)
print(data)
print(data.shape)
数据展示
1728条数据,6个特征
第二步:我们将数据中的字符转换为数字
def process_data(data):
# 预处理数据
# 1.将字符串转化为数字
label_code = []
X_encoded = np.empty(data.shape)
for i, item in enumerate(data[1]):
label_code.append(preprocessing.LabelEncoder())
X_encoded[:, i] = label_code[-1].fit_transform(data[:, i])
X = X_encoded[:, : -1].astype(np.int)
y = X_encoded[:, -1].astype(np.int)
return X, y
if __name__ == '__main__':
path = './data/car.data.txt'
data = read_data(path)
data = np.array(data)
# print(data)
# print(data.shape)
# 下一步 我们将字符串转换数字
X, y = process_data(data)
[/code]
我们此处通过LabelEncoder()进行转换
### 第三步:建立模型进行训练
此处采用随机森林,用100个基分类器
```code
def model(X, y):
rf = RandomForestClassifier(n_estimators=100, max_depth=8, random_state=42)
rf.fit(X, y)
accuracy = cross_val_score(rf, X, y, scoring='accuracy', cv=3)
print(accuracy)
if __name__ == '__main__':
path = './data/car.data.txt'
data = read_data(path)
data = np.array(data)
# print(data)
# print(data.shape)
# 下一步 我们将字符串转换数字
X, y = process_data(data)
# 建立模型 并训练数据
model(X, y)
输出结果:
第四步: 画验证曲线
我们可以通过该随机森林中基分类器的个数,然后画出随机森林中验证集准确率与基分类器个数的关系
def get_validation_curve(X, y):
classifer = RandomForestClassifier(max_depth=4, random_state=42)
parameter_grid = np.linspace(25, 200, 8).astype(int)
train_scores, validation_scores = validation_curve(classifer, X, y, 'n_estimators', parameter_grid, cv=5)
print("训练数据的准确率:", train_scores)
print("验证数据的准确率:", validation_scores)
# 把数据画成图像
plt.figure(figsize=(10, 8), dpi=80)
plt.plot(parameter_grid, 100*np.average(train_scores, axis=1), color='black')
plt.title('Training curve')
plt.xlabel('Number of estimators')
plt.ylabel('Accuracy')
plt.show()
if __name__ == '__main__':
path = './data/car.data.txt'
data = read_data(path)
data = np.array(data)
# print(data)
# print(data.shape)
# 下一步 我们将字符串转换数字
X, y = process_data(data)
# 建立模型 并训练数据
# model(X, y)
get_validation_curve(X, y)
[/code]
输出结果:
![](https://img-blog.csdnimg.cn/20190305105900997.png?x-oss-
process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3NoYXdyb2FkODg=,size_16,color_FFFFFF,t_70)
![](https://img-blog.csdnimg.cn/20190305105944970.png?x-oss-
process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3NoYXdyb2FkODg=,size_16,color_FFFFFF,t_70)
可以看出基分类器是越多准确率越高。 同理, 我们验证一下每个基分类器中树的高度对准确率的影响
```code
def get_validation_curve1(X, y):
classifer = RandomForestClassifier(n_estimators=30, random_state=42)
parameter_grid = np.linspace(2, 10, 5).astype(int)
train_scores, validation_scores = validation_curve(classifer, X, y, 'max_depth', parameter_grid, cv=5)
print("训练数据的准确率:", train_scores)
print("验证数据的准确率:", validation_scores)
# 把数据画成图像
plt.figure(figsize=(10, 8), dpi=80)
plt.plot(parameter_grid, 100 * np.average(train_scores, axis=1), color='black')
plt.title('Validation curve')
plt.xlabel('Maximum depth of the tree')
plt.ylabel('Accuracy')
plt.show()
if __name__ == '__main__':
path = './data/car.data.txt'
data = read_data(path)
data = np.array(data)
# print(data)
# print(data.shape)
# 下一步 我们将字符串转换数字
X, y = process_data(data)
# 建立模型 并训练数据
# model(X, y)
# get_validation_curve(X, y)
# 可以用类似的方法对max_depth参数进行验证
get_validation_curve1(X, y)
输出结果:
![](https://img-blog.csdnimg.cn/20190305110357288.png?x-oss-
process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3NoYXdyb2FkODg=,size_16,color_FFFFFF,t_70)
![](https://img-blog.csdnimg.cn/20190305110418614.png?x-oss-
process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3NoYXdyb2FkODg=,size_16,color_FFFFFF,t_70)
深度越深,验证集的准确率越高。
第五步:画学习曲线
也就是根据样本量的大小,看准确率的变化啊
def get_learning_curve(X, y):
classifer = RandomForestClassifier(random_state=42)
parameter_grid = np.array([200, 500, 800, 1100])
train_size, train_scores, validation_scores = learning_curve(classifer, X, y, train_sizes=parameter_grid, cv=5)
print("训练数据的准确率:", train_scores)
print("验证数据的准确率:", validation_scores)
# 把数据画成图像
plt.figure(figsize=(10, 8), dpi=80)
plt.plot(parameter_grid, 100 * np.average(train_scores, axis=1), color='black')
plt.title('Learning curve')
plt.xlabel('Number of training samples')
plt.ylabel('Accuracy')
plt.show()
if __name__ == '__main__':
path = './data/car.data.txt'
data = read_data(path)
data = np.array(data)
# print(data)
# print(data.shape)
# 下一步 我们将字符串转换数字
X, y = process_data(data)
# 建立模型 并训练数据
# model(X, y)
# get_validation_curve(X, y)
# 可以用类似的方法对max_depth参数进行验证
# get_validation_curve1(X, y)
# 生成学习曲线 也就是训练样本和准确率之间的关系
get_learning_curve(X, y)
[/code]
输出结果:
![](https://img-blog.csdnimg.cn/20190305110725452.png?x-oss-
process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3NoYXdyb2FkODg=,size_16,color_FFFFFF,t_70)
学习曲线
![](https://img-blog.csdnimg.cn/20190305110806319.png?x-oss-
process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3NoYXdyb2FkODg=,size_16,color_FFFFFF,t_70)
基本可以确定样本快600的时候准确率很高,超过600出现了过拟合。验证集表现不佳
_源代码:_
```code
import numpy as np
import pandas as pd
from sklearn import preprocessing
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import KFold
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import validation_curve
import matplotlib.pyplot as plt
from sklearn.model_selection import learning_curve
import warnings
warnings.filterwarnings('ignore')
def read_data(path):
data = []
with open(path) as trainData:
lines_data = trainData.readlines()
for line in lines_data:
line_ = line.strip().split(',')
line = line.strip('\n').split(',')
data.append(line)
return data
def process_data(data):
# 预处理数据
# 1.将字符串转化为数字
label_code = []
X_encoded = np.empty(data.shape)
for i, item in enumerate(data[1]):
label_code.append(preprocessing.LabelEncoder())
X_encoded[:, i] = label_code[-1].fit_transform(data[:, i])
X = X_encoded[:, : -1].astype(np.int)
y = X_encoded[:, -1].astype(np.int)
return X, y
def model(X, y):
rf = RandomForestClassifier(n_estimators=100, max_depth=8, random_state=42)
rf.fit(X, y)
accuracy = cross_val_score(rf, X, y, scoring='accuracy', cv=3)
print('随机森林的三折交叉验证的准确率:\n', accuracy)
# # 使用交叉验证
# kfold = KFold(n_splits=5, shuffle=True, random_state=42)
# for train, test in kfold.split(X):
# rf = RandomForestClassifier(n_estimators=100, max_depth=8, random_state=42)
# rf.fit(X[train], y[train])
# print("训练集上的准确率:{}, 测试集上的准确率:{}".format(
# rf.score(X[train], y[train]),
# rf.score(X[test], y[test])
# ))
def get_validation_curve(X, y):
classifer = RandomForestClassifier(max_depth=4, random_state=42)
parameter_grid = np.linspace(25, 200, 8).astype(int)
train_scores, validation_scores = validation_curve(classifer, X, y, 'n_estimators', parameter_grid, cv=5)
print("训练数据的准确率:", train_scores)
print("验证数据的准确率:", validation_scores)
# 把数据画成图像
plt.figure(figsize=(10, 8), dpi=80)
plt.plot(parameter_grid, 100*np.average(train_scores, axis=1), color='black')
plt.title('Training curve')
plt.xlabel('Number of estimators')
plt.ylabel('Accuracy')
plt.show()
def get_validation_curve1(X, y):
classifer = RandomForestClassifier(n_estimators=30, random_state=42)
parameter_grid = np.linspace(2, 10, 5).astype(int)
train_scores, validation_scores = validation_curve(classifer, X, y, 'max_depth', parameter_grid, cv=5)
print("训练数据的准确率:\n", train_scores)
print("验证数据的准确率:\n", validation_scores)
# 把数据画成图像
plt.figure(figsize=(10, 8), dpi=80)
plt.plot(parameter_grid, 100 * np.average(train_scores, axis=1), color='black')
plt.title('Validation curve')
plt.xlabel('Maximum depth of the tree')
plt.ylabel('Accuracy')
plt.show()
def get_learning_curve(X, y):
classifer = RandomForestClassifier(random_state=42)
parameter_grid = np.array([200, 500, 800, 1100])
train_size, train_scores, validation_scores = learning_curve(classifer, X, y, train_sizes=parameter_grid, cv=5)
print("训练数据的准确率:\n", train_scores)
print("验证数据的准确率:\n", validation_scores)
# 把数据画成图像
plt.figure(figsize=(10, 8), dpi=80)
plt.plot(parameter_grid, 100 * np.average(train_scores, axis=1), color='black')
plt.title('Learning curve')
plt.xlabel('Number of training samples')
plt.ylabel('Accuracy')
plt.show()
if __name__ == '__main__':
path = './data/car.data.txt'
data = read_data(path)
data = np.array(data)
# print(data)
# print(data.shape)
# 下一步 我们将字符串转换数字
X, y = process_data(data)
# 建立模型 并训练数据
# model(X, y)
# get_validation_curve(X, y)
# 可以用类似的方法对max_depth参数进行验证
# get_validation_curve1(X, y)
# 生成学习曲线 也就是训练样本和准确率之间的关系
get_learning_curve(X, y)