一.数据产生
1 from sklearn.datasets import make_classification, make_blobs 2 from matplotlib.colors import ListedColormap 3 from sklearn.datasets import load_breast_cancer 4 from adspy_shared_utilities import load_crime_dataset 5 6 cmap_bold = ListedColormap(['#FFFF00', '#00FF00', '#0000FF','#000000']) 7 8 #make_regression:随机产生回归模型的数据 9 #参数：n_samples : 数据个数 10 #n_features:数据中变量个数 11 #n_informative:有关变量个数 12 #bias:线性模型中的偏差项 13 #noise:高斯分布的标准差 14 #random_state:随机数的种子生成器 15 16 # 简单(一个参数)的回归数据 17 from sklearn.datasets import make_regression 18 plt.figure() 19 plt.title('Sample regression problem with one input variable') 20 X_R1, y_R1 = make_regression(n_samples = 100, n_features=1, 21 n_informative=1, bias = 150.0, 22 noise = 30, random_state=0) 23 plt.scatter(X_R1, y_R1, marker= 'o', s=50) 24 plt.show() 25 26 27 # 复杂(多参)的回归数据产生 28 from sklearn.datasets import make_friedman1 29 plt.figure() 30 plt.title('Complex regression problem with one input variable') 31 X_F1, y_F1 = make_friedman1(n_samples = 100, 32 n_features = 7, random_state=0) 33 34 plt.scatter(X_F1[:, 2], y_F1, marker= 'o', s=50) 35 plt.show() 36 37 # 分类模型的数据生成 38 plt.figure() 39 plt.title('Sample binary classification problem with two informative features') 40 X_C2, y_C2 = make_classification(n_samples = 100, n_features=2, 41 n_redundant=0, n_informative=2, 42 n_clusters_per_class=1, flip_y = 0.1, 43 class_sep = 0.5, random_state=0) 44 plt.scatter(X_C2[:, 0], X_C2[:, 1], c=y_C2, 45 marker= 'o', s=50, cmap=cmap_bold) 46 plt.show() 47 48 49 # more difficult synthetic dataset for classification (binary) 50 # with classes that are not linearly separable 51 X_D2, y_D2 = make_blobs(n_samples = 100, n_features = 2, centers = 8, 52 cluster_std = 1.3, random_state = 4) 53 y_D2 = y_D2 % 2 54 plt.figure() 55 plt.title('Sample binary classification problem with non-linearly separable classes') 56 plt.scatter(X_D2[:,0], X_D2[:,1], c=y_D2, 57 marker= 'o', s=50, cmap=cmap_bold) 58 plt.show() 59 60 61 # 乳腺癌分类数据集 62 cancer = load_breast_cancer() 63 (X_cancer, y_cancer) = load_breast_cancer(return_X_y = True) 64 65 66 # Communities and Crime dataset 67 (X_crime, y_crime) = load_crime_dataset()
KNN分类
1 from adspy_shared_utilities import plot_two_class_knn 2 3 X_train, X_test, y_train, y_test = train_test_split(X_C2, y_C2, 4 random_state=0) 5 6 #k为knn所选最近邻居个数 7 plot_two_class_knn(X_train, y_train, 1, 'uniform', X_test, y_test) 8 plot_two_class_knn(X_train, y_train, 3, 'uniform', X_test, y_test) 9 plot_two_class_knn(X_train, y_train, 11, 'uniform', X_test, y_test)
KNN回归预测
1 from sklearn.neighbors import KNeighborsRegressor 2 3 X_train, X_test, y_train, y_test = train_test_split(X_R1, y_R1, random_state = 0) 4 5 knnreg = KNeighborsRegressor(n_neighbors = 5).fit(X_train, y_train) 6 7 print(knnreg.predict(X_test)) 8 print('R-squared test score: {:.3f}' 9 .format(knnreg.score(X_test, y_test)))
[ 231.71 148.36 150.59 150.59 72.15 166.51 141.91 235.57 208.26 102.1 191.32 134.5 228.32 148.36 159.17 113.47 144.04 199.23 143.19 166.51 231.71 208.26 128.02 123.14 141.91] R-squared test score: 0.425
#检验k对KNN预测模型结果的影响
1 fig, subaxes = plt.subplots(1, 2, figsize=(8,4)) 2 X_predict_input = np.linspace(-3, 3, 50).reshape(-1,1) 3 X_train, X_test, y_train, y_test = train_test_split(X_R1[0::5], y_R1[0::5], random_state = 0) 4 5 for thisaxis, K in zip(subaxes, [1, 3]): 6 knnreg = KNeighborsRegressor(n_neighbors = K).fit(X_train, y_train) 7 y_predict_output = knnreg.predict(X_predict_input) 8 thisaxis.set_xlim([-2.5, 0.75]) 9 thisaxis.plot(X_predict_input, y_predict_output, '^', markersize = 10, 10 label='Predicted', alpha=0.8) 11 thisaxis.plot(X_train, y_train, 'o', label='True Value', alpha=0.8) 12 thisaxis.set_xlabel('Input feature') 13 thisaxis.set_ylabel('Target value') 14 thisaxis.set_title('KNN regression (K={})'.format(K)) 15 thisaxis.legend() 16 plt.tight_layout()
1 # plot k-NN regression on sample dataset for different values of K 2 fig, subaxes = plt.subplots(5, 1, figsize=(5,20))
#生成（-3，3）区间内500个数据 3 X_predict_input = np.linspace(-3, 3, 500).reshape(-1,1) 4 X_train, X_test, y_train, y_test = train_test_split(X_R1, y_R1, 5 random_state = 0) 6 7 for thisaxis, K in zip(subaxes, [1, 3, 7, 15, 55]): 8 knnreg = KNeighborsRegressor(n_neighbors = K).fit(X_train, y_train) 9 y_predict_output = knnreg.predict(X_predict_input) 10 train_score = knnreg.score(X_train, y_train) 11 test_score = knnreg.score(X_test, y_test)
#通过下面这个plot画出线条(其实是离散的点较密形成的) 12 thisaxis.plot(X_predict_input, y_predict_output) 13 thisaxis.plot(X_train, y_train, 'o', alpha=0.9, label='Train') 14 thisaxis.plot(X_test, y_test, '^', alpha=0.9, label='Test') 15 thisaxis.set_xlabel('Input feature') 16 thisaxis.set_ylabel('Target value') 17 thisaxis.set_title('KNN Regression (K={})\n\ 18 Train $R^2 = {:.3f}$, Test $R^2 = {:.3f}$' 19 .format(K, train_score, test_score)) 20 thisaxis.legend() 21 plt.tight_layout(pad=0.4, w_pad=0.5, h_pad=1.0)
KNN参数k对回归预测的影响
线性回归预测模型
1 from sklearn.linear_model import LinearRegression 2 3 X_train, X_test, y_train, y_test = train_test_split(X_R1, y_R1, 4 random_state = 0) 5 linreg = LinearRegression().fit(X_train, y_train) 6 7 #coef_：偏置参数 8 print('linear model coeff (w): {}' 9 .format(linreg.coef_)) 10 #intercept_：各个参数前面的权重 11 #(intercept_[0]*x[0]+...+intercept_[n]*x[n] = y 12 print('linear model intercept (b): {:.3f}' 13 .format(linreg.intercept_)) 14 print('R-squared score (training): {:.3f}' 15 .format(linreg.score(X_train, y_train))) 16 print('R-squared score (test): {:.3f}' 17 .format(linreg.score(X_test, y_test)))
linear model coeff (w): [ 45.71] linear model intercept (b): 148.446 R-squared score (training): 0.679 R-squared score (test): 0.492
线性回归图示
1 plt.figure(figsize=(5,4)) 2 plt.scatter(X_R1, y_R1, marker= 'o', s=50, alpha=0.8) 3 #画出拟合出来的直线 4 plt.plot(X_R1, linreg.coef_ * X_R1 + linreg.intercept_, 'r-') 5 plt.title('Least-squares linear regression') 6 plt.xlabel('Feature value (x)') 7 plt.ylabel('Target value (y)') 8 plt.show()
多元线性回归预测
1 X_train, X_test, y_train, y_test = train_test_split(X_crime, y_crime, 2 random_state = 0) 3 linreg = LinearRegression().fit(X_train, y_train) 4 5 print('Crime dataset') 6 print('linear model intercept: {}' 7 .format(linreg.intercept_)) 8 print('linear model coeff:\n{}' 9 .format(linreg.coef_)) 10 print('R-squared score (training): {:.3f}' 11 .format(linreg.score(X_train, y_train))) 12 print('R-squared score (test): {:.3f}' 13 .format(linreg.score(X_test, y_test)))
linear model intercept: 3861.708902399444 linear model coeff: [ 1.62e-03 -1.03e+02 1.61e+01 -2.94e+01 -1.92e+00 -1.47e+01 -2.41e-03 1.46e+00 -1.46e-02 -1.08e+01 4.35e+01 -6.92e+00 4.95e+00 -4.11e+00 -3.63e+00 8.98e-03 8.33e-03 4.84e-03 -5.25e+00 -1.59e+01 7.47e+00 2.31e+00 -2.48e-01 1.22e+01 -2.90e+00 -1.49e+00 4.96e+00 5.21e+00 1.82e+02 1.15e+01 1.54e+02 -3.40e+02 -1.22e+02 2.75e+00 -2.87e+01 2.39e+00 9.44e-01 3.18e+00 -1.17e+01 -5.46e-03 4.24e+01 -1.10e-03 -9.23e-01 5.13e+00 -4.69e+00 1.13e+00 -1.70e+01 -5.00e+01 5.64e+01 -2.94e+01 3.42e-01 -3.10e+01 2.89e+01 -5.46e+01 6.75e+02 8.54e+01 -3.35e+02 -3.17e+01 2.96e+01 7.07e+00 7.46e+01 2.01e-02 -3.96e-01 3.15e+01 1.00e+01 -1.60e+00 -5.63e-01 2.82e+00 -2.96e+01 1.08e+11 -1.01e-03 -1.08e+11 1.08e+11 -3.13e+08 -4.95e-01 3.13e+08 -3.13e+08 1.47e+00 -2.78e+00 1.12e+00 -3.70e+01 1.09e-01 3.07e-01 2.06e+01 9.24e-01 -6.05e-01 -1.92e+00 5.88e-01] R-squared score (training): 0.668 R-squared score (test): 0.520
岭回归
1 from sklearn.linear_model import Ridge 2 X_train, X_test, y_train, y_test = train_test_split(X_crime, y_crime, 3 random_state = 0) 4 #alpha为岭回归的正则化系数 5 linridge = Ridge(alpha=20.0).fit(X_train, y_train) 6 7 print('Crime dataset') 8 print('ridge regression linear model intercept: {}' 9 .format(linridge.intercept_)) 10 print('ridge regression linear model coeff:\n{}' 11 .format(linridge.coef_)) 12 print('R-squared score (training): {:.3f}' 13 .format(linridge.score(X_train, y_train))) 14 print('R-squared score (test): {:.3f}' 15 .format(linridge.score(X_test, y_test))) 16 print('Number of non-zero features: {}' 17 .format(np.sum(linridge.coef_ != 0)))
Crime dataset ridge regression linear model intercept: -3352.4230358464793 ridge regression linear model coeff: [ 1.95e-03 2.19e+01 9.56e+00 -3.59e+01 6.36e+00 -1.97e+01 -2.81e-03 1.66e+00 -6.61e-03 -6.95e+00 1.72e+01 -5.63e+00 8.84e+00 6.79e-01 -7.34e+00 6.70e-03 9.79e-04 5.01e-03 -4.90e+00 -1.79e+01 9.18e+00 -1.24e+00 1.22e+00 1.03e+01 -3.78e+00 -3.73e+00 4.75e+00 8.43e+00 3.09e+01 1.19e+01 -2.05e+00 -3.82e+01 1.85e+01 1.53e+00 -2.20e+01 2.46e+00 3.29e-01 4.02e+00 -1.13e+01 -4.70e-03 4.27e+01 -1.23e-03 1.41e+00 9.35e-01 -3.00e+00 1.12e+00 -1.82e+01 -1.55e+01 2.42e+01 -1.32e+01 -4.20e-01 -3.60e+01 1.30e+01 -2.81e+01 4.39e+01 3.87e+01 -6.46e+01 -1.64e+01 2.90e+01 4.15e+00 5.34e+01 1.99e-02 -5.47e-01 1.24e+01 1.04e+01 -1.57e+00 3.16e+00 8.78e+00 -2.95e+01 -2.34e-04 3.14e-04 -4.13e-04 -1.80e-04 -5.74e-01 -5.18e-01 -4.21e-01 1.53e-01 1.33e+00 3.85e+00 3.03e+00 -3.78e+01 1.38e-01 3.08e-01 1.57e+01 3.31e-01 3.36e+00 1.61e-01 -2.68e+00] R-squared score (training): 0.671 R-squared score (test): 0.494 Number of non-zero features: 88
岭回归使用归一化变量
1 from sklearn.preprocessing import MinMaxScaler 2 scaler = MinMaxScaler() 3 4 from sklearn.linear_model import Ridge 5 X_train, X_test, y_train, y_test = train_test_split(X_crime, y_crime, 6 random_state = 0) 7 8 #数据归一化 9 X_train_scaled = scaler.fit_transform(X_train) 10 X_test_scaled = scaler.transform(X_test) 11 12 linridge = Ridge(alpha=20.0).fit(X_train_scaled, y_train) 13 14 print('Crime dataset') 15 print('ridge regression linear model intercept: {}' 16 .format(linridge.intercept_)) 17 print('ridge regression linear model coeff:\n{}' 18 .format(linridge.coef_)) 19 print('R-squared score (training): {:.3f}' 20 .format(linridge.score(X_train_scaled, y_train))) 21 print('R-squared score (test): {:.3f}' 22 .format(linridge.score(X_test_scaled, y_test))) 23 print('Number of non-zero features: {}' 24 .format(np.sum(linridge.coef_ != 0)))
Crime dataset
ridge regression linear model intercept: 933.3906385044113
ridge regression linear model coeff:
[ 88.69 16.49 -50.3 -82.91 -65.9 -2.28 87.74 150.95 18.88
-31.06 -43.14 -189.44 -4.53 107.98 -76.53 2.86 34.95 90.14
52.46 -62.11 115.02 2.67 6.94 -5.67 -101.55 -36.91 -8.71
29.12 171.26 99.37 75.07 123.64 95.24 -330.61 -442.3 -284.5
-258.37 17.66 -101.71 110.65 523.14 24.82 4.87 -30.47 -3.52
50.58 10.85 18.28 44.11 58.34 67.09 -57.94 116.14 53.81
49.02 -7.62 55.14 -52.09 123.39 77.13 45.5 184.91 -91.36
1.08 234.09 10.39 94.72 167.92 -25.14 -1.18 14.6 36.77
53.2 -78.86 -5.9 26.05 115.15 68.74 68.29 16.53 -97.91
205.2 75.97 61.38 -79.83 67.27 95.67 -11.88]
R-squared score (training): 0.615
R-squared score (test): 0.599
Number of non-zero features: 88
正则化参数的岭回归的影响
1 print('Ridge regression: effect of alpha regularization parameter\n') 2 #改变alpha(正则化参数) 3 for this_alpha in [0, 1, 10, 20, 50, 100, 1000]: 4 linridge = Ridge(alpha = this_alpha).fit(X_train_scaled, y_train) 5 r2_train = linridge.score(X_train_scaled, y_train) 6 r2_test = linridge.score(X_test_scaled, y_test) 7 num_coeff_bigger = np.sum(abs(linridge.coef_) > 1.0) 8 print('Alpha = {:.2f}\nnum abs(coeff) > 1.0: {}, \ 9 r-squared training: {:.2f}, r-squared test: {:.2f}\n' 10 .format(this_alpha, num_coeff_bigger, r2_train, r2_test))
Ridge regression: effect of alpha regularization parameter Alpha = 0.00 num abs(coeff) > 1.0: 87, r-squared training: 0.67, r-squared test: 0.50 Alpha = 1.00 num abs(coeff) > 1.0: 87, r-squared training: 0.66, r-squared test: 0.56 Alpha = 10.00 num abs(coeff) > 1.0: 87, r-squared training: 0.63, r-squared test: 0.59 Alpha = 20.00 num abs(coeff) > 1.0: 88, r-squared training: 0.61, r-squared test: 0.60 Alpha = 50.00 num abs(coeff) > 1.0: 86, r-squared training: 0.58, r-squared test: 0.58 Alpha = 100.00 num abs(coeff) > 1.0: 87, r-squared training: 0.55, r-squared test: 0.55 Alpha = 1000.00 num abs(coeff) > 1.0: 84, r-squared training: 0.31, r-squared test: 0.30
Lasso 回归
1 from sklearn.linear_model import Lasso 2 from sklearn.preprocessing import MinMaxScaler 3 scaler = MinMaxScaler() 4 5 X_train, X_test, y_train, y_test = train_test_split(X_crime, y_crime, 6 random_state = 0) 7 8 X_train_scaled = scaler.fit_transform(X_train) 9 X_test_scaled = scaler.transform(X_test) 10 11 linlasso = Lasso(alpha=2.0, max_iter = 10000).fit(X_train_scaled, y_train) 12 13 print('Crime dataset') 14 print('lasso regression linear model intercept: {}' 15 .format(linlasso.intercept_)) 16 print('lasso regression linear model coeff:\n{}' 17 .format(linlasso.coef_)) 18 print('Non-zero features: {}' 19 .format(np.sum(linlasso.coef_ != 0))) 20 print('R-squared score (training): {:.3f}' 21 .format(linlasso.score(X_train_scaled, y_train))) 22 print('R-squared score (test): {:.3f}\n' 23 .format(linlasso.score(X_test_scaled, y_test))) 24 print('Features with non-zero weight (sorted by absolute magnitude):') 25 26 for e in sorted (list(zip(list(X_crime), linlasso.coef_)), 27 key = lambda e: -abs(e[1])): 28 if e[1] != 0: 29 print('\t{}, {:.3f}'.format(e[0], e[1]))
Crime dataset
lasso regression linear model intercept: 1186.6120619985809
lasso regression linear model coeff:
[ 0. 0. -0. -168.18 -0. -0. 0. 119.69
0. -0. 0. -169.68 -0. 0. -0. 0.
0. 0. -0. -0. 0. -0. 0. 0.
-57.53 -0. -0. 0. 259.33 -0. 0. 0.
0. -0. -1188.74 -0. -0. -0. -231.42 0.
1488.37 0. -0. -0. -0. 0. 0. 0.
0. 0. -0. 0. 20.14 0. 0. 0.
0. 0. 339.04 0. 0. 459.54 -0. 0.
122.69 -0. 91.41 0. -0. 0. 0. 73.14
0. -0. 0. 0. 86.36 0. 0. 0.
-104.57 264.93 0. 23.45 -49.39 0. 5.2 0. ]
Non-zero features: 20
R-squared score (training): 0.631
R-squared score (test): 0.624
Features with non-zero weight (sorted by absolute magnitude):
PctKidsBornNeverMar, 1488.365
PctKids2Par, -1188.740
HousVacant, 459.538
PctPersDenseHous, 339.045
NumInShelters, 264.932
MalePctDivorce, 259.329
PctWorkMom, -231.423
pctWInvInc, -169.676
agePct12t29, -168.183
PctVacantBoarded, 122.692
pctUrban, 119.694
MedOwnCostPctIncNoMtg, -104.571
MedYrHousBuilt, 91.412
RentQrange, 86.356
OwnOccHiQuart, 73.144
PctEmplManu, -57.530
PctBornSameState, -49.394
PctForeignBorn, 23.449
PctLargHouseFam, 20.144
PctSameCity85, 5.198
k(正则化系数）对Lasso回归的影响
1 print('Lasso regression: effect of alpha regularization\n\ 2 parameter on number of features kept in final model\n') 3 4 for alpha in [0.5, 1, 2, 3, 5, 10, 20, 50]: 5 linlasso = Lasso(alpha, max_iter = 10000).fit(X_train_scaled, y_train) 6 r2_train = linlasso.score(X_train_scaled, y_train) 7 r2_test = linlasso.score(X_test_scaled, y_test) 8 9 print('Alpha = {:.2f}\nFeatures kept: {}, r-squared training: {:.2f}, \ 10 r-squared test: {:.2f}\n' 11 .format(alpha, np.sum(linlasso.coef_ != 0), r2_train, r2_test))
Lasso regression: effect of alpha regularization parameter on number of features kept in final model Alpha = 0.50 Features kept: 35, r-squared training: 0.65, r-squared test: 0.58 Alpha = 1.00 Features kept: 25, r-squared training: 0.64, r-squared test: 0.60 Alpha = 2.00 Features kept: 20, r-squared training: 0.63, r-squared test: 0.62 Alpha = 3.00 Features kept: 17, r-squared training: 0.62, r-squared test: 0.63 Alpha = 5.00 Features kept: 12, r-squared training: 0.60, r-squared test: 0.61 Alpha = 10.00 Features kept: 6, r-squared training: 0.57, r-squared test: 0.58 Alpha = 20.00 Features kept: 2, r-squared training: 0.51, r-squared test: 0.50 Alpha = 50.00 Features kept: 1, r-squared training: 0.31, r-squared test: 0.30
多元回归
1 from sklearn.linear_model import LinearRegression 2 from sklearn.linear_model import Ridge 3 from sklearn.preprocessing import PolynomialFeatures 4 5 6 X_train, X_test, y_train, y_test = train_test_split(X_F1, y_F1, 7 random_state = 0) 8 linreg = LinearRegression().fit(X_train, y_train) 9 10 print('linear model coeff (w): {}' 11 .format(linreg.coef_)) 12 print('linear model intercept (b): {:.3f}' 13 .format(linreg.intercept_)) 14 print('R-squared score (training): {:.3f}' 15 .format(linreg.score(X_train, y_train))) 16 print('R-squared score (test): {:.3f}' 17 .format(linreg.score(X_test, y_test))) 18 19 print('对参数开方，使参数变成平方的表示形式') 20 poly = PolynomialFeatures(degree=2) 21 X_F1_poly = poly.fit_transform(X_F1) 22 23 X_train, X_test, y_train, y_test = train_test_split(X_F1_poly, y_F1, 24 random_state = 0) 25 linreg = LinearRegression().fit(X_train, y_train) 26 27 print('(poly deg 2) linear model coeff (w):\n{}' 28 .format(linreg.coef_)) 29 print('(poly deg 2) linear model intercept (b): {:.3f}' 30 .format(linreg.intercept_)) 31 print('(poly deg 2) R-squared score (training): {:.3f}' 32 .format(linreg.score(X_train, y_train))) 33 print('(poly deg 2) R-squared score (test): {:.3f}\n' 34 .format(linreg.score(X_test, y_test))) 35 36 print('使用平方的参数，会很容易导致过拟合，通过对平方参数加上\n 37 正则化，减少过拟合情况，类似岭回归') 38 39 X_train, X_test, y_train, y_test = train_test_split(X_F1_poly, y_F1, 40 random_state = 0) 41 linreg = Ridge().fit(X_train, y_train) 42 43 print('(poly deg 2 + ridge) linear model coeff (w):\n{}' 44 .format(linreg.coef_)) 45 print('(poly deg 2 + ridge) linear model intercept (b): {:.3f}' 46 .format(linreg.intercept_)) 47 print('(poly deg 2 + ridge) R-squared score (training): {:.3f}' 48 .format(linreg.score(X_train, y_train))) 49 print('(poly deg 2 + ridge) R-squared score (test): {:.3f}' 50 .format(linreg.score(X_test, y_test)))
linear model coeff (w): [ 4.42 6. 0.53 10.24 6.55 -2.02 -0.32] linear model intercept (b): 1.543 R-squared score (training): 0.722 R-squared score (test): 0.722 对参数开方，使参数变成平方的表示形式 (poly deg 2) linear model coeff (w): [ 3.41e-12 1.66e+01 2.67e+01 -2.21e+01 1.24e+01 6.93e+00 1.05e+00 3.71e+00 -1.34e+01 -5.73e+00 1.62e+00 3.66e+00 5.05e+00 -1.46e+00 1.95e+00 -1.51e+01 4.87e+00 -2.97e+00 -7.78e+00 5.15e+00 -4.65e+00 1.84e+01 -2.22e+00 2.17e+00 -1.28e+00 1.88e+00 1.53e-01 5.62e-01 -8.92e-01 -2.18e+00 1.38e+00 -4.90e+00 -2.24e+00 1.38e+00 -5.52e-01 -1.09e+00] (poly deg 2) linear model intercept (b): -3.206 (poly deg 2) R-squared score (training): 0.969 (poly deg 2) R-squared score (test): 0.805 使用平方的参数，会很容易导致过拟合，通过对平方参数加上 正则化，减少过拟合情况，类似岭回归 (poly deg 2 + ridge) linear model coeff (w): [ 0. 2.23 4.73 -3.15 3.86 1.61 -0.77 -0.15 -1.75 1.6 1.37 2.52 2.72 0.49 -1.94 -1.63 1.51 0.89 0.26 2.05 -1.93 3.62 -0.72 0.63 -3.16 1.29 3.55 1.73 0.94 -0.51 1.7 -1.98 1.81 -0.22 2.88 -0.89] (poly deg 2 + ridge) linear model intercept (b): 5.418 (poly deg 2 + ridge) R-squared score (training): 0.826 (poly deg 2 + ridge) R-squared score (test): 0.825
线性模型用于分类
逻辑回归
使用高度，跨度作为参数进行水果类型分类
1 from sklearn.linear_model import LogisticRegression 2 from adspy_shared_utilities import ( 3 plot_class_regions_for_classifier_subplot) 4 5 fig, subaxes = plt.subplots(1, 1, figsize=(7, 5)) 6 y_fruits_apple = y_fruits_2d == 1 # make into a binary problem: apples vs everything else #as_matrix()把表格转化成矩阵，方便计算
7 X_train, X_test, y_train, y_test = ( 8 train_test_split(X_fruits_2d.as_matrix(), 9 y_fruits_apple.as_matrix(), 10 random_state = 0)) 11 12 clf = LogisticRegression(C=100).fit(X_train, y_train) 13 plot_class_regions_for_classifier_subplot(clf, X_train, y_train, None, 14 None, 'Logistic regression \ 15 for binary classification\nFruit dataset: Apple vs others', 16 subaxes) 17 18 h = 6 19 w = 8 20 print('A fruit with height {} and width {} is predicted to be: {}' 21 .format(h,w, ['not an apple', 'an apple'][clf.predict([[h,w]])[0]])) 22 23 h = 10 24 w = 7 25 print('A fruit with height {} and width {} is predicted to be: {}' 26 .format(h,w, ['not an apple', 'an apple'][clf.predict([[h,w]])[0]])) 27 subaxes.set_xlabel('height') 28 subaxes.set_ylabel('width') 29 30 print('Accuracy of Logistic regression classifier on training set: {:.2f}' 31 .format(clf.score(X_train, y_train))) 32 print('Accuracy of Logistic regression classifier on test set: {:.2f}' 33 .format(clf.score(X_test, y_test)))
A fruit with height 6 and width 8 is predicted to be: an apple A fruit with height 10 and width 7 is predicted to be: not an apple Accuracy of Logistic regression classifier on training set: 0.77 Accuracy of Logistic regression classifier on test set: 0.73
1 from sklearn.linear_model import LogisticRegression 2 from adspy_shared_utilities import ( 3 plot_class_regions_for_classifier_subplot) 4 5 6 X_train, X_test, y_train, y_test = train_test_split(X_C2, y_C2, 7 random_state = 0) 8 9 fig, subaxes = plt.subplots(1, 1, figsize=(7, 5)) 10 clf = LogisticRegression().fit(X_train, y_train) 11 title = 'Logistic regression, simple synthetic dataset C = {:.3f}'.format(1.0) 12 plot_class_regions_for_classifier_subplot(clf, X_train, y_train, 13 None, None, title, subaxes) 14 15 print('Accuracy of Logistic regression classifier on training set: {:.2f}' 16 .format(clf.score(X_train, y_train))) 17 print('Accuracy of Logistic regression classifier on test set: {:.2f}' 18 .format(clf.score(X_test, y_test))) 19
Accuracy of Logistic regression classifier on training set: 0.80 Accuracy of Logistic regression classifier on test set: 0.80
逻辑回归正则化参数的影响
1 X_train, X_test, y_train, y_test = ( 2 train_test_split(X_fruits_2d.as_matrix(), 3 y_fruits_apple.as_matrix(), 4 random_state=0)) 5 6 fig, subaxes = plt.subplots(3, 1, figsize=(4, 10)) 7 8 for this_C, subplot in zip([0.1, 1, 100], subaxes): 9 clf = LogisticRegression(C=this_C).fit(X_train, y_train) 10 title ='Logistic regression (apple vs rest), C = {:.3f}'.format(this_C) 11 12 plot_class_regions_for_classifier_subplot(clf, X_train, y_train, 13 X_test, y_test, title, 14 subplot) 15 plt.tight_layout()
应用于真实数据
1 from sklearn.linear_model import LogisticRegression 2 3 X_train, X_test, y_train, y_test = train_test_split(X_cancer, y_cancer, random_state = 0) 4 5 clf = LogisticRegression().fit(X_train, y_train) 6 print('Breast cancer dataset') 7 print('Accuracy of Logistic regression classifier on training set: {:.2f}' 8 .format(clf.score(X_train, y_train))) 9 print('Accuracy of Logistic regression classifier on test set: {:.2f}' 10 .format(clf.score(X_test, y_test)))
Breast cancer dataset Accuracy of Logistic regression classifier on training set: 0.96 Accuracy of Logistic regression classifier on test set: 0.96
SVM
线性SVM
1 from sklearn.svm import SVC 2 from adspy_shared_utilities import plot_class_regions_for_classifier_subplot 3 4 5 X_train, X_test, y_train, y_test = train_test_split(X_C2, y_C2, random_state = 0) 6 7 fig, subaxes = plt.subplots(1, 1, figsize=(7, 5)) 8 this_C = 1.0 9 #线性核函数 10 clf = SVC(kernel = 'linear', C=this_C).fit(X_train, y_train) 11 title = 'Linear SVC, C = {:.3f}'.format(this_C) 12 plot_class_regions_for_classifier_subplot(clf, X_train, y_train, None, None, title, subaxes)
Linear Support Vector Machine: C parameter
1 from sklearn.svm import LinearSVC 2 from adspy_shared_utilities import plot_class_regions_for_classifier 3 4 X_train, X_test, y_train, y_test = train_test_split(X_C2, y_C2, random_state = 0) 5 fig, subaxes = plt.subplots(1, 2, figsize=(8, 4)) 6 7 for this_C, subplot in zip([0.00001, 100], subaxes): 8 clf = LinearSVC(C=this_C).fit(X_train, y_train) 9 title = 'Linear SVC, C = {:.5f}'.format(this_C) 10 plot_class_regions_for_classifier_subplot(clf, X_train, y_train, 11 None, None, title, subplot) 12 plt.tight_layout()
线性SVM使用于真实数据
1 from sklearn.svm import LinearSVC 2 X_train, X_test, y_train, y_test = train_test_split(X_cancer, y_cancer, random_state = 0) 3 4 clf = LinearSVC().fit(X_train, y_train) 5 print('Breast cancer dataset') 6 print('Accuracy of Linear SVC classifier on training set: {:.2f}' 7 .format(clf.score(X_train, y_train))) 8 print('Accuracy of Linear SVC classifier on test set: {:.2f}' 9 .format(clf.score(X_test, y_test)))
Breast cancer dataset Accuracy of Linear SVC classifier on training set: 0.74 Accuracy of Linear SVC classifier on test set: 0.74
使用线性模型进行多分类任务
1 from sklearn.svm import LinearSVC 2 3 X_train, X_test, y_train, y_test = train_test_split(X_fruits_2d, y_fruits_2d, random_state = 0) 4 5 clf = LinearSVC(C=5, random_state = 67).fit(X_train, y_train) 6 print('Coefficients:\n', clf.coef_) 7 print('Intercepts:\n', clf.intercept_)
Coefficients: [[-0.26 0.71] [-1.63 1.16] [ 0.03 0.29] [ 1.24 -1.64]] Intercepts: [-3.29 1.2 -2.72 1.16]
在水果数据集上使用多分类
1 plt.figure(figsize=(6,6)) 2 colors = ['r', 'g', 'b', 'y'] 3 cmap_fruits = ListedColormap(['#FF0000', '#00FF00', '#0000FF','#FFFF00']) 4 5 plt.scatter(X_fruits_2d[['height']], X_fruits_2d[['width']], 6 c=y_fruits_2d, cmap=cmap_fruits, edgecolor = 'black', alpha=.7) 7 8 x_0_range = np.linspace(-10, 15) 9 10 for w, b, color in zip(clf.coef_, clf.intercept_, ['r', 'g', 'b', 'y']): 11 # Since class prediction with a linear model uses the formula y = w_0 x_0 + w_1 x_1 + b, 12 # and the decision boundary is defined as being all points with y = 0, to plot x_1 as a 13 # function of x_0 we just solve w_0 x_0 + w_1 x_1 + b = 0 for x_1: 14 plt.plot(x_0_range, -(x_0_range * w[0] + b) / w[1], c=color, alpha=.8) 15 16 plt.legend(target_names_fruits) 17 plt.xlabel('height') 18 plt.ylabel('width') 19 plt.xlim(-2, 12) 20 plt.ylim(-2, 15) 21 plt.show()
核化SVM
分类模型
1 from sklearn.svm import SVC 2 from adspy_shared_utilities import plot_class_regions_for_classifier 3 4 X_train, X_test, y_train, y_test = train_test_split(X_D2, y_D2, random_state = 0) 5 6 # RBF核函数 7 plot_class_regions_for_classifier(SVC().fit(X_train, y_train), 8 X_train, y_train, None, None, 9 'Support Vector Classifier: RBF kernel') 10 11 # 多项式核函数polynomial kernel, degree = 3 12 plot_class_regions_for_classifier(SVC(kernel = 'poly', degree = 3) 13 .fit(X_train, y_train), X_train, 14 y_train, None, None, 15 'Support Vector Classifier: Polynomial kernel, degree = 3')
γ参数对RBF核函数SVM的影响
1 from adspy_shared_utilities import plot_class_regions_for_classifier 2 3 X_train, X_test, y_train, y_test = train_test_split(X_D2, y_D2, random_state = 0) 4 fig, subaxes = plt.subplots(3, 1, figsize=(4, 11)) 5 6 for this_gamma, subplot in zip([0.01, 1.0, 10.0], subaxes): 7 clf = SVC(kernel = 'rbf', gamma=this_gamma).fit(X_train, y_train) 8 title = 'Support Vector Classifier: \nRBF kernel, gamma = {:.2f}'.format(this_gamma) 9 plot_class_regions_for_classifier_subplot(clf, X_train, y_train, 10 None, None, title, subplot) 11 plt.tight_layout()
γ和C对RBP核函数SVM的影响
1 from sklearn.svm import SVC 2 from adspy_shared_utilities import plot_class_regions_for_classifier_subplot 3 4 from sklearn.model_selection import train_test_split 5 6 7 X_train, X_test, y_train, y_test = train_test_split(X_D2, y_D2, random_state = 0) 8 fig, subaxes = plt.subplots(3, 4, figsize=(15, 10), dpi=50) 9 10 for this_gamma, this_axis in zip([0.01, 1, 5], subaxes): 11 12 for this_C, subplot in zip([0.1, 1, 15, 250], this_axis): 13 title = 'gamma = {:.2f}, C = {:.2f}'.format(this_gamma, this_C) 14 clf = SVC(kernel = 'rbf', gamma = this_gamma, 15 C = this_C).fit(X_train, y_train) 16 plot_class_regions_for_classifier_subplot(clf, X_train, y_train, 17 X_test, y_test, title, 18 subplot) 19 plt.tight_layout(pad=0.4, w_pad=0.5, h_pad=1.0)
非标准化数据应用于SVM
1 from sklearn.svm import SVC 2 X_train, X_test, y_train, y_test = train_test_split(X_cancer, y_cancer, 3 random_state = 0) 4 5 clf = SVC(C=10).fit(X_train, y_train) 6 print('Breast cancer dataset (unnormalized features)') 7 print('Accuracy of RBF-kernel SVC on training set: {:.2f}' 8 .format(clf.score(X_train, y_train))) 9 print('Accuracy of RBF-kernel SVC on test set: {:.2f}' 10 .format(clf.score(X_test, y_test)))
Breast cancer dataset (unnormalized features) Accuracy of RBF-kernel SVC on training set: 1.00 Accuracy of RBF-kernel SVC on test set: 0.63
SVM应用于标准化数据
1 from sklearn.preprocessing import MinMaxScaler 2 scaler = MinMaxScaler() 3 X_train_scaled = scaler.fit_transform(X_train) 4 X_test_scaled = scaler.transform(X_test) 5 6 clf = SVC(C=10).fit(X_train_scaled, y_train) 7 print('Breast cancer dataset (normalized with MinMax scaling)') 8 print('RBF-kernel SVC (with MinMax scaling) training set accuracy: {:.2f}' 9 .format(clf.score(X_train_scaled, y_train))) 10 print('RBF-kernel SVC (with MinMax scaling) test set accuracy: {:.2f}' 11 .format(clf.score(X_test_scaled, y_test)))
Breast cancer dataset (normalized with MinMax scaling) RBF-kernel SVC (with MinMax scaling) training set accuracy: 0.98 RBF-kernel SVC (with MinMax scaling) test set accuracy: 0.96
交叉验证
1 from sklearn.model_selection import cross_val_score 2 3 clf = KNeighborsClassifier(n_neighbors = 5) 4 X = X_fruits_2d.as_matrix() 5 y = y_fruits_2d.as_matrix() 6 #进行交叉验证 7 cv_scores = cross_val_score(clf, X, y) 8 9 print('Cross-validation scores (3-fold):', cv_scores) 10 print('Mean cross-validation score (3-fold): {:.3f}' 11 .format(np.mean(cv_scores)))
Cross-validation scores (3-fold): [ 0.77 0.74 0.83] Mean cross-validation score (3-fold): 0.781
验证曲线
1 from sklearn.svm import SVC 2 from sklearn.model_selection import validation_curve 3 4 param_range = np.logspace(-3, 3, 4) 5 train_scores, test_scores = validation_curve(SVC(), X, y, 6 param_name='gamma', 7 param_range=param_range, cv=3)
1 print(train_scores)
[[ 0.49 0.42 0.41] [ 0.84 0.72 0.76] [ 0.92 0.9 0.93] [ 1. 1. 0.98]]
1 print(test_scores)
[[ 0.45 0.32 0.33] [ 0.82 0.68 0.61] [ 0.41 0.84 0.67] [ 0.36 0.21 0.39]]
1 # This code based on scikit-learn validation_plot example 2 # See: http://scikit-learn.org/stable/auto_examples/model_selection/plot_validation_curve.html 3 plt.figure() 4 5 train_scores_mean = np.mean(train_scores, axis=1) 6 train_scores_std = np.std(train_scores, axis=1) 7 test_scores_mean = np.mean(test_scores, axis=1) 8 test_scores_std = np.std(test_scores, axis=1) 9 10 plt.title('Validation Curve with SVM') 11 plt.xlabel('$\gamma$ (gamma)') 12 plt.ylabel('Score') 13 plt.ylim(0.0, 1.1) 14 lw = 2 15 16 plt.semilogx(param_range, train_scores_mean, label='Training score', 17 color='darkorange', lw=lw) 18 19 plt.fill_between(param_range, train_scores_mean - train_scores_std, 20 train_scores_mean + train_scores_std, alpha=0.2, 21 color='darkorange', lw=lw) 22 23 plt.semilogx(param_range, test_scores_mean, label='Cross-validation score', 24 color='navy', lw=lw) 25 26 plt.fill_between(param_range, test_scores_mean - test_scores_std, 27 test_scores_mean + test_scores_std, alpha=0.2, 28 color='navy', lw=lw) 29 30 plt.legend(loc='best') 31 plt.show()
决策树
1 from sklearn.datasets import load_iris 2 from sklearn.tree import DecisionTreeClassifier 3 from adspy_shared_utilities import plot_decision_tree 4 from sklearn.model_selection import train_test_split 5 6 7 iris = load_iris() 8 9 X_train, X_test, y_train, y_test = train_test_split(iris.data, iris.target, random_state = 3) 10 clf = DecisionTreeClassifier().fit(X_train, y_train) 11 12 print('Accuracy of Decision Tree classifier on training set: {:.2f}' 13 .format(clf.score(X_train, y_train))) 14 print('Accuracy of Decision Tree classifier on test set: {:.2f}' 15 .format(clf.score(X_test, y_test)))
设置树的深度避免过拟合
#max_depth设置决策树的最大深度
1 clf2 = DecisionTreeClassifier(max_depth = 3).fit(X_train, y_train) 2 3 print('Accuracy of Decision Tree classifier on training set: {:.2f}' 4 .format(clf2.score(X_train, y_train))) 5 print('Accuracy of Decision Tree classifier on test set: {:.2f}' 6 .format(clf2.score(X_test, y_test)))
Accuracy of Decision Tree classifier on training set: 0.98 Accuracy of Decision Tree classifier on test set: 0.97
可视化决策树
1 plot_decision_tree(clf, iris.feature_names, iris.target_names)
1 #决策树的最大深度为3 2 plot_decision_tree(clf2, iris.feature_names, iris.target_names)
变量的重要性
1 from adspy_shared_utilities import plot_feature_importances 2 3 plt.figure(figsize=(10,4), dpi=80) 4 plot_feature_importances(clf, iris.feature_names) 5 plt.show() 6 7 print('Feature importances: {}'.format(clf.feature_importances_))
Feature importances: [ 0. 0.02 0.56 0.42]
1 from sklearn.tree import DecisionTreeClassifier 2 from adspy_shared_utilities import plot_class_regions_for_classifier_subplot 3 4 X_train, X_test, y_train, y_test = train_test_split(iris.data, iris.target, random_state = 0) 5 fig, subaxes = plt.subplots(6, 1, figsize=(6, 32)) 6 7 pair_list = [[0,1], [0,2], [0,3], [1,2], [1,3], [2,3]] 8 tree_max_depth = 4 9 10 for pair, axis in zip(pair_list, subaxes): 11 X = X_train[:, pair] 12 y = y_train 13 14 clf = DecisionTreeClassifier(max_depth=tree_max_depth).fit(X, y) 15 title = 'Decision Tree, max_depth = {:d}'.format(tree_max_depth) 16 plot_class_regions_for_classifier_subplot(clf, X, y, None, 17 None, title, axis, 18 iris.target_names) 19 20 axis.set_xlabel(iris.feature_names[pair[0]]) 21 axis.set_ylabel(iris.feature_names[pair[1]]) 22 23 plt.tight_layout() 24 plt.show()
决策树应用于真实数据
1 from sklearn.tree import DecisionTreeClassifier 2 from adspy_shared_utilities import plot_decision_tree 3 from adspy_shared_utilities import plot_feature_importances 4 5 X_train, X_test, y_train, y_test = train_test_split(X_cancer, y_cancer, random_state = 0) 6 7 clf = DecisionTreeClassifier(max_depth = 4, min_samples_leaf = 8, 8 random_state = 0).fit(X_train, y_train) 9 10 plot_decision_tree(clf, cancer.feature_names, cancer.target_names)
1 print('Breast cancer dataset: decision tree') 2 print('Accuracy of DT classifier on training set: {:.2f}' 3 .format(clf.score(X_train, y_train))) 4 print('Accuracy of DT classifier on test set: {:.2f}' 5 .format(clf.score(X_test, y_test))) 6 7 plt.figure(figsize=(10,6),dpi=80) 8 plot_feature_importances(clf, cancer.feature_names) 9 plt.tight_layout() 10 11 plt.show()
Breast cancer dataset: decision tree Accuracy of DT classifier on training set: 0.96 Accuracy of DT classifier on test set: 0.94
