这是半成品, 已完成了 fit() 部分, 形成了包含一棵完整树的 node 对象.
后续工作是需解析该 node 对象, 完成 predict() 工作.
# !/usr/bin/python
# -*- coding:utf-8 -*-
"""
Re-implement ID3 algorithm as a practice
Only information gain criterion supplied in our DT algorithm.
使用该 ID3 re-implement 的前提：
1. train data 的标签必须转成0,1,2,...的形式
2. 只能处理连续特征
"""
# Author: 相忠良(Zhong-Liang Xiang) <ugoood@163.com>
# Finished at July ***, 2017
import numpy as np
from sklearn import datasets, cross_validation
## load data
def load_data():
iris = datasets.load_iris()
return cross_validation.train_test_split(iris.data, iris.target, test_size=0.25, random_state=0)
class DecisionNode():
def __init__(self, feature_i=None, threshold=None, value=None, left_branch=None, right_branch=None):
self.feature_i = feature_i # Best feature's index
self.threshold = threshold # Best split threshold in the feature
self.value = value # Value if the node is a leaf in the tree
self.left_branch = left_branch # 'Left' subtree
self.right_branch = right_branch # 'Right' subtree
# print feature_i, 'feature_i'
print self.value, 'value'
class MyDecisionTreeClassifier():
trees = []
num_eles_in_class_label = 3 # 分类标签类的个数
tree = {}
predict_label = []
X_train = []
y_train = []
max_depth = 3
max_leaf_nodes = 30
min_samples_leaf = 1
count = 0
def __init__(self, ):
self.root = None
# TODO
def fit(self, X, y):
self.root = DecisionNode(self.createTree(X, y))
def predict(self, X):
pass
def score(self, X, y):
pass
## entropy
# e.g entropy(y_test)
def __entropy(self, label_list):
bincount = np.bincount(label_list, minlength=self.num_eles_in_class_label)
sum = np.sum(bincount)
# print 'sum in entropy ', sum
temp = 1.0 * bincount / sum
tot = 0
# to avoid log2(0)
for e in temp:
if (e != 0):
tot += e * (-np.log2(e))
return tot
def gain(self, pre_split_label_list, after_split_label_list_2d):
total = 0
n = after_split_label_list_2d[0].__len__() + after_split_label_list_2d[1].__len__()
for item in after_split_label_list_2d:
total += self.__entropy(item) * (1.0 * item.__len__() / n)
return self.__entropy(pre_split_label_list) - total
## 针对np.bincount()的结果，如[37 34 41]，判断是否为纯节点，既[0 22 0]的形式
def isPure(self, bincount_list):
sb = sorted(bincount_list)
if ((sb[-1] != 0) & (sb[-2] == 0)):
return True
else:
return False
## 计算出现次数最多的类别标签
def maxCate(self, bincount_list):
bincount_list = np.array(bincount_list)
return bincount_list.argmax()
## 递归停止条件：
# 如果样例小于等于１０，停止
# 如果样例大于１０ 且 点纯，停止
# 否则 继续分裂
def createTree(self, X, y):
bincount_list = np.bincount(y, minlength=self.num_eles_in_class_label)
if ((self.isPure(bincount_list)) & (np.sum(bincount_list) > 10)):
print bincount_list, '11111'
return DecisionNode(value=self.maxCate(bincount_list))
elif (np.sum(bincount_list) <= 10):
print bincount_list, '22222'
return DecisionNode(value=self.maxCate(bincount_list))
else:
print bincount_list, '33333'
f, v, g = self.seek_best_split_feature(X, y)
mask_big = X[:, f] > v
mask_sma = X[:, f] <= v
bigger_X = []
bigger_y = []
smaller_X = []
smaller_y = []
bigger_X.append(X[mask_big])
bigger_y.append(y[mask_big])
smaller_X.append(X[mask_sma])
smaller_y.append(y[mask_sma])
left_branch = self.createTree(bigger_X[0], bigger_y[0])
right_branch = self.createTree(smaller_X[0], smaller_y[0])
return DecisionNode(feature_i=f, threshold=v, left_branch=left_branch, right_branch=right_branch)
## k>=2 特征区间切分点个数
# samples 样本
# labels 样本对应的标签
# return: best_feature, best_split_point, gain_on_that_point
def seek_best_split_feature(self, samples, labels, k=10): # 2 2.84 0.915290847812
samples = np.array(samples)
labels = np.array(labels)
best_split_point_pool = {} # 最佳分裂特征，点，及对应的gain
col_indx = 0
# 遍历所有特征，寻找某特征最佳分裂点
while col_indx < samples.shape[1]:
max = np.max(samples[:, col_indx])
min = np.min(samples[:, col_indx])
split_point = np.linspace(min, max, k, False)[1:]
# 寻找某特征最佳分裂点
temp = []
dic = {}
for p in split_point:
index_less = np.where(samples[:, col_indx] < p)[0] # [1 2]
index_bigger = np.where(samples[:, col_indx] >= p)[0]
label_less = labels[index_less]
label_bigger = labels[index_bigger]
temp.append(list(label_less))
temp.append(list(label_bigger))
g = self.gain(labels, temp)
dic[p] = g
temp = []
best_key = sorted(dic, key=lambda x: dic[x])[-1] # 返回value最大的那个key
dic_temp = {}
dic_temp[best_key] = dic[best_key]
best_split_point_pool[col_indx] = dic_temp
col_indx += 1
# 特征列表
feature_name_box = list(best_split_point_pool.keys())
b = list(best_split_point_pool.values()) # 临时表
# 最大gain列表
gain_box = []
# 最佳切分点列表
point_box = []
for item in b:
gain_box.append(item.values()[0])
point_box.append(item.keys()[0])
best_feature = feature_name_box[np.argmax(gain_box)]
best_split_point = point_box[np.argmax(gain_box)]
gain_on_that_point = np.max(gain_box)
return best_feature, best_split_point, gain_on_that_point
## 测试用例
X_train, X_test, y_train, y_test = load_data()
cls = MyDecisionTreeClassifier()
a = [[9, 2, 3, 4],
[5, 6, 7, 8],
[1, 10, 11, 12],
[13, 14, 15, 16]]
b = [0, 1, 2, 3]
a = np.array(a)
b = np.array(b)
# xx = [2,1,1]
# print cls.maxCate(xx),'11111111111111111111111'
cls.fit(X_train, y_train)
tree = cls.root
print type(cls.root)
'''
下面是编程过程中留下的经验
'''
# 重要1: np.linspace(0,1,5) 0-1之间，等分５份，包括首尾
# np.linspace(0,1,5)
# [ 0. 0.25 0.5 0.75 1. ]
# 重要2: np.where(a[:,0]>2) 返回矩阵a中第０列值大于２的那些行的索引号
# 返回值的样子 (array([1, 2]),)
# 重要3: 返回value最大的那个key
# print(sorted(dic, key=lambda x: dic[x])[-1])
# 重要4: np.bincount()指定最小长度
# xxx = [1,1,1,1,1]
# print np.bincount(xxx,minlength=3)
# 结果: [0 5 0]
