KNN算法的实现

KNN算法是机器学习经典十大算法之一，简单易懂。这里给出KNN的实现，由两个版本：

1.机器学习实战上作者的实现版本，我自己又敲了一遍感觉还是蛮有收获的；

2.用自己的理解的一个实现，主要的区别就是效率没有第一个高，因为第一个大量使用矩阵向量的运算，速度比较快，还有就是作者的代码比较简介好看。自己的代码就是比较好懂。

 

1.《机器学习实战》的代码

# -*- coding: utf-8 -*-

'''
 function: 根据《实战》实现KNN算法,用于约会网站进行匹配
 date: 2017.8.5
'''

from numpy import *
import matplotlib.pyplot as plt
import operator

#产生数据集和标签
def createDataSet():
    group = array([[1.0,1.1], [1.0,1.0], [0,0], [0,0.1]])
    labels = ['A', 'A', 'B', 'B']
    return group, labels

#knn分类算法
def classify0(inX, dataSet, labels, k):
    dataSetSize = dataSet.shape[0] #返回m,n,shape[0] == m 行数,代表由几个训练数据

    #计算距离;将测试向量 inX 在咧方向上重复一次，行方向重复m次，与对应的训练数据相减
    diffMat = tile(inX, (dataSetSize, 1)) - dataSet 

    sqDiffMat = diffMat**2 #平方的意思，不能用^
    sqDistances = sqDiffMat.sum(axis=1) #axis=1表示按行相加，axis=0表示案列相加
    distances = sqDistances**0.5 #测试点距离所有训练点的距离向量集合

    sortedDistIndices = distances.argsort() #距离从小到大排序，返回的是排序后的下表组成的数组；
    classCount = {}

    for i in range(k):
        voteIlabel = labels[sortedDistIndices[i]]
        classCount[voteIlabel] = classCount.get(voteIlabel,0) + 1 #get函数返回value，若不存在则返回0
    
    #items()返回key,value,itemgetter(1)，按照第二个元素进行排序，从大到小
    sortedClassCount = sorted(classCount.items(), key=operator.itemgetter(1),reverse=True)

    return sortedClassCount[0][0]

#将文本记录转换为nmpy的解析程序
def file2matrix(filename):
    fr = open(filename)
    arrayOLines = fr.readlines()
    numberOfLines = len(arrayOLines)
    returnMat = zeros((numberOfLines,3)) #返回的矩阵 是： m*3,3个特征值
    classLabelVector = []
    index = 0
    for line in arrayOLines:
        line = line.strip() #去掉换行符
        listFromLine = line.split('\t') #将每一行用tab分割
        returnMat[index,:] = listFromLine[0:3] #每行前三个是特征
        classLabelVector.append(int(listFromLine[-1])) #最后一列是标签，并且告诉list是int类型
        index += 1
    return returnMat, classLabelVector

#draw the point
def drawPoint(datingDatamat, datingLabels):
    fig = plt.figure()
    ax = fig.add_subplot(111)

    #总共三个特征，可以选择其中任意两个来画图，看哪个区分效果好
    ax.scatter(datingDatamat[:,0],datingDatamat[:,1],15.0*array(datingLabels),15.0*array(datingLabels))
    #ax.scatter(datingDatamat[:,1],datingDatamat[:,2],15.0*array(datingLabels),15.0*array(datingLabels))
    plt.show()


#对数据集归一化处理 autoNormanization
def autoNorm(dataSet):
    minVals = dataSet.min(0) #get the minimal row
    maxVals = dataSet.max(0) #get the maximal row
    ranges = maxVals - minVals + ones_like(maxVals) #个人感觉这里可能由除〇的风险，加一效果可能好一点
    normDataSet = zeros(shape(dataSet)) #the same size of original dataset: m*n
    m = dataSet.shape[0]
    normDataSet = dataSet - tile(minVals, (m,1)) #repeat the minimal row m times on the row
    normDataSet = dataSet / tile(ranges, (m,1))
    return normDataSet, ranges, minVals

#分类器针对约会网站的测试代码
def datingClassTest():
    haRatio = 0.10
    datingDataMat, datingLabels = file2matrix('datingTestSet2.txt')
    normMat, ranges, minVals = autoNorm(datingDataMat)
    m = normMat.shape[0] #获得矩阵的行数，数据集的个数
    numTestVecs = int(m*haRatio) #用十分之一的数据来做测试，剩下的作为训练数据
    errorCount = 0.0
    for i in range(numTestVecs): #对每个测试用例开始计算
        #0~numtestvecs都是用来测试的，每次测试一行，从numtestvecs~m都是训练数据
        classifierResult = classify0(normMat[i,:],normMat[numTestVecs:m,:],datingLabels[numTestVecs:m],3)
        print('the classifier came back with : %d, the real answer is : %d'\
         % (classifierResult,datingLabels[i]))
        if(classifierResult != datingLabels[i]):
            errorCount += 1
    print('the total error rate is: %f' % (errorCount / float(numTestVecs)))


#让用户输入自己的一些信息，用输入的信息进行分类
def classifyPerson():
    resultList = ['not at all', 'in small doses', 'in large doses']
    percentTats = float(input('persentage of time spent playing video games?'))
    ffMiles = float(input('frequent fliers miles each year?'))
    iceCream = float(input('liters of ice cream consumed per year?'))
    datingDatamat, datingLabels = file2matrix('datingTestSet2.txt')
    normMat, ranges, minVals = autoNorm(datingDatamat)
    inArr = array([percentTats, ffMiles, iceCream])
    #对输入的一个测试数据进行归一化然后分类
    classifierResult = classify0((inArr - minVals) / ranges, normMat, datingLabels, 3)
    print('you will probably like this person: ', resultList[classifierResult - 1])
    print(classifierResult)
    
# #测试分类效果和加载数据的程序
# group, labels = createDataSet()
# result = classify0([0,0], group, labels, 3)
# print(result)

# #get numpy metrix from file
# returnMat, classLabelVector = file2matrix('datingTestSet2.txt')
# print(returnMat)
# print(classLabelVector)

# #auto normanization
# normDataSet, ranges, minVals = autoNorm(returnMat)

# #测试我们的分类器
# datingClassTest()

#请求用户输入几个特征，然后进行分类，判断喜爱程度
classifyPerson()

#draw
#drawPoint(normDataSet, classLabelVector)

 

2.自己的实现

# -*- coding: utf-8 -*-

'''
function: 使用二维数据点集(x,y)来实现knn算法核心功能
note: 计算欧式距离还可以用 numpy.linalg.norm(array1 - array2) 必须是array
date: 2017.07.23
'''
from numpy import *
from random import *
import matplotlib.pyplot as plt

#创建用来训练的数据集和用来测试的数据集，及标注信息,先用100个点训练,20个点测试
def createData():
    trainingList0 = [[(1.0,1.0) for j in range(1)] for i in range(50)]
    trainingList1 = [[(1.0,1.0) for j in range(1)] for i in range(50)]
    trainingLabel,trainingLabe0 = [], []
    for i in range(50):
        x = randint(1,10)   #产生1，10之间的随机浮点数，类别0
        y = randint(1,10)
        trainingList0[i] = [(x,y)]
        trainingLabel.append(0) #属于类别0
    for j in range(50):
        x = randint(10,20) #产生十位数，10-20，类别1
        y = randint(10,20)
        trainingList1[j] = [(x,y)]
        trainingLabel.append(1)   #属于类别1
    trainingList = trainingList0 + trainingList1

    #产生测试的数据集
    testList0 = [[(1.0,1.0) for j in range(1)] for i in range(10)]
    testList1 = [[(1.0,1.0) for j in range(1)] for i in range(10)]
    testLabel = []
    for i in range(10):
        x = randint(4,9)
        y = randint(2,10)
        testList0[i] = [(x,y)]
        testLabel.append(0)
    for j in range(10):
        x = randint(11,19)
        y = randint(10,18)
        testList1[j] = [(x,y)]
        testLabel.append(1)
    testList = testList0 + testList1
    print(trainingList)
    return trainingList, trainingLabel, testList, testLabel

#对测试数据集和训练数据集之间的距离进行计算，采用欧式距离 d = ((x1-x2)^2 + (y1-y2)^2)^0.5
def calculateKNN(trainingList, trainingLabel, testList, testLabel, k):
    #20行，100列距离，每行代表一个测试点距离100个训练点的距离,初始化为0
    d = [[0 for j in range(100)] for i in range(20)] 
    #kPointDisList = [1000] * k #初始化k个最近距离为1000
    for i in range(20):
        for j in range(100):
            d[i][j] = getDistance(testList[i], trainingList[j]) #计算距离
            #这里判断求得的距离是否是最小的K个之一，若是则记录下j,更新此测试点的标签类别
            #kPointDisList = updateKPoint(d[i][j], kPointDisList)
    testLable = getKMin(d,k,trainingLabel)
    print(testLabel)
    return testLabel

#计算列表里面的K个最小值，找出对应的标签类别号
def getKMin(dList,k,trainingLabel):
    testLabel = []
    sortedIndexList = argsort(dList)  #返回dLlist从小到大排列的索引的数组
    print(sortedIndexList) 
    print(len(sortedIndexList))
    for x in range(20):
        type1 = 0 
        type0 = 0
        for i in range(k):  #计算最近的K个点，按照类别的多少进行投票表决
            if trainingLabel[sortedIndexList[x][i]] == 1:
                type1 += 1
            else:
                type0 += 1
        if type1 > type0:
            testLabel.append(1)
        else:
            testLabel.append(0)
    return testLabel

#跟新最近的K个点坐标，标签
def updateKPoint(d, kPointDisList):
    if d < max(kPointDisList):
        kPointDisList[kPointDisList.index(max(kPointDisList))] = d
    else:
        pass
    return kPointDisList

#计算两个点的欧式距离,因为有float所以只能用**
def getDistance(a, b):
    return ((a[0][0] - b[0][0])**2 + (a[0][1] - b[0][1])**2)**0.5

#计算KNN分类结果的正确率
def getCorrectRate(testLabel, resultLabel):
    correctNum = 0
    for i in range(len(testLabel)):
        if testLabel[i] == resultLabel[i]:
            correctNum += 1
    return correctNum / float(len(testLabel))

#把分类结果用图形画出来
def drawKNN(trainingList, testList):
    #产生画图数据
    x1,x2,y1,y2 = [],[],[],[]
    for i in range(len(trainingList)):
        x1.append(trainingList[i][0][0])
        y1.append(trainingList[i][0][1])
    for i in range(len(testList)):
        x2.append(testList[i][0][0])
        y2.append(testList[i][0][1])
    #创建一个绘图对象
    fig = plt.figure()
    ax1 = fig.add_subplot(111)
    #添加坐标图修饰 
    plt.xlabel('times(m)')
    plt.ylabel('money(y)')
    plt.title('first pic')
    #画散点图,maker代表散点图形状，c代表颜色
    pl1 = ax1.scatter(x1,y1,c = 'r',marker = '.')
    pl2 = ax1.scatter(x2,y2,c = 'b',marker = '+')
    #设置图标
    plt.legend([pl1,pl2],('train','test'))
    #显示图形
    plt.show()
    #保存图形到本地
    plt.savefig('test.png')
#系统的总体控制逻辑
def testKNN():
    #设定系统的K值
    k = 3
    trainingList, trainingLabel, testList, testLabel = createData()
    resultLabel = calculateKNN(trainingList, trainingLabel, testList, testLabel, k)
    correctRate = getCorrectRate(testLabel, resultLabel)
    print('correctRate is  =  ' + str(correctRate))
    drawKNN(trainingList, testList)

testKNN()
#drawKNN()