python实现knn

邻近算法，或者说K最近邻(kNN，k-NearestNeighbor)分类算法是数据挖掘分类技术中最简单的方法之一。所谓K最近邻，就是k个最近的邻居的意思，说的是每个样本都可以用它最接近的k个邻居来代表。

kNN算法的核心思想是如果一个样本在特征空间中的k个最相邻的样本中的大多数属于某一个类别，则该样本也属于这个类别，并具有这个类别上样本的特性。

概念很简单，更多的解释可以参考百度百科，有图有示例，讲的非常清楚。

接下来看看怎么用python实现KNN，代码中都是详细的注释：

 

首先是对载入数据的部分函数，这里主要看看CIFIA10的数据格式就知道代码的意思了

from __future__ import print_function

from six.moves import cPickle as pickle
import numpy as np
import os
from scipy.misc import imread
import platform

def load_pickle(f):
    version = platform.python_version_tuple()
    if version[0] == '2':
        return  pickle.load(f)
    elif version[0] == '3':
        return  pickle.load(f, encoding='latin1')
    raise ValueError("invalid python version: {}".format(version))

def load_CIFAR_batch(filename):
  """ CIRAR的数据是分批的，这个函数的功能是载入一批数据 """
  with open(filename, 'rb') as f:
    datadict = load_pickle(f) #以二进制方式打开文件
    X = datadict['data']
    Y = datadict['labels']
    X = X.reshape(10000, 3, 32, 32).transpose(0,2,3,1).astype("float")
    Y = np.array(Y)
    return X, Y

def load_CIFAR10(ROOT):
  """ load 所有的数据 """
  xs = []
  ys = []
  for b in range(1,6):
    f = os.path.join(ROOT, 'data_batch_%d' % (b, ))
    X, Y = load_CIFAR_batch(f)
    xs.append(X)
    ys.append(Y)    
  Xtr = np.concatenate(xs)
  Ytr = np.concatenate(ys)
  del X, Y
  Xte, Yte = load_CIFAR_batch(os.path.join(ROOT, 'test_batch'))
  return Xtr, Ytr, Xte, Yte

然后是KNN类，定义了KNN的距离的计算方式、训练和预测函数：

import numpy as np

class KNearestNeighbor(object):
  """ 
  kNN 分类器 
  这里度量两张图片之间的距离就直接简单的采用L2距离
  实际上要达到比较好的效果需要设计更好的距离距离方式 
  """

  def __init__(self):
    pass

  def train(self, X, y):
    """
    训练过程基本上没有什么操作，只是简单的记录下所有的数据

    Inputs:
    - X(N, D) N个输入图片，每张图片表示为D位向量 
    - y(N,) 标签
    """
    self.X_train = X
    self.y_train = y
    
  def predict(self, X, k=1, num_loops=0):
    """
    对于新的输入，给出预测分类

    Inputs:
    - X(num_test, D) 
    - k: 选择用来决定输出的最相近邻居的个数
    - num_loops:这里实现了3种方式来实现L2距离的计算，比较一下计算速度，
                都是利用了numpy的broadcast机制。
                可以看到使用numpy内置的方式计算速度远远高于自己写的循环

    Returns:
    - y(num_test,)：预测的分类下标 
 
    """
    if num_loops == 0:
      dists = self.compute_distances_no_loops(X)
    elif num_loops == 1:
      dists = self.compute_distances_one_loop(X)
    elif num_loops == 2:
      dists = self.compute_distances_two_loops(X)
    else:
      raise ValueError('Invalid value %d for num_loops' % num_loops)

    return self.predict_labels(dists, k=k)

  def compute_distances_two_loops(self, X):
    """
    Inputs:
    - X(num_test, D)：test data.

    Returns:
    - dists(num_test, num_train)：dists[i, j]表示测试数据i和训练数据j之间的L2距离
    """

    num_test = X.shape[0]
    num_train = self.X_train.shape[0]
    dists = np.zeros((num_test, num_train))
    for i in range(num_test):
      for j in range(num_train):
        dists[i,j]=np.sqrt(np.sum(np.square(X[i]-self.X_train[j])))
    return dists

  def compute_distances_one_loop(self, X):
    num_test = X.shape[0]
    num_train = self.X_train.shape[0]
    dists = np.zeros((num_test, num_train))
    for i in range(num_test):
      dists[i,:]=np.sqrt(np.sum(np.square(X[i]-self.X_train),axis=1))
    return dists

  def compute_distances_no_loops(self, X):
 
    num_test = X.shape[0]
    num_train = self.X_train.shape[0]
    dists = np.zeros((num_test, num_train)) 
    #这里需要使用一点矩阵和广播的小技巧，具体的看下面的操作自己体会
    dists+=(np.sum(np.square(X),axis=1)).reshape(-1,1)
    dists+=(np.sum(np.square(self.X_train),axis=1)).reshape(1,-1)
    dists-=2*np.dot(X,self.X_train.T)
    dists=np.sqrt(dists)
    
    return dists

  def predict_labels(self, dists, k=1):
    """
    给出测试图片和训练图片的距离矩阵，为每个测试图片分类

    Inputs:
    - dists(num_test, num_train) 

    Returns:
    - y: (num_test,)   
    """

    num_test = dists.shape[0]
    y_pred = np.zeros(num_test)
    for i in range(num_test):
      # 长度为k的list保存第i张测试图片距离最近的训练数据的下标
      closest_y = []
      closest_y=self.y_train[np.argsort(dists[i])[:k]]
      y_pred[i]=np.argmax(np.bincount(closest_y))
    return y_pred

最后是主函数部分，载入数据，调用KNN类的实例去训练和预测。并使用k折交叉验证去选择合适的超参数k：

# coding: utf-8

# KNN
# KNN分类器主要分为两个步骤：
# - 训练阶段, 简单的记忆所有的输入数据（存储）
# - 预测阶段, 对与每一个输入，在所有的存储数据中选择k个与输入最接近的
# - k是超参数
# 


import random
import numpy as np
from cs231n.data_utils import load_CIFAR10
import matplotlib.pyplot as plt


#get_ipython().run_line_magic('matplotlib', 'inline')
plt.rcParams['figure.figsize'] = (10.0, 8.0) # set default size of plots
plt.rcParams['image.interpolation'] = 'nearest'
plt.rcParams['image.cmap'] = 'gray'

# Load CIFAR-10 的数据.
cifar10_dir = 'cs231n/datasets/cifar-10-batches-py'
X_train, y_train, X_test, y_test = load_CIFAR10(cifar10_dir)

# 通过输出数据的维度检查数据加载是否正确
print('Training data shape: ', X_train.shape)
print('Training labels shape: ', y_train.shape)
print('Test data shape: ', X_test.shape)
print('Test labels shape: ', y_test.shape)


# 可视化一些数据集中的样例.
classes = ['plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck']
num_classes = len(classes)
samples_per_class = 7
for y, cls in enumerate(classes):
    idxs = np.flatnonzero(y_train == y)   #得到每一类对应图片的下标
    idxs = np.random.choice(idxs, samples_per_class, replace=False) #在该类的所有图片中随机选择
    for i, idx in enumerate(idxs):
        plt_idx = i * num_classes + y + 1
        plt.subplot(samples_per_class, num_classes, plt_idx)
        plt.imshow(X_train[idx].astype('uint8'))
        plt.axis('off')
        if i == 0:
            plt.title(cls)
plt.show()


# 采样，不使用全部数据，训练的更快一点，先来看看效果
# 程序全部跑通之后可以优化一下方式，使用全部数据来试试效果
num_training = 5000
mask = list(range(num_training))
X_train = X_train[mask]
y_train = y_train[mask]

num_test = 500
mask = list(range(num_test))
X_test = X_test[mask]
y_test = y_test[mask]


# 把图片Reshape到一维 
X_train = np.reshape(X_train, (X_train.shape[0], -1))
X_test = np.reshape(X_test, (X_test.shape[0], -1))
print(X_train.shape, X_test.shape)


from cs231n.classifiers import KNearestNeighbor

classifier = KNearestNeighbor()
classifier.train(X_train, y_train)

dists = classifier.compute_distances_two_loops(X_test)
print(dists.shape)

# 可视化距离矩阵，每一行代表一张输入图片到所有训练数据的距离
plt.imshow(dists, interpolation='none')
plt.show()



y_test_pred = classifier.predict_labels(dists, k=1)
num_correct = np.sum(y_test_pred == y_test)
accuracy = float(num_correct) / num_test
print('Got %d / %d correct => accuracy: %f' % (num_correct, num_test, accuracy))



y_test_pred = classifier.predict_labels(dists, k=5)
num_correct = np.sum(y_test_pred == y_test)
accuracy = float(num_correct) / num_test
print('Got %d / %d correct => accuracy: %f' % (num_correct, num_test, accuracy))




dists_one = classifier.compute_distances_one_loop(X_test)

# 验证2种实现方式得到的距离矩阵是否等价
difference = np.linalg.norm(dists - dists_one, ord='fro')
print('Difference was: %f' % (difference, ))
if difference < 0.001:
    print('Good! The distance matrices are the same')
else:
    print('Uh-oh! The distance matrices are different')



dists_two = classifier.compute_distances_no_loops(X_test)
difference = np.linalg.norm(dists - dists_two, ord='fro')
print('Difference was: %f' % (difference, ))
if difference < 0.001:
    print('Good! The distance matrices are the same')
else:
    print('Uh-oh! The distance matrices are different')



def time_function(f, *args):
    """
    计算完成f函数花费的时间
    """
    import time
    tic = time.time()
    f(*args)
    toc = time.time()
    return toc - tic

two_loop_time = time_function(classifier.compute_distances_two_loops, X_test)
print('Two loop version took %f seconds' % two_loop_time)

one_loop_time = time_function(classifier.compute_distances_one_loop, X_test)
print('One loop version took %f seconds' % one_loop_time)

no_loop_time = time_function(classifier.compute_distances_no_loops, X_test)
print('No loop version took %f seconds' % no_loop_time)


#使用交叉验证决定k值
num_folds = 5
k_choices = [1, 3, 5, 8, 10, 12, 15, 20, 50, 100]

X_train_folds = []
y_train_folds = []
X_train_folds=np.array_split(X_train,num_folds)
y_train_folds=np.array_split(y_train,num_folds)
print(X_train_folds[0].shape)
print(y_train_folds[0].shape)


#记录不同的k值对应的正确率，每个k值会对应num_folds个正确率
k_to_accuracies = {}

for k_ in k_choices:
    k_to_accuracies.setdefault(k_, [])
for i in range(num_folds):
    classifier = KNearestNeighbor()
    X_val_train = np.concatenate(X_train_folds[0:i] + X_train_folds[i+1:],axis=0)
    y_val_train = np.concatenate(y_train_folds[0:i] + y_train_folds[i+1:],axis=0)
    classifier.train(X_val_train, y_val_train)
    for k_ in k_choices:
        y_val_pred = classifier.predict(X_train_folds[i], k=k_)
        num_correct = np.sum(y_val_pred == y_train_folds[i])
        accuracy = float(num_correct) / len(y_val_pred)
        k_to_accuracies[k_] = k_to_accuracies[k_] + [accuracy]



for k in sorted(k_to_accuracies):
    for accuracy in k_to_accuracies[k]:
        print('k = %d, accuracy = %f' % (k, accuracy))


for k in k_choices:
    accuracies = k_to_accuracies[k]
    plt.scatter([k] * len(accuracies), accuracies)


accuracies_mean = np.array([np.mean(v) for k,v in sorted(k_to_accuracies.items())])
accuracies_std = np.array([np.std(v) for k,v in sorted(k_to_accuracies.items())])
plt.errorbar(k_choices, accuracies_mean, yerr=accuracies_std)
plt.title('Cross-validation on k')
plt.xlabel('k')
plt.ylabel('Cross-validation accuracy')
plt.show()


#选择最好的k值计算正确率
best_k = k_choices[np.argmax(accuracies_mean)]

classifier = KNearestNeighbor()
classifier.train(X_train, y_train)
y_test_pred = classifier.predict(X_test, k=best_k)


num_correct = np.sum(y_test_pred == y_test)
accuracy = float(num_correct) / num_test
print('Got %d / %d correct => accuracy: %f' % (num_correct, num_test, accuracy))