横1. np.concatenate(list, axis=0) 将数据进行串接，这里主要是可以将列表进行x轴获得y轴的串接
参数说明：list表示需要串接的列表，axis=0，表示从上到下进行串接
2.np.hstack(list) 将列表进行横向排列
参数说明：list.append([1, 2]), list.append([3, 4]) np.hstack(list) , list等于[1, 2, 3, 4]
3. hasattr(optim, 'sgd') 判断optim.py中是否存在一个函数名为'sgd'的函数， ‘sgd’ = getattr(optim, 'sgd') 把函数功能赋予给‘sgd’字符串
4..reshape(N, 3, 32, 32) 将输入的数据array进行维度的变化
参数说明:N表示样本数,3, 32, 32第二个维度变为3，第三个维度变为32，第四个维度变为32
5..transpose(0, 2, 3, 1) # 进行矩阵维度的位置替换，
参数说明：0表示第一个位置，2表示把三号位置的维度变为二号位置，3表示把三号位置的维度变为2号位置，1表示将1号位置变为4号位置
6.pick.load(f, encoding='latin1') # 将二进制打开的文件f,使用pick.load进行读入
参数说明：f需要读入的二进制文件，encoding表示读入的方式
7.np.argmax(score, axis=1) 表示返回行上最大值得索引值
参数说明：score表示输入的得分值，axis=1表示从左到右进行表示
8.np.maximum(0, x) 如果x小于0，则输入0，否者输出x
参数说明：0表示阈值，x表示输入参数
cifar神经网络的代码说明：
数据主要分为三部分：
第一部分：数据的准备
第二部分：神经网络模型的构造，返回loss和梯度值
第三部分：将数据与模型输入到函数中，用于进行模型的训练，同时进行验证集的预测，来判断验证集的预测结果，保留最好的验证集结果的参数组合
第一部分：数据的准备
第一步：构造列表，使用with open() as f: pickle.load进行数据的载入, 使用.reshape(1000, 3, 32, 32).transpose(0, 3, 1, 2).astype('float')
第二步：使用np.concatenate()将列表进行串接, 选出5000个数据集做为训练集，选择5000到5500个数据做为验证集，从测试集的数据中挑选出500个数据作为测试集
第三步：将返回的训练样本和测试样本进行拆分，从训练样本中取出5000个数据作为训练集，500个样本作为验证集，从测试数据中取出500个数据作为测试集
第四步：将图像样本减去均值), 即- np.mean(train_X, axis=0) ,并使用transpose()将样本数据的维度进行变换
第五步：返回训练样本，验证集，测试集的字典
代码：data_utils.py
import pickle as pickle import numpy as np import os import importlib import sys importlib.reload(sys) #from scipy.misc import imread def load_CIFAR_batch(filename): """ load single batch of cifar """ # 第一步：使用pick.load读取数据，使用.reshape进行矩阵变化和.tanspose进行维度变化 with open(filename, 'rb') as f: datadict = pickle.load(f, encoding='latin1') X = datadict['data'] Y = datadict['labels'] X = X.reshape(10000, 3, 32, 32).transpose(0,2,3,1).astype("float") Y = np.array(Y) return X, Y def load_CIFAR10(ROOT): """ load all of cifar """ xs = [] ys = [] # 第二步：使用列表数据添加，并使用np.concatenate进行串接，去除矩阵的维度 for b in range(1,2): f = os.path.join(ROOT, 'data_batch_%d' % (b, )) X, Y = load_CIFAR_batch(f) xs.append(X) ys.append(Y) # 将数据进行串接 Xtr = np.concatenate(xs) Ytr = np.concatenate(ys) del X, Y # 加载测试数据 Xte, Yte = load_CIFAR_batch(os.path.join(ROOT, 'test_batch')) return Xtr, Ytr, Xte, Yte def get_CIFAR10_data(num_training=5000, num_validation=500, num_test=500): """ Load the CIFAR-10 dataset from disk and perform preprocessing to prepare it for classifiers. These are the same steps as we used for the SVM, but condensed to a single function. """ # Load the raw CIFAR-10 data cifar10_dir = 'D://BaiduNetdiskDownload//神经网络入门基础（PPT，代码）//绁炵粡缃戠粶鍏ラ棬鍩虹锛圥PT锛屼唬鐮侊級//cifar-10-batches-py//' X_train, y_train, X_test, y_test = load_CIFAR10(cifar10_dir) print(X_train.shape) # Subsample the data # 第三步：将返回训练样本和测试样本，进行数据的拆分，分出5000个训练集，验证集和测试集 mask = range(num_training, num_training + num_validation) X_val = X_train[mask] y_val = y_train[mask] mask = range(num_training) X_train = X_train[mask] y_train = y_train[mask] mask = range(num_test) X_test = X_test[mask] y_test = y_test[mask] # Normalize the data: subtract the mean image # 第四步:减去图片的均值，将训练集，验证集和测试集 mean_image = np.mean(X_train, axis=0) X_train -= mean_image X_val -= mean_image X_test -= mean_image # Transpose so that channels come first X_train = X_train.transpose(0, 3, 1, 2).copy() X_val = X_val.transpose(0, 3, 1, 2).copy() X_test = X_test.transpose(0, 3, 1, 2).copy() # Package data into a dictionary # 第五步：返回训练集，验证集和测试集的字典 return { 'X_train': X_train, 'y_train': y_train, 'X_val': X_val, 'y_val': y_val, 'X_test': X_test, 'y_test': y_test, }
第二部分：神经网络模型的构造，用于计算损失值和梯度
第一步：def __iniit(数据维度，隐藏层维度，输出层维度，权重初始值范围，正则化惩罚项初始化)
第二步：构造初始化的self.params用于存放权重参数，初始化权重参数w1,b1, w2, b2
第三步：构造loss函数,前向传播求得loss，反向传播计算各个权重的梯度值
前向传播：
第一步：对输入的X进行第一次的前向传播，包括x * w + b 线性变化和relu激活层函数np.maximum(0, x)
第二步：对第一层的输出结果，在第二层进行线性变化x * w + b， 获得各个类别得分
第三步：如果没有标签，作为预测结果，直接返回得分值
第四步：计算类别的概率值softmax, e^(x-max(x)) / ∑( e^(x-max(x)) ),使用np.sum(-np.log(prob([np.arange(N), y]))) 来表示交叉熵损失函数
第五步：求得softmax / dx 的值为， softmax - 1， 即prob[np.arange(x), y] - 1, 将损失值和softmax对应于x的梯度进行返回
反向传播：
第一步：对于前向传播求得的softmax/dx获得的导数值dout，将其回传到第二层，求得dx(用于第一层的回传)，dw2， db2 = dout * w(第二层的权重w), dout * x(第二层输入)， np.sum(dout, axis=0)
第二步：对于第二层回传的dx，进行第一层的回传，第一层进行了两步操作，第一步是线性变化，第二步是relu激活层，先对激活层进行回传，对于激活层的回传，输入值大于0的，回传的结果不变，输入值小于0的，回传的结果为0，即dx[x<0] = 0 ， 将回传的结果用于线性dx, dw1, db1与上述步骤相同
第三步：将求得的dw2，db2，dw1， db1保存在grads中，将loss和梯度值进行返回
主代码：fc_net.py
from layer_utils import * import numpy as np class TwoLayerNet(object): # 第一步：构造初始化超参数,在书写代码的时候可以使用 def __init__(self, input_dim=3*32*32, hidden_dim=100, num_classes=10, weight_scale=1e-3, reg=0.0): """ Initialize a new network. Inputs: - input_dim: An integer giving the size of the input - hidden_dim: An integer giving the size of the hidden layer - num_classes: An integer giving the number of classes to classify - dropout: Scalar between 0 and 1 giving dropout strength. - weight_scale: Scalar giving the standard deviation for random initialization of the weights. - reg: Scalar giving L2 regularization strength. """ # 第二步：构造权重字典，并且进行w1,b1,w2,b2的权重初始化 self.params = {} self.reg = reg self.params['W1'] = weight_scale * np.random.randn(input_dim, hidden_dim) self.params['b1'] = np.zeros((1, hidden_dim)) self.params['W2'] = weight_scale * np.random.randn(hidden_dim, num_classes) self.params['b2'] = np.zeros((1, num_classes)) # 第三步：构造loss函数用于进行前向传播和反向传播，返回loss和权重梯度grads def loss(self, X, y=None): """ Compute loss and gradient for a minibatch of data. Inputs: - X: Array of input data of shape (N, d_1, ..., d_k) - y: Array of labels, of shape (N,). y[i] gives the label for X[i]. Returns: If y is None, then run a test-time forward pass of the model and return: - scores: Array of shape (N, C) giving classification scores, where scores[i, c] is the classification score for X[i] and class c. If y is not None, then run a training-time forward and backward pass and return a tuple of: - loss: Scalar value giving the loss - grads: Dictionary with the same keys as self.params, mapping parameter names to gradients of the loss with respect to those parameters. """ # 前向传播，计算得分和损失值 scores = None N = X.shape[0] # Unpack variables from the params dictionary # 权重参数w和b, # 获得当前的参数值 W1, b1 = self.params['W1'], self.params['b1'] W2, b2 = self.params['W2'], self.params['b2'] # 第一步：第一层神经网络进行线性变化和relu变化 第一层的输出结果 h1, cache1 = affine_relu_forward(X, W1, b1) # 第二步：第二层神经网络进行线性变化 out, cache2 = affine_forward(h1, W2, b2) scores = out # (N,C) # 第三步：如果没有labels，直接返回得分值作为预测结果 if y is None: return scores # 第四步：计算损失值和softmax的反向传播的结果 loss, grads = 0, {} data_loss, dscores = softmax_loss(scores, y) # 加上L2正则化惩罚项 reg_loss = 0.5 * self.reg * np.sum(W1*W1) + 0.5 * self.reg * np.sum(W2*W2) loss = data_loss + reg_loss # 反向传播，用于计算梯度值 # 第一步：计算传到第二层的反向传播的结果，即dw2和db2 dh1, dW2, db2 = affine_backward(dscores, cache2) # 第二步：计算relu的反向传播以及x*w + b 反向传播的结果 dX, dW1, db1 = affine_relu_backward(dh1, cache1) # Add the regularization gradient contribution # 加入正则化求导的梯度值dw2 和 dw1 dW2 += self.reg * W2 dW1 += self.reg * W1 # 第三步：将梯度值加入到grads的字典中, 返回损失值和grads梯度值 grads['W1'] = dW1 grads['b1'] = db1 grads['W2'] = dW2 grads['b2'] = db2 return loss, grads
调用代码：layer_utils.py
from layers import * def affine_relu_forward(x, w, b): """ Convenience layer that perorms an affine transform followed by a ReLU Inputs: - x: Input to the affine layer - w, b: Weights for the affine layer Returns a tuple of: - out: Output from the ReLU - cache: Object to give to the backward pass """ a, fc_cache = affine_forward(x, w, b) out, relu_cache = relu_forward(a) cache = (fc_cache, relu_cache) return out, cache def affine_relu_backward(dout, cache): """ Backward pass for the affine-relu convenience layer """ fc_cache, relu_cache = cache da = relu_backward(dout, relu_cache) dx, dw, db = affine_backward(da, fc_cache) return dx, dw, db pass def conv_relu_forward(x, w, b, conv_param): """ A convenience layer that performs a convolution followed by a ReLU. Inputs: - x: Input to the convolutional layer - w, b, conv_param: Weights and parameters for the convolutional layer Returns a tuple of: - out: Output from the ReLU - cache: Object to give to the backward pass """ a, conv_cache = conv_forward_fast(x, w, b, conv_param) out, relu_cache = relu_forward(a) cache = (conv_cache, relu_cache) return out, cache def conv_relu_backward(dout, cache): """ Backward pass for the conv-relu convenience layer. """ conv_cache, relu_cache = cache da = relu_backward(dout, relu_cache) dx, dw, db = conv_backward_fast(da, conv_cache) return dx, dw, db def conv_relu_pool_forward(x, w, b, conv_param, pool_param): """ Convenience layer that performs a convolution, a ReLU, and a pool. Inputs: - x: Input to the convolutional layer - w, b, conv_param: Weights and parameters for the convolutional layer - pool_param: Parameters for the pooling layer Returns a tuple of: - out: Output from the pooling layer - cache: Object to give to the backward pass """ a, conv_cache = conv_forward_fast(x, w, b, conv_param) s, relu_cache = relu_forward(a) out, pool_cache = max_pool_forward_fast(s, pool_param) cache = (conv_cache, relu_cache, pool_cache) return out, cache def conv_relu_pool_backward(dout, cache): """ Backward pass for the conv-relu-pool convenience layer """ conv_cache, relu_cache, pool_cache = cache ds = max_pool_backward_fast(dout, pool_cache) da = relu_backward(ds, relu_cache) dx, dw, db = conv_backward_fast(da, conv_cache) return dx, dw, db
调用代码：layers
import numpy as np def affine_forward(x, w, b): """ Computes the forward pass for an affine (fully-connected) layer. The input x has shape (N, d_1, ..., d_k) and contains a minibatch of N examples, where each example x[i] has shape (d_1, ..., d_k). We will reshape each input into a vector of dimension D = d_1 * ... * d_k, and then transform it to an output vector of dimension M. Inputs: - x: A numpy array containing input data, of shape (N, d_1, ..., d_k) - w: A numpy array of weights, of shape (D, M) - b: A numpy array of biases, of shape (M,) Returns a tuple of: - out: output, of shape (N, M) - cache: (x, w, b) """ out = None # Reshape x into rows N = x.shape[0] x_row = x.reshape(N, -1) # (N,D) out = np.dot(x_row, w) + b # (N,M) cache = (x, w, b) return out, cache def affine_backward(dout, cache): """ Computes the backward pass for an affine layer. Inputs: - dout: Upstream derivative, of shape (N, M) - cache: Tuple of: - x: Input data, of shape (N, d_1, ... d_k) - w: Weights, of shape (D, M) Returns a tuple of: - dx: Gradient with respect to x, of shape (N, d1, ..., d_k) - dw: Gradient with respect to w, of shape (D, M) - db: Gradient with respect to b, of shape (M,) """ x, w, b = cache dx, dw, db = None, None, None dx = np.dot(dout, w.T) # (N,D) dx = np.reshape(dx, x.shape) # (N,d1,...,d_k) x_row = x.reshape(x.shape[0], -1) # (N,D) dw = np.dot(x_row.T, dout) # (D,M) db = np.sum(dout, axis=0, keepdims=True) # (1,M) return dx, dw, db def relu_forward(x): """ Computes the forward pass for a layer of rectified linear units (ReLUs). Input: - x: Inputs, of any shape Returns a tuple of: - out: Output, of the same shape as x - cache: x """ out = None out = ReLU(x) cache = x return out, cache def relu_backward(dout, cache): """ Computes the backward pass for a layer of rectified linear units (ReLUs). Input: - dout: Upstream derivatives, of any shape - cache: Input x, of same shape as dout Returns: - dx: Gradient with respect to x """ dx, x = None, cache dx = dout dx[x <= 0] = 0 return dx def svm_loss(x, y): """ Computes the loss and gradient using for multiclass SVM classification. Inputs: - x: Input data, of shape (N, C) where x[i, j] is the score for the jth class for the ith input. - y: Vector of labels, of shape (N,) where y[i] is the label for x[i] and 0 <= y[i] < C Returns a tuple of: - loss: Scalar giving the loss - dx: Gradient of the loss with respect to x """ N = x.shape[0] correct_class_scores = x[np.arange(N), y] margins = np.maximum(0, x - correct_class_scores[:, np.newaxis] + 1.0) margins[np.arange(N), y] = 0 loss = np.sum(margins) / N num_pos = np.sum(margins > 0, axis=1) dx = np.zeros_like(x) dx[margins > 0] = 1 dx[np.arange(N), y] -= num_pos dx /= N return loss, dx def softmax_loss(x, y): """ Computes the loss and gradient for softmax classification. Inputs: - x: Input data, of shape (N, C) where x[i, j] is the score for the jth class for the ith input. - y: Vector of labels, of shape (N,) where y[i] is the label for x[i] and 0 <= y[i] < C Returns a tuple of: - loss: Scalar giving the loss - dx: Gradient of the loss with respect to x """ # 计算概率值 probs = np.exp(x - np.max(x, axis=1, keepdims=True)) probs /= np.sum(probs, axis=1, keepdims=True) N = x.shape[0] # 计算损失值函数 loss = -np.sum(np.log(probs[np.arange(N), y])) / N # 计算softmax回传即dsoftmax / dx 的结果 dx = probs.copy() dx[np.arange(N), y] -= 1 dx /= N return loss, dx def ReLU(x): """ReLU non-linearity.""" return np.maximum(0, x)
第三部分：将数据与模型进行输入，使用bacth_size获得loss和grads，使用grads对参数params进行更新，使用验证集数据来判别准确率，返回验证集最好的参数集
第一步：获得输入字典中的各个参数值，如果不存在就使用默认值，多出来的参数值就报错，使用hasattr判断.py文件中是否有这个名字的函数,使用getattr将函数对应的功能复制给函数名
第二步：进行参数的初始化操作，比如epoch=0， 以及一些loss和accuracy的列表，因为要进行动量梯度化，因此对学习率做一个参数名和学习率对应的字典
第三步：进行模型的参数训练，使用num_sample / batch_size计算一个epoch循环的次数，使用epoch*every_epoch，计算所有的循环次数
第四步：循环，进行参数的更新：
第一步：从样本中，随机选择batch_size个序列号，获得一个batch_size的训练数据和标签值
第二步：带入到model.loss函数中计算loss和grads
第三步：使用动量梯度下降v = 0.9*v - 学习率*dw， w = w + v 进行参数更新，同时返回w和v，对数据中的w和v进行更新
第五步：每循环一个epoch，学习率*0.9， 降低一次，每次循环epoch计算一次训练集和验证集的准确率，使用model.loss，对于返回的score值，使用
(y == np.argmax(scores)).mean() 计算准确率
第六步：对于验证集结果比最好验证集结果要好的参数集，进行保存，在循环的末尾，将其赋值给当前的params
代码：
主函数：Solver.py
import numpy as np import optim class Solver(object): """ A Solver encapsulates all the logic necessary for training classification models. The Solver performs stochastic gradient descent using different update rules defined in optim.py. The solver accepts both training and validataion data and labels so it can periodically check classification accuracy on both training and validation data to watch out for overfitting. To train a model, you will first construct a Solver instance, passing the model, dataset, and various optoins (learning rate, batch size, etc) to the constructor. You will then call the train() method to run the optimization procedure and train the model. After the train() method returns, model.params will contain the parameters that performed best on the validation set over the course of training. In addition, the instance variable solver.loss_history will contain a list of all losses encountered during training and the instance variables solver.train_acc_history and solver.val_acc_history will be lists containing the accuracies of the model on the training and validation set at each epoch. Example usage might look something like this: data = { 'X_train': # training data 'y_train': # training labels 'X_val': # validation data 'X_train': # validation labels } model = MyAwesomeModel(hidden_size=100, reg=10) solver = Solver(model, data, update_rule='sgd', optim_config={ 'learning_rate': 1e-3, }, lr_decay=0.95, num_epochs=10, batch_size=100, print_every=100) solver.train() A Solver works on a model object that must conform to the following API: - model.params must be a dictionary mapping string parameter names to numpy arrays containing parameter values. - model.loss(X, y) must be a function that computes training-time loss and gradients, and test-time classification scores, with the following inputs and outputs: Inputs: - X: Array giving a minibatch of input data of shape (N, d_1, ..., d_k) - y: Array of labels, of shape (N,) giving labels for X where y[i] is the label for X[i]. Returns: If y is None, run a test-time forward pass and return: - scores: Array of shape (N, C) giving classification scores for X where scores[i, c] gives the score of class c for X[i]. If y is not None, run a training time forward and backward pass and return a tuple of: - loss: Scalar giving the loss - grads: Dictionary with the same keys as self.params mapping parameter names to gradients of the loss with respect to those parameters. """ def __init__(self, model, data, **kwargs): """ Construct a new Solver instance. Required arguments: - model: A model object conforming to the API described above - data: A dictionary of training and validation data with the following: 'X_train': Array of shape (N_train, d_1, ..., d_k) giving training images 'X_val': Array of shape (N_val, d_1, ..., d_k) giving validation images 'y_train': Array of shape (N_train,) giving labels for training images 'y_val': Array of shape (N_val,) giving labels for validation images Optional arguments: - update_rule: A string giving the name of an update rule in optim.py. Default is 'sgd'. - optim_config: A dictionary containing hyperparameters that will be passed to the chosen update rule. Each update rule requires different hyperparameters (see optim.py) but all update rules require a 'learning_rate' parameter so that should always be present. - lr_decay: A scalar for learning rate decay; after each epoch the learning rate is multiplied by this value. - batch_size: Size of minibatches used to compute loss and gradient during training. - num_epochs: The number of epochs to run for during training. - print_every: Integer; training losses will be printed every print_every iterations. - verbose: Boolean; if set to false then no output will be printed during training. """ # 第一步：样本的初始化，并获得输入的字典的参数，使用pop来获得参数值, 使用hasattr判断函数是否存在 # 使用getattr获得函数功能的函数名 self.model = model self.X_train = data['X_train'] self.y_train = data['y_train'] self.X_val = data['X_val'] self.y_val = data['y_val'] # Unpack keyword arguments # .pop表示的如果不传入的话，就使用当前值做替代 self.update_rule = kwargs.pop('update_rule', 'sgd') self.optim_config = kwargs.pop('optim_config', {}) self.lr_decay = kwargs.pop('lr_decay', 1.0) self.batch_size = kwargs.pop('batch_size', 100) self.num_epochs = kwargs.pop('num_epochs', 10) self.print_every = kwargs.pop('print_every', 10) self.verbose = kwargs.pop('verbose', True) # Throw an error if there are extra keyword arguments if len(kwargs) > 0: extra = ', '.join('"%s"' % k for k in kwargs.keys()) raise ValueError('Unrecognized arguments %s' % extra) # Make sure the update rule exists, then replace the string # name with the actual function if not hasattr(optim, self.update_rule): raise ValueError('Invalid update_rule "%s"' % self.update_rule) self.update_rule = getattr(optim, self.update_rule) self._reset() def _reset(self): """ Set up some book-keeping variables for optimization. Don't call this manually. """ # Set up some variables for book-keeping # 第二步：初始化参数，其次是对dw与参数名对应的字典，用于进行动量梯度下降 self.epoch = 0 self.best_val_acc = 0 self.best_params = {} self.loss_history = [] self.train_acc_history = [] self.val_acc_history = [] # Make a deep copy of the optim_config for each parameter self.optim_configs = {} for p in self.model.params: d = {k: v for k, v in self.optim_config.items()} self.optim_configs[p] = d def _step(self): """ Make a single gradient update. This is called by train() and should not be called manually. """ # Make a minibatch of training data num_train = self.X_train.shape[0] batch_mask = np.random.choice(num_train, self.batch_size) X_batch = self.X_train[batch_mask] y_batch = self.y_train[batch_mask] # Compute loss and gradient loss, grads = self.model.loss(X_batch, y_batch) self.loss_history.append(loss) # Perform a parameter update for p, w in self.model.params.items(): dw = grads[p] config = self.optim_configs[p] next_w, next_config = self.update_rule(w, dw, config) self.model.params[p] = next_w self.optim_configs[p] = next_config def check_accuracy(self, X, y, num_samples=None, batch_size=100): """ Check accuracy of the model on the provided data. Inputs: - X: Array of data, of shape (N, d_1, ..., d_k) - y: Array of labels, of shape (N,) - num_samples: If not None, subsample the data and only test the model on num_samples datapoints. - batch_size: Split X and y into batches of this size to avoid using too much memory. c Returns: - acc: Scalar giving the fraction of instances that were correctly classified by the model. """ # 对训练数据进行下采样，用于进行训练集样本的准确率计算 N = X.shape[0] if num_samples is not None and N > num_samples: mask = np.random.choice(N, num_samples) N = num_samples X = X[mask] y = y[mask] # Compute predictions in batches num_batches = N / batch_size if N % batch_size != 0: num_batches += 1 y_pred = [] # batch进行交叉验证，每次只进行部分的结果预测。 for i in range(int(num_batches)): start = i * batch_size end = (i + 1) * batch_size scores = self.model.loss(X[start:end]) y_pred.append(np.argmax(scores, axis=1)) y_pred = np.hstack(y_pred) acc = np.mean(y_pred == y) return acc def train(self): """ Run optimization to train the model. """ # 第三步：计算循环的次数 num_train = self.X_train.shape[0] iterations_per_epoch = max(num_train / self.batch_size, 1) num_iterations = self.num_epochs * iterations_per_epoch for t in range(int(num_iterations)): # 第四步：每一次循环，随机选择一个batch_size的数据获得损失值和更新的参数,将其存入，同时更新dw，用做下一次的动量梯度 self._step() # Maybe print training loss # 打印当前的损失值 if self.verbose and t % self.print_every == 0: print('(Iteration %d / %d) loss: %f' % ( t + 1, num_iterations, self.loss_history[-1])) # At the end of every epoch, increment the epoch counter and decay the # learning rate. epoch_end = (t + 1) % iterations_per_epoch == 0 # 第五步： 循环一个epoch值，对optim_configs进行学习率的更新 if epoch_end: self.epoch += 1 for k in self.optim_configs: self.optim_configs[k]['learning_rate'] *= self.lr_decay # Check train and val accuracy on the first iteration, the last # iteration, and at the end of each epoch. # 第六步：每经过一个epoch值，计算一次准确率 first_it = (t == 0) last_it = (t == num_iterations + 1) if first_it or last_it or epoch_end: train_acc = self.check_accuracy(self.X_train, self.y_train, num_samples=1000) val_acc = self.check_accuracy(self.X_val, self.y_val) self.train_acc_history.append(train_acc) self.val_acc_history.append(val_acc) if self.verbose: print('(Epoch %d / %d) train acc: %f; val_acc: %f' % ( self.epoch, self.num_epochs, train_acc, val_acc)) # Keep track of the best model if val_acc > self.best_val_acc: self.best_val_acc = val_acc self.best_params = {} for k, v in self.model.params.items(): self.best_params[k] = v.copy() # At the end of training swap the best params into the model self.model.params = self.best_params
副函数：optim.py, 里面的参数更新的方法：sgd， sgd_momentum,rmsprop, adam
import numpy as np def sgd(w, dw, config=None): """ Performs vanilla stochastic gradient descent. config format: - learning_rate: Scalar learning rate. """ if config is None: config = {} config.setdefault('learning_rate', 1e-2) w -= config['learning_rate'] * dw return w, config def sgd_momentum(w, dw, config=None): """ Performs stochastic gradient descent with momentum. config format: - learning_rate: Scalar learning rate. - momentum: Scalar between 0 and 1 giving the momentum value. Setting momentum = 0 reduces to sgd. - velocity: A numpy array of the same shape as w and dw used to store a moving average of the gradients. """ if config is None: config = {} config.setdefault('learning_rate', 1e-2) config.setdefault('momentum', 0.9) v = config.get('velocity', np.zeros_like(w)) next_w = None v = config['momentum'] * v - config['learning_rate'] * dw next_w = w + v config['velocity'] = v return next_w, config def rmsprop(x, dx, config=None): """ Uses the RMSProp update rule, which uses a moving average of squared gradient values to set adaptive per-parameter learning rates. config format: - learning_rate: Scalar learning rate. - decay_rate: Scalar between 0 and 1 giving the decay rate for the squared gradient cache. - epsilon: Small scalar used for smoothing to avoid dividing by zero. - cache: Moving average of second moments of gradients. """ if config is None: config = {} config.setdefault('learning_rate', 1e-2) config.setdefault('decay_rate', 0.99) config.setdefault('epsilon', 1e-8) config.setdefault('cache', np.zeros_like(x)) next_x = None cache = config['cache'] decay_rate = config['decay_rate'] learning_rate = config['learning_rate'] epsilon = config['epsilon'] cache = decay_rate * cache + (1 - decay_rate) * (dx**2) x += - learning_rate * dx / (np.sqrt(cache) + epsilon) config['cache'] = cache next_x = x return next_x, config def adam(x, dx, config=None): """ Uses the Adam update rule, which incorporates moving averages of both the gradient and its square and a bias correction term. config format: - learning_rate: Scalar learning rate. - beta1: Decay rate for moving average of first moment of gradient. - beta2: Decay rate for moving average of second moment of gradient. - epsilon: Small scalar used for smoothing to avoid dividing by zero. - m: Moving average of gradient. - v: Moving average of squared gradient. - t: Iteration number. """ if config is None: config = {} config.setdefault('learning_rate', 1e-3) config.setdefault('beta1', 0.9) config.setdefault('beta2', 0.999) config.setdefault('epsilon', 1e-8) config.setdefault('m', np.zeros_like(x)) config.setdefault('v', np.zeros_like(x)) config.setdefault('t', 0) next_x = None m = config['m'] v = config['v'] beta1 = config['beta1'] beta2 = config['beta2'] learning_rate = config['learning_rate'] epsilon = config['epsilon'] t = config['t'] t += 1 m = beta1 * m + (1 - beta1) * dx v = beta2 * v + (1 - beta2) * (dx**2) m_bias = m / (1 - beta1**t) v_bias = v / (1 - beta2**t) x += - learning_rate * m_bias / (np.sqrt(v_bias) + epsilon) next_x = x config['m'] = m config['v'] = v config['t'] = t return next_x, config
上述代码的主函数 two_layer_fc_net_start.py
import matplotlib.pyplot as plt from fc_net import * from data_utils import get_CIFAR10_data from solver import Solver import numpy as np # 第一部分：读取数据 data = get_CIFAR10_data() # 第二部分：构建模型 model = TwoLayerNet(reg=0.9) # 第三部分：进行模型的参数训练 solver = Solver(model, data, lr_decay=0.95, print_every=100, num_epochs=40, batch_size=400, update_rule='sgd_momentum', optim_config={'learning_rate': 5e-4, 'momentum': 0.9}) solver.train() # 画图 plt.subplot(2, 1, 1) plt.title('Training loss') plt.plot(solver.loss_history, 'o') plt.xlabel('Iteration') plt.subplot(2, 1, 2) plt.title('Accuracy') plt.plot(solver.train_acc_history, '-o', label='train') plt.plot(solver.val_acc_history, '-o', label='val') plt.plot([0.5] * len(solver.val_acc_history), 'k--') plt.xlabel('Epoch') plt.legend(loc='lower right') plt.gcf().set_size_inches(15, 12) plt.show() # 对模型计算验证集和测试集 best_model = model y_test_pred = np.argmax(best_model.loss(data['X_test']), axis=1) y_val_pred = np.argmax(best_model.loss(data['X_val']), axis=1) print('Validation set accuracy: ', (y_val_pred == data['y_val']).mean()) print('Test set accuracy: ', (y_test_pred == data['y_test']).mean()) # Validation set accuracy: about 52.9% # Test set accuracy: about 54.7% # Visualize the weights of the best network """ from vis_utils import visualize_grid def show_net_weights(net): W1 = net.params['W1'] W1 = W1.reshape(3, 32, 32, -1).transpose(3, 1, 2, 0) plt.imshow(visualize_grid(W1, padding=3).astype('uint8')) plt.gca().axis('off') show_net_weights(best_model) plt.show() """
