从零和使用mxnet实现dropout
需求:
- 从零和使用mxnet实现dropout
数据集：
- 使用load_digits()手写数字数据集
要求：
- 使用1个掩藏层n_hidden1 = 36，激活函数为relu，损失函数为softmax交叉熵损失函数
注意：
- drop函数的实现方法
- 训练和测试时drop的区别
1.从零实现dropout
from sklearn import datasets
from mxnet import gluon,nd,autograd,init
from mxnet.gluon import nn,data as gdata,loss as gloss,trainer
# 加载数据集
digits = datasets.load_digits()
features,labels = nd.array(digits['data']),nd.array(digits['target'])
print(features.shape,labels.shape)
labels_onehot = nd.one_hot(labels,10)
print(labels_onehot.shape)
(1797, 64) (1797,)
(1797, 10)
class NeuroNet:
def __init__(self,n_inputs,n_hidden1,n_outputs):
hidden_layer = Layer(n_inputs,n_hidden1)
output_layer = Layer(n_hidden1,n_outputs)
self.layers = [hidden_layer,output_layer]
for layer in self.layers:
for param in layer.params:
param.attach_grad()
def softmax(self,x):
step1 = x.exp()
step2 = step1 / step1.sum(axis=1,keepdims=True)
return step2
def softmaxCrossEntropyLoss(self,y_pred,y):
step1 = -y * y_pred.log()
step2 = step1.sum(axis=1)
loss = step2.sum(axis=0) / len(y)
return loss
def drop(self,x,drop_probability,train=True):
'''
神经元被丢弃的概率为p
'''
if train:
mask = nd.random.uniform(0,1,shape=x.shape,dtype='float32') > drop_probability
return mask * x / (1 - drop_probability)
else:
return x
def forward(self,x,train=True):
for layer in self.layers[:-1]:
step1 = layer.forward(x)
step2 = self.drop(step1,0.2,train)
x = step2
output_layer = self.layers[-1]
return self.softmax(output_layer.forward(x))
def sgd(self,learning_rate,batch_size):
'''
使用随机梯度下降更新所有权重和偏置
'''
for layer in self.layers:
layer.sgd(learning_rate,batch_size)
def dataIter(self,x,y,batch_size):
dataset = gdata.ArrayDataset(x,y)
return gdata.DataLoader(dataset,batch_size,shuffle=True)
def fit(self,x,y,epoches,batch_size,learning_rate):
for epoch in range(epoches):
for x_batch,y_batch in self.dataIter(x,y,batch_size):
with autograd.record():
y_pred = self.forward(x_batch,train=True)
loss = self.softmaxCrossEntropyLoss(y_pred,y_batch)
loss.backward()
self.sgd(learning_rate,batch_size)
if epoch % 50 == 0:
y_pred_all = self.forward(x,train=False)
loss_all = self.softmaxCrossEntropyLoss(y_pred_all,y)
accuracy_score = self.accuracyScore(y_pred_all,y)
print('epoch:{},loss:{},accuracy:{}'.format(epoch+50,loss_all,accuracy_score))
def predict(self,x):
y_pred = self.forward(x)
return y_pred.argmax(axis=0)
def accuracyScore(self,y_pred,y):
acc_sum = (y_pred.argmax(axis=1) == y.argmax(axis=1)).sum().asscalar()
return acc_sum / len(y)
class Layer:
def __init__(self,n_inputs,n_outputs):
weight = nd.random.normal(scale=0.01,shape=(n_inputs,n_outputs))
bias = nd.zeros(shape=(n_outputs))
self.params = [weight,bias]
def relu(self,x):
return nd.maximum(x,0)
def forward(self,x):
step1 = nd.dot(x,self.params[0]) + self.params[1]
return self.relu(step1)
def sgd(self,learning_rate,batch_size):
for param in self.params:
param[:] = param - learning_rate * param.grad / batch_size
def print_params(self):
for param in self.params:
print(param)
net = NeuroNet(64,36,10)
net.fit(features,labels_onehot,epoches=500,batch_size=200,learning_rate=0.5)
epoch:50,loss:
[2.2988722]
<NDArray 1 @cpu(0)>,accuracy:0.18308291597106288
epoch:100,loss:
[1.4126126]
<NDArray 1 @cpu(0)>,accuracy:0.7395659432387313
epoch:150,loss:
[0.46316707]
<NDArray 1 @cpu(0)>,accuracy:0.9259877573734001
epoch:200,loss:
[0.24678323]
<NDArray 1 @cpu(0)>,accuracy:0.9493600445186422
epoch:250,loss:
[0.17839472]
<NDArray 1 @cpu(0)>,accuracy:0.9610461880912632
epoch:300,loss:
[0.14298467]
<NDArray 1 @cpu(0)>,accuracy:0.9688369504730105
epoch:350,loss:
[0.1198809]
<NDArray 1 @cpu(0)>,accuracy:0.9738452977184195
epoch:400,loss:
[0.10388324]
<NDArray 1 @cpu(0)>,accuracy:0.9782971619365609
epoch:450,loss:
[0.0917427]
<NDArray 1 @cpu(0)>,accuracy:0.9827490261547023
epoch:500,loss:
[0.08237094]
<NDArray 1 @cpu(0)>,accuracy:0.9849749582637729
print('预测结果：',net.predict(features[:10]))
print('真实结果：',labels[:10])
预测结果：
[0. 1. 2. 3. 4. 5. 6. 7. 8. 9.]
<NDArray 10 @cpu(0)>
真实结果：
[0. 1. 2. 3. 4. 5. 6. 7. 8. 9.]
<NDArray 10 @cpu(0)>
2.使用mxnet实现dropout
n_inputs = 64
n_hiddens = 36
n_outputs = 10
# 定义模型
net = nn.Sequential()
net.add(nn.Dense(n_hiddens,activation='relu'))
net.add(nn.Dropout(rate=0.2))
net.add(nn.Dense(n_outputs))
# 初始化模型
net.initialize(init.Normal(sigma=0.01))
# 损失函数
loss = gloss.SoftmaxCrossEntropyLoss(sparse_label=False)
optimizer = trainer.Trainer(net.collect_params(), 'sgd', {'learning_rate':0.5})
# 训练模型
epoches = 500
batch_size = 200
dataset = gdata.ArrayDataset(features,labels_onehot)
dataIter = gdata.DataLoader(dataset,batch_size,shuffle=True)
for epoch in range(epoches):
for x_batch,y_batch in dataIter:
with autograd.record():
y_pred = net.forward(x_batch)
l = loss(y_pred, y_batch).sum() / batch_size
l.backward()
optimizer.step(batch_size)
if epoch % 50 == 0:
y_all_pred = net.forward(features)
acc_sum = (y_all_pred.argmax(axis=1) == labels_onehot.argmax(axis=1)).sum().asscalar()
print('epoch:{},loss:{},accuracy:{}'.format(epoch+50,loss(y_all_pred,labels_onehot).sum() / len(labels_onehot),acc_sum/len(y_all_pred)))
epoch:50,loss:
[2.2981045]
<NDArray 1 @cpu(0)>,accuracy:0.16304952698942682
epoch:100,loss:
[0.97166663]
<NDArray 1 @cpu(0)>,accuracy:0.867557039510295
epoch:150,loss:
[0.3836201]
<NDArray 1 @cpu(0)>,accuracy:0.9243183082915971
epoch:200,loss:
[0.24329802]
<NDArray 1 @cpu(0)>,accuracy:0.9449081803005008
epoch:250,loss:
[0.18068495]
<NDArray 1 @cpu(0)>,accuracy:0.9577072899276572
epoch:300,loss:
[0.14546551]
<NDArray 1 @cpu(0)>,accuracy:0.9660545353366722
epoch:350,loss:
[0.1219953]
<NDArray 1 @cpu(0)>,accuracy:0.9727323316638843
epoch:400,loss:
[0.10563282]
<NDArray 1 @cpu(0)>,accuracy:0.9760712298274903
epoch:450,loss:
[0.09357208]
<NDArray 1 @cpu(0)>,accuracy:0.9788536449638287
epoch:500,loss:
[0.08368526]
<NDArray 1 @cpu(0)>,accuracy:0.9816360601001669
