用java写bp神经网络（二）

接上篇。

Net和Propagation具备后，我们就可以训练了。训练师要做的事情就是，怎么把一大批样本分成小批训练，然后把小批的结果合并成完整的结果（批量/增量）；什么时候调用学习师根据训练的结果进行学习，然后改进网络的权重和状态；什么时候决定训练结束。

那么这两位老师儿长的什么样子，又是怎么做到的呢？

public interface Trainer {
    public void train(Net net,DataProvider provider);
}

public interface Learner {
    public void learn(Net net,TrainResult trainResult);
}

 所谓Trainer即是给定数据，对指定网络进行训练；所谓Learner即是给定训练结果，然后对指定网络进行权重调整。

下面给出这两个接口的简单实现。

Trainer

Trainer实现简单的批量训练功能，在给定的迭代次数后停止。代码示例如下。

public class CommonTrainer implements Trainer {
	int ecophs;
	Learner learner;
	List<Double> costs = new ArrayList<>();
	List<Double> accuracys = new ArrayList<>();
	int batchSize = 1;

	public CommonTrainer(int ecophs, Learner learner) {
		super();
		this.ecophs = ecophs;
		this.learner = learner == null ? new MomentAdaptLearner() : learner;
	}

	public CommonTrainer(int ecophs, Learner learner, int batchSize) {
		this(ecophs, learner);
		this.batchSize = batchSize;
	}

	public void trainOne(final Net net, DataProvider provider) {
		final Propagation propagation = new Propagation(net);
		DoubleMatrix input = provider.getInput();
		DoubleMatrix target = provider.getTarget();
		final int allLen = target.columns;
		final int[] nodesNum = net.getNodesNum();
		final int layersNum = net.getLayersNum();

		List<DoubleMatrix> inputBatches = this.getBatches(input);
		final List<DoubleMatrix> targetBatches = this.getBatches(target);

		final List<Integer> batchLen = MatrixUtil.getEndPosition(targetBatches);

		final BackwardResult backwardResult = new BackwardResult(net, allLen);

　　　　　　　　　 // 分批并行训练
		Parallel.For(inputBatches, new Parallel.Operation<DoubleMatrix>() {
			@Override
			public void perform(int index, DoubleMatrix subInput) {
				ForwardResult subResult = propagation.forward(subInput);
				DoubleMatrix subTarget = targetBatches.get(index);
				BackwardResult backResult = propagation.backward(subTarget,
						subResult);

				DoubleMatrix cost = backwardResult.cost;
				DoubleMatrix accuracy = backwardResult.accuracy;
				DoubleMatrix inputDeltas = backwardResult.getInputDelta();

				int start = index == 0 ? 0 : batchLen.get(index - 1);
				int end = batchLen.get(index) - 1;
				int[] cIndexs = ArraysHelper.makeArray(start, end);

				cost.put(cIndexs, backResult.cost);

				if (accuracy != null) {
					accuracy.put(cIndexs, backResult.accuracy);
				}

				inputDeltas.put(ArraysHelper.makeArray(0, nodesNum[0] - 1),
						　　cIndexs, backResult.getInputDelta());

				for (int i = 0; i < layersNum; i++) {
					DoubleMatrix gradients = backwardResult.gradients.get(i);
					DoubleMatrix biasGradients = backwardResult.biasGradients
							.get(i);

　　					DoubleMatrix subGradients = backResult.gradients.get(i)
							.muli(backResult.cost.columns);
					DoubleMatrix subBiasGradients = backResult.biasGradients
							.get(i).muli(backResult.cost.columns);
					gradients.addi(subGradients);
					biasGradients.addi(subBiasGradients);
				}
			}
		});
　　　　　　　　　// 求均值
		for(DoubleMatrix gradient:backwardResult.gradients){
			gradient.divi(allLen);
		}
		for(DoubleMatrix gradient:backwardResult.biasGradients){
			gradient.divi(allLen);
		}
		
		// this.mergeBackwardResult(backResults, net, input.columns);
		TrainResult trainResult = new TrainResult(null, backwardResult);

		learner.learn(net, trainResult);

		Double cost = backwardResult.getMeanCost();
		Double accuracy = backwardResult.getMeanAccuracy();
		if (cost != null)
			costs.add(cost);
		if (accuracy != null)
			accuracys.add(accuracy);

　　		System.out.println(cost);
		System.out.println(accuracy);
	}

	@Override
	public void train(Net net, DataProvider provider) {
		for (int i = 0; i < this.ecophs; i++) {
			this.trainOne(net, provider);
		}

	}
}

 

Learner

Learner是具体的调整算法，当梯度计算出来后，它负责对网络权重进行调整。调整算法的选择直接影响着网络收敛的快慢。本文的实现采用简单的动量-自适应学习率算法。

其迭代公式如下：

$$W(t+1)=W(t)+\Delta W(t)$$

$$\Delta W(t)=rate(t)(1-moment(t))G(t)+moment(t)\Delta W(t-1)$$

$$rate(t+1)=\begin{cases} rate(t)\times 1.05 & \mbox{if } cost(t)<cost(t-1)\\ rate(t)\times 0.7 & \mbox{else if } cost(t)<cost(t-1)\times 1.04\\ 0.01 & \mbox{else} \end{cases}$$

$$moment(t+1)=\begin{cases} 0.9 & \mbox{if } cost(t)<cost(t-1)\\ moment(t)\times 0.7 & \mbox{else if } cost(t)<cost(t-1)\times 1.04\\ 1-0.9 & \mbox{else} \end{cases}$$

示例代码如下：

public class MomentAdaptLearner implements Learner {

	Net net;
	double moment = 0.9;
	double lmd = 1.05;
	double preCost = 0;
	double eta = 0.01;
	double currentEta=eta;
	double currentMoment=moment;
	TrainResult preTrainResult;
	
	public MomentAdaptLearner(double moment, double eta) {
		super();
		this.moment = moment;
		this.eta = eta;
		this.currentEta=eta;
		this.currentMoment=moment;
	}

	@Override
	public void learn(Net net, TrainResult trainResult) {
		if (this.net == null)
			init(net);
		
		BackwardResult backwardResult = trainResult.backwardResult;
		BackwardResult preBackwardResult = preTrainResult.backwardResult;
		double cost=backwardResult.getMeanCost();
		this.modifyParameter(cost);
		System.out.println("current eta:"+this.currentEta);
		System.out.println("current moment:"+this.currentMoment);
		for (int j = 0; j < net.getLayersNum(); j++) {
			DoubleMatrix weight = net.getWeights().get(j);
			DoubleMatrix gradient = backwardResult.gradients.get(j);

			gradient = gradient.muli(currentEta * (1 - this.currentMoment)).addi(
					preBackwardResult.gradients.get(j).muli(this.currentMoment));
			preBackwardResult.gradients.set(j, gradient);

			weight.subi(gradient);
			
			DoubleMatrix b = net.getBs().get(j);
			DoubleMatrix bgradient = backwardResult.biasGradients.get(j);

			bgradient = bgradient.muli(currentEta * (1 - this.currentMoment)).addi(
					preBackwardResult.biasGradients.get(j).muli(this.currentMoment));
			preBackwardResult.biasGradients.set(j, bgradient);

			b.subi(bgradient);
		}

	}

	public void modifyParameter(double cost){
		if(cost<this.preCost){
			this.currentEta*=1.05;
			this.currentMoment=moment;
		}else if(cost<1.04*this.preCost){
			this.currentEta*=0.7;
			this.currentMoment*=0.7;
		}else{
			this.currentEta=eta;
			this.currentMoment=1-moment;
		}
		this.preCost=cost;
	}
	public void init(Net net) {
		this.net =  net;
		BackwardResult bResult = new BackwardResult();

		for (DoubleMatrix weight : net.getWeights()) {
			bResult.gradients.add(DoubleMatrix.zeros(weight.rows,
					weight.columns));
		}
		for (DoubleMatrix b : net.getBs()) {
			bResult.biasGradients.add(DoubleMatrix.zeros(b.rows, b.columns));
		}
		preTrainResult=new TrainResult(null,bResult);
	}


}

现在，一个简单的神经网路从生成到训练已经简单实现完毕。

下一步，使用Levenberg-Marquardt学习算法改进收敛速率。