Dive into DL TF2 --BatchNormalization
1 import numpy as np 2 import tensorflow as tf 3 4 class BatchNormalization(tf.keras.layers.Layer): 5 def __init__(self, decay=0.9, epsilon=1e-5, **kwargs): 6 self.decay = decay 7 self.epsilon = epsilon 8 super(BatchNormalization, self).__init__(**kwargs) 9 10 def build(self, input_shape): 11 self.gamma = self.add_weight(name='game', 12 shape = [input_shape[-1],], 13 initializer = tf.initializers.ones, 14 trainable = True) 15 self.beta = self.add_weight(name='beta', 16 shape=[input_shape[-1], ], 17 initializer = tf.initializers.zeros, 18 trainable=True) 19 self.moving_mean = self.add_weight(name='moving_mean', 20 shape=[input_shape[-1],], 21 initializer=tf.initializers.zeros, 22 trainable=False) 23 self.moving_variance = self.add_weight(name='moving_variance', 24 shape=[input_shape[-1],], 25 initializer=tf.initializers.ones, 26 trainable=False) 27 super(BatchNormalization, self).build(input_shape) 28 29 def assign_moving_average(self, variable, value): 30 """variable = variable * decay + value * (1 - decay)""" 31 delta = variable * self.decay + value * (1 - self.decay) 32 return variable.assign(delta) 33 34 @tf.function 35 def call(self, inputs, training): 36 if training: 37 batch_mean, batch_variance = tf.nn.moments(inputs, list(range(len(inputs.shape) - 1))) 38 mean_update = self.assign_moving_average(self.moving_mean, batch_mean) 39 variance_update = self.assign_moving_average(self.moving_variance, batch_variance) 40 self.add_update(mean_update) 41 self.add_update(variance_update) 42 mean, variance = batch_mean, batch_variance 43 else: 44 mean, variance = self.moving_mean, self.moving_variance 45 output = tf.nn.batch_normalization(inputs, 46 mean=mean, 47 variance=variance, 48 offset=self.beta, 49 scale=self.gamma, 50 variance_epsilon=self.epsilon) 51 return output 52 53 def compute_output_shape(self, input_shape): 54 return input_shape 55 56 57 #使用批量归一化层的LeNet 58 net = tf.keras.models.Sequential( 59 [tf.keras.layers.Conv2D(filters=6, kernel_size=5), 60 BatchNormalization(), 61 tf.keras.layers.Activation('sigmoid'), 62 tf.keras.layers.MaxPool2D(pool_size=2, strides=2), 63 tf.keras.layers.Conv2D(filters=16, kernel_size=5), 64 BatchNormalization(), 65 tf.keras.layers.Activation('sigmoid'), 66 tf.keras.layers.MaxPool2D(pool_size=2, strides=2), 67 tf.keras.layers.Flatten(), 68 tf.keras.layers.Dense(120), 69 BatchNormalization(), 70 tf.keras.layers.Activation('sigmoid'), 71 tf.keras.layers.Dense(84), 72 BatchNormalization(), 73 tf.keras.layers.Activation('sigmoid'), 74 tf.keras.layers.Dense(10, activation='softmax')] 75 ) 76 77 78 #训练修改后的模型 79 (x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data() 80 x_train = x_train.reshape((60000, 28, 28, 1)).astype('float32')/255 81 x_test = x_test.reshape((10000, 28, 28, 1)).astype('float32')/255 82 83 net.compile(loss='sparse_categorical_crossentropy', 84 optimizer=tf.keras.optimizers.RMSprop(), 85 metrics=['accuracy']) 86 87 history = net.fit(x_train, y_train, batch_size=64, epochs=5, validation_split=0.2) 88 89 test_scores = net.evaluate(x_test, y_test, verbose=2) 90 print('Test loss:', test_scores[0]) 91 print('Test accuracy:', test_scores[1]) 92 93 94 #第一个批量归一化层学习到的拉伸参数gamma和偏移参数beta 95 net.get_layer(index=1).gamma, net.get_layer(index=1).beta 96 97 98 #用keras实现使用批量归一化的LeNet 99 100 net = tf.keras.models.Sequential() 101 net.add(tf.keras.layers.Conv2D(filters=6, kernel_size=5)) 102 net.add(tf.keras.layers.BatchNormalization()) 103 net.add(tf.keras.layers.Activation('sigmoid')) 104 net.add(tf.keras.layers.MaxPool2D(pool_size=2, strides=2)) 105 net.add(tf.keras.layers.Conv2D(filters=16, kernel_size=5)) 106 net.add(tf.keras.layers.BatchNormalization()) 107 net.add(tf.keras.layers.Activation('sigmoid')) 108 net.add(tf.keras.layers.MaxPool2D(pool_size=2, strides=2)) 109 net.add(tf.keras.layers.Flatten()) 110 net.add(tf.keras.layers.Dense(120)) 111 net.add(tf.keras.layers.BatchNormalization()) 112 net.add(tf.keras.layers.Activation('sigmoid')) 113 net.add(tf.keras.layers.Dense(84)) 114 net.add(tf.keras.layers.BatchNormalization()) 115 net.add(tf.keras.layers.Activation('sigmoid')) 116 net.add(tf.keras.layers.Dense(10, activation='sigmoid')) 117 118 119 #训练修改后的模型 120 (x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data() 121 x_train = x_train.reshape((60000, 28, 28, 1)).astype('float32')/255 122 x_test = x_test.reshape((10000, 28, 28, 1)).astype('float32')/255 123 124 net.compile(loss='sparse_categorical_crossentropy', 125 optimizer=tf.keras.optimizers.RMSprop(), 126 metrics=['accuracy']) 127 128 history = net.fit(x_train, y_train, batch_size=64, epochs=5, validation_split=0.2) 129 130 test_scores = net.evaluate(x_test, y_test, verbose=2) 131 print('Test loss:', test_scores[0]) 132 print('Test accuracy:', test_scores[1])