| 17 | |
| 18 | |
| 19 | class HiddenLayerBatchNorm(object): |
| 20 | def __init__(self, M1, M2, f): |
| 21 | self.M1 = M1 |
| 22 | self.M2 = M2 |
| 23 | self.f = f |
| 24 | |
| 25 | W = init_weight(M1, M2).astype(np.float32) |
| 26 | gamma = np.ones(M2).astype(np.float32) |
| 27 | beta = np.zeros(M2).astype(np.float32) |
| 28 | |
| 29 | self.W = tf.Variable(W) |
| 30 | self.gamma = tf.Variable(gamma) |
| 31 | self.beta = tf.Variable(beta) |
| 32 | |
| 33 | # for test time |
| 34 | self.running_mean = tf.Variable(np.zeros(M2).astype(np.float32), trainable=False) |
| 35 | self.running_var = tf.Variable(np.zeros(M2).astype(np.float32), trainable=False) |
| 36 | |
| 37 | def forward(self, X, is_training, decay=0.9): |
| 38 | activation = tf.matmul(X, self.W) |
| 39 | if is_training: |
| 40 | batch_mean, batch_var = tf.nn.moments(activation, [0]) |
| 41 | update_running_mean = tf.assign( |
| 42 | self.running_mean, |
| 43 | self.running_mean * decay + batch_mean * (1 - decay) |
| 44 | ) |
| 45 | update_running_var = tf.assign( |
| 46 | self.running_var, |
| 47 | self.running_var * decay + batch_var * (1 - decay) |
| 48 | ) |
| 49 | |
| 50 | with tf.control_dependencies([update_running_mean, update_running_var]): |
| 51 | out = tf.nn.batch_normalization( |
| 52 | activation, |
| 53 | batch_mean, |
| 54 | batch_var, |
| 55 | self.beta, |
| 56 | self.gamma, |
| 57 | 1e-4 |
| 58 | ) |
| 59 | else: |
| 60 | out = tf.nn.batch_normalization( |
| 61 | activation, |
| 62 | self.running_mean, |
| 63 | self.running_var, |
| 64 | self.beta, |
| 65 | self.gamma, |
| 66 | 1e-4 |
| 67 | ) |
| 68 | return self.f(out) |
| 69 | |
| 70 | |
| 71 | class HiddenLayer(object): |