(xs, ys, norm)
| 51 | |
| 52 | |
| 53 | def built_net(xs, ys, norm): |
| 54 | def add_layer(inputs, in_size, out_size, activation_function=None, norm=False): |
| 55 | # weights and biases (bad initialization for this case) |
| 56 | Weights = tf.Variable(tf.random_normal([in_size, out_size], mean=0., stddev=1.)) |
| 57 | biases = tf.Variable(tf.zeros([1, out_size]) + 0.1) |
| 58 | |
| 59 | # fully connected product |
| 60 | Wx_plus_b = tf.matmul(inputs, Weights) + biases |
| 61 | |
| 62 | # normalize fully connected product |
| 63 | if norm: |
| 64 | # Batch Normalize |
| 65 | fc_mean, fc_var = tf.nn.moments( |
| 66 | Wx_plus_b, |
| 67 | axes=[0], # the dimension you wanna normalize, here [0] for batch |
| 68 | # for image, you wanna do [0, 1, 2] for [batch, height, width] but not channel |
| 69 | ) |
| 70 | scale = tf.Variable(tf.ones([out_size])) |
| 71 | shift = tf.Variable(tf.zeros([out_size])) |
| 72 | epsilon = 0.001 |
| 73 | |
| 74 | # apply moving average for mean and var when train on batch |
| 75 | ema = tf.train.ExponentialMovingAverage(decay=0.5) |
| 76 | def mean_var_with_update(): |
| 77 | ema_apply_op = ema.apply([fc_mean, fc_var]) |
| 78 | with tf.control_dependencies([ema_apply_op]): |
| 79 | return tf.identity(fc_mean), tf.identity(fc_var) |
| 80 | mean, var = mean_var_with_update() |
| 81 | |
| 82 | Wx_plus_b = tf.nn.batch_normalization(Wx_plus_b, mean, var, shift, scale, epsilon) |
| 83 | # similar with this two steps: |
| 84 | # Wx_plus_b = (Wx_plus_b - fc_mean) / tf.sqrt(fc_var + 0.001) |
| 85 | # Wx_plus_b = Wx_plus_b * scale + shift |
| 86 | |
| 87 | # activation |
| 88 | if activation_function is None: |
| 89 | outputs = Wx_plus_b |
| 90 | else: |
| 91 | outputs = activation_function(Wx_plus_b) |
| 92 | |
| 93 | return outputs |
| 94 | |
| 95 | fix_seed(1) |
| 96 | |
| 97 | if norm: |
| 98 | # BN for the first input |
| 99 | fc_mean, fc_var = tf.nn.moments( |
| 100 | xs, |
| 101 | axes=[0], |
| 102 | ) |
| 103 | scale = tf.Variable(tf.ones([1])) |
| 104 | shift = tf.Variable(tf.zeros([1])) |
| 105 | epsilon = 0.001 |
| 106 | # apply moving average for mean and var when train on batch |
| 107 | ema = tf.train.ExponentialMovingAverage(decay=0.5) |
| 108 | def mean_var_with_update(): |
| 109 | ema_apply_op = ema.apply([fc_mean, fc_var]) |
| 110 | with tf.control_dependencies([ema_apply_op]): |
no test coverage detected