Run layer normalization on the last dimension of the tensor.
(input_tensor, name=None, epsilon=1e-5)
| 112 | |
| 113 | |
| 114 | def layer_norm(input_tensor, name=None, epsilon=1e-5): |
| 115 | """Run layer normalization on the last dimension of the tensor.""" |
| 116 | name2use = f'LayerNorm_{name}' if name is not None else name |
| 117 | with tf.compat.v1.variable_scope(name2use, default_name='LayerNorm'): |
| 118 | dim = input_tensor.shape[-1].value |
| 119 | gamma = tf.compat.v1.get_variable('gamma', [dim], initializer=tf.constant_initializer(1)) |
| 120 | beta = tf.compat.v1.get_variable('beta', [dim], initializer=tf.constant_initializer(0)) |
| 121 | mean = tf.reduce_mean(input_tensor, axis=-1, keepdims=True) |
| 122 | std = tf.reduce_mean(tf.square(input_tensor - mean), axis=-1, keepdims=True) |
| 123 | input_tensor = (input_tensor - mean) * tf.math.rsqrt(std + epsilon) |
| 124 | input_tensor = input_tensor * gamma + beta |
| 125 | return input_tensor |
| 126 | |
| 127 | |
| 128 | def dropout(input_tensor, dropout_prob): |
no outgoing calls
no test coverage detected