(x)
| 55 | # Apply bias to the given activation tensor. |
| 56 | |
| 57 | def apply_bias(x): |
| 58 | b = tf.get_variable('bias', shape=[x.shape[1]], initializer=tf.initializers.zeros()) |
| 59 | b = tf.cast(b, x.dtype) |
| 60 | if len(x.shape) == 2: |
| 61 | return x + b |
| 62 | return x + tf.reshape(b, [1, -1, 1, 1]) |
| 63 | |
| 64 | #---------------------------------------------------------------------------- |
| 65 | # Leaky ReLU activation. Same as tf.nn.leaky_relu, but supports FP16. |