Function batch_normalization

tflearn/layers/normalization.py:20–117

Batch Normalization. Normalize activations of the previous layer at each batch. Arguments: incoming: `Tensor`. Incoming Tensor. beta: `float`. Default: 0.0. gamma: `float`. Default: 1.0. epsilon: `float`. Defalut: 1e-5. decay: `float`. Default: 0.9.

(incoming, beta=0.0, gamma=1.0, epsilon=1e-5,
                        decay=0.9, stddev=0.002, trainable=True,
                        restore=True, reuse=False, scope=None,
                        name="BatchNormalization")

def batch_normalization(incoming, beta=0.0, gamma=1.0, epsilon=1e-5,
                        decay=0.9, stddev=0.002, trainable=True,
                        restore=True, reuse=False, scope=None,
                        name="BatchNormalization"):
    """ Batch Normalization.

    Normalize activations of the previous layer at each batch.

    Arguments:
        incoming: `Tensor`. Incoming Tensor.
        beta: `float`. Default: 0.0.
        gamma: `float`. Default: 1.0.
        epsilon: `float`. Defalut: 1e-5.
        decay: `float`. Default: 0.9.
        stddev: `float`. Standard deviation for weights initialization.
        trainable: `bool`. If True, weights will be trainable.
        restore: `bool`. If True, this layer weights will be restored when
            loading a model.
        reuse: `bool`. If True and 'scope' is provided, this layer variables
            will be reused (shared).
        scope: `str`. Define this layer scope (optional). A scope can be
            used to share variables between layers. Note that scope will
            override name.
        name: `str`. A name for this layer (optional).

    References:
        Batch Normalization: Accelerating Deep Network Training by Reducing
        Internal Covariate Shif. Sergey Ioffe, Christian Szegedy. 2015.

    Links:
        [http://arxiv.org/pdf/1502.03167v3.pdf](http://arxiv.org/pdf/1502.03167v3.pdf)

    """

    input_shape = utils.get_incoming_shape(incoming)
    input_ndim = len(input_shape)

    gamma_init = tf.random_normal_initializer(mean=gamma, stddev=stddev)

    with tf.variable_scope(scope, default_name=name, values=[incoming],
                           reuse=reuse) as scope:
        name = scope.name
        beta = vs.variable('beta', shape=[input_shape[-1]],
                           initializer=tf.constant_initializer(beta),
                           trainable=trainable, restore=restore)
        gamma = vs.variable('gamma', shape=[input_shape[-1]],
                            initializer=gamma_init, trainable=trainable,
                            restore=restore)
        # Track per layer variables
        tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, beta)
        tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, gamma)
        if not restore:
            tf.add_to_collection(tf.GraphKeys.EXCL_RESTORE_VARS, beta)
            tf.add_to_collection(tf.GraphKeys.EXCL_RESTORE_VARS, gamma)

        axis = list(range(input_ndim - 1))

        moving_mean = vs.variable('moving_mean', input_shape[-1:],