Method apply_gradients

tensorflow/python/training/optimizer.py:619–746

Apply gradients to variables. This is the second part of `minimize()`. It returns an `Operation` that applies gradients. Args: grads_and_vars: List of (gradient, variable) pairs as returned by `compute_gradients()`. global_step: Optional `Variable` to increment by o

(self, grads_and_vars, global_step=None, name=None)

    return grad * loss_scale_reciprical

  def apply_gradients(self, grads_and_vars, global_step=None, name=None):
    """Apply gradients to variables.

    This is the second part of `minimize()`. It returns an `Operation` that
    applies gradients.

    Args:
      grads_and_vars: List of (gradient, variable) pairs as returned by
        `compute_gradients()`.
      global_step: Optional `Variable` to increment by one after the
        variables have been updated.
      name: Optional name for the returned operation.  Default to the
        name passed to the `Optimizer` constructor.

    Returns:
      An `Operation` that applies the specified gradients. If `global_step`
      was not None, that operation also increments `global_step`.

    Raises:
      TypeError: If `grads_and_vars` is malformed.
      ValueError: If none of the variables have gradients.
      RuntimeError: If you should use `_distributed_apply()` instead.
    """
    # This is a default implementation of apply_gradients() that can be shared
    # by most optimizers.  It relies on the subclass implementing the following
    # methods: _create_slots(), _prepare(), _apply_dense(), and _apply_sparse().

    # TODO(isaprykin): Get rid of `has_strategy()` check by
    # always calling _distributed_apply(), using the default distribution
    # as needed.
    if distribute_ctx.has_strategy():
      # Handle DistributionStrategy case.
      if distribute_ctx.in_cross_replica_context():
        raise RuntimeError("Use `_distributed_apply()` instead of "
                           "`apply_gradients()` in a cross-replica context.")

      grads_and_vars = get_filtered_grad_fn(lambda: grads_and_vars)()
      return distribute_ctx.get_replica_context().merge_call(
          self._distributed_apply, args=(grads_and_vars, global_step, name))

    name = name if name is not None else self.get_name()
    grads_and_vars = tuple(grads_and_vars)  # Make sure repeat iteration works.
    def apply_fn():
      # No DistributionStrategy case.
      if not grads_and_vars:
        raise ValueError("No variables provided.")
      converted_grads_and_vars = []
      for g, v in grads_and_vars:
        if g is not None:
          try:
            # Convert the grad to Tensor or IndexedSlices if necessary.
            g = ops.convert_to_tensor_or_indexed_slices(g)
          except TypeError:
            raise TypeError(
                "Gradient must be convertible to a Tensor"
                " or IndexedSlices, or None: %s" % g)
          if not isinstance(g, (ops.Tensor, ops.IndexedSlices)):
            raise TypeError(