Method evaluate_continuously

orbit/controller.py:400–439

Continuously monitors a directory and evaluates new checkpoints in it. This method continuously monitors a directory as specified by this Controller's CheckpointManager init arg and runs evaluation on the checkpoints found there. Args: steps: The number of steps to run when e

(
      self,
      steps: int = -1,
      timeout: Optional[Union[int, float]] = None,
      timeout_fn: Optional[Callable[[], bool]] = None,
  )

    return output

  def evaluate_continuously(
      self,
      steps: int = -1,
      timeout: Optional[Union[int, float]] = None,
      timeout_fn: Optional[Callable[[], bool]] = None,
  ) -> Optional[runner.Output]:
    """Continuously monitors a directory and evaluates new checkpoints in it.

    This method continuously monitors a directory as specified by this
    Controller's CheckpointManager init arg and runs evaluation on the
    checkpoints found there.

    Args:
      steps: The number of steps to run when evaluating. If -1, this method will
        evaluate over the entire evaluation dataset.
      timeout: The maximum number of seconds to wait between checkpoints. See
        tf.train.checkpoints_iterator documentation.
      timeout_fn: Optional callable to call after a timeout. If the function
        returns True, then it means that no new checkpoints will be generated
        and the iterator will exit.

    Returns:
      The evaluation results as a dictionary mapping names to NumPy values.

    Raises:
      ValueError: If no checkpoint found in `self.checkpoint_manager.directory`.
      ValueError: If `evaluator` was not provided as a controller init arg.
    """
    self._require("evaluator", for_method="evaluate_continuously")
    self._require("checkpoint_manager", for_method="evaluate_continuously")

    output = None
    assert isinstance(self.checkpoint_manager, tf.train.CheckpointManager)
    for checkpoint_path in tf.train.checkpoints_iterator(
        self.checkpoint_manager.directory,
        timeout=timeout,
        timeout_fn=timeout_fn):
      self.restore_checkpoint(checkpoint_path)
      output = self.evaluate(steps)
    return output

  def restore_checkpoint(self, checkpoint_path: Optional[str] = None):
    """Restores the model from a checkpoint.