Method evaluation_loop

trainer/trainer.py:793–900

Prediction/evaluation loop, shared by :obj:`Trainer.evaluate()` and :obj:`Trainer.predict()`. Works both with or without labels.

(
            self,
            dataloader: DataLoader,
            description: str,
            metric_key_prefix: str = "eval",
    )

        return images, weighted_loss, losses

    def evaluation_loop(
            self,
            dataloader: DataLoader,
            description: str,
            metric_key_prefix: str = "eval",
    ) -> Tuple[Dict[str, float], int]:
        """
        Prediction/evaluation loop, shared by :obj:`Trainer.evaluate()` and :obj:`Trainer.predict()`.

        Works both with or without labels.
        """

        batch_size = dataloader.batch_size

        logger.info(f"***** Running {description} *****")
        if isinstance(dataloader.dataset, collections.abc.Sized):
            logger.info(f"  Num examples = {len(dataloader.dataset)}")
        else:
            logger.info("  Num examples: Unknown")
        logger.info(f"  Batch size = {batch_size}")

        self.model.eval()

        # Do this before wrapping.
        eval_dataset = dataloader.dataset

        # Initialize containers
        # losses/preds/labels on GPU/TPU (accumulated for eval_accumulation_steps)
        prediction_outputs_host = None
        # losses/preds/labels on CPU (final containers)
        all_prediction_outputs = None
        # Will be useful when we have an iterable dataset so don't know its length.

        # Main evaluation loop
        for step, inputs in tqdm(enumerate(dataloader)):
            # Prediction step
            prediction_outputs = self.prediction_step(inputs)

            # Update containers on host
            if prediction_outputs is not None:
                prediction_outputs = distributed_concat(prediction_outputs)
            prediction_outputs_host = (
                prediction_outputs if prediction_outputs_host is None else
                nested_concat(prediction_outputs_host, prediction_outputs, padding_index=-100)
            )

            # Gather all tensors and put them back on the CPU if we have done enough accumulation steps.
            if self.args.eval_accumulation_steps is not None and (step + 1) % self.args.eval_accumulation_steps == 0:
                if prediction_outputs_host is not None:
                    prediction_outputs = nested_cpu(prediction_outputs_host)
                    all_prediction_outputs = (
                        prediction_outputs if all_prediction_outputs is None else
                        nested_concat(all_prediction_outputs, prediction_outputs, padding_index=-100)
                    )

                # Set back to None to begin a new accumulation
                prediction_outputs_host = None