Class EvaluationModule

src/evaluate/module.py:149–779

A `EvaluationModule` is the base class and common API for metrics, comparisons, and measurements. Args: config_name (`str`): This is used to define a hash specific to a module computation script and prevents the module's data to be overridden when the module load

class EvaluationModule(EvaluationModuleInfoMixin):
    """A `EvaluationModule` is the base class and common API for metrics, comparisons, and measurements.

    Args:
        config_name (`str`):
            This is used to define a hash specific to a module computation script and prevents the module's data
            to be overridden when the module loading script is modified.
        keep_in_memory (`bool`):
            Keep all predictions and references in memory. Not possible in distributed settings.
        cache_dir (`str`):
            Path to a directory in which temporary prediction/references data will be stored.
            The data directory should be located on a shared file-system in distributed setups.
        num_process (`int`):
            Specify the total number of nodes in a distributed settings.
            This is useful to compute module in distributed setups (in particular non-additive modules like F1).
        process_id (`int`):
            Specify the id of the current process in a distributed setup (between 0 and num_process-1)
            This is useful to compute module in distributed setups (in particular non-additive metrics like F1).
        seed (`int`, optional):
            If specified, this will temporarily set numpy's random seed when [`~evaluate.EvaluationModule.compute`] is run.
        experiment_id (`str`):
            A specific experiment id. This is used if several distributed evaluations share the same file system.
            This is useful to compute module in distributed setups (in particular non-additive metrics like F1).
        hash (`str`):
            Used to identify the evaluation module according to the hashed file contents.
        max_concurrent_cache_files (`int`):
            Max number of concurrent module cache files (default `10000`).
        timeout (`Union[int, float]`):
            Timeout in second for distributed setting synchronization.
    """

    def __init__(
        self,
        config_name: Optional[str] = None,
        keep_in_memory: bool = False,
        cache_dir: Optional[str] = None,
        num_process: int = 1,
        process_id: int = 0,
        seed: Optional[int] = None,
        experiment_id: Optional[str] = None,
        hash: str = None,
        max_concurrent_cache_files: int = 10000,
        timeout: Union[int, float] = 100,
        **kwargs,
    ):
        # prepare info
        self.config_name = config_name or "default"
        info = self._info()
        info.module_name = camelcase_to_snakecase(self.__class__.__name__)
        info.config_name = self.config_name
        info.experiment_id = experiment_id or "default_experiment"
        EvaluationModuleInfoMixin.__init__(self, info)  # For easy access on low level

        # Safety checks on num_process and process_id
        if not isinstance(process_id, int) or process_id < 0:
            raise ValueError("'process_id' should be a number greater than 0")
        if not isinstance(num_process, int) or num_process <= process_id:
            raise ValueError("'num_process' should be a number greater than process_id")