Class TaskManager

lm_eval/tasks/manager.py:37–367

Central entry point for discovering and loading evaluation tasks. On construction, scans one or more directories for YAML task configs and builds an in-memory index of every known task, group, and tag. Callers then use :meth:`load` to instantiate tasks by name, glob pattern, file p

class TaskManager:
    """Central entry point for discovering and loading evaluation tasks.

    On construction, scans one or more directories for YAML task configs and
    builds an in-memory index of every known task, group, and tag.  Callers
    then use :meth:`load` to instantiate tasks by name, glob pattern, file
    path, or inline config dict.

    Example::

        tm = TaskManager(include_path="my_tasks/")
        result = tm.load(["mmlu", "hellaswag"])
        result["tasks"]   # {name: Task, ...}
        result["groups"]  # {name: Group, ...}
    """

    def __init__(
        self,
        verbosity: str | None = None,
        include_path: str | Path | list[str | Path] | None = None,
        include_defaults: bool = True,
        metadata: dict[str, dict[str, Any] | str] | None = None,
    ) -> None:
        """
        Args:
            verbosity: Logging level (e.g., "INFO", "DEBUG") (deprecated, use standard logging configuration instead)
            include_path: Custom paths to scan for task configs (takes precedence)
            include_defaults: Whether to include built-in tasks from lm_eval/tasks/
            metadata: Extra metadata to attach to all loaded tasks
        """
        if verbosity:
            warnings.warn(
                "The `verbosity` argument is deprecated. Use logging configuration instead.",
                DeprecationWarning,
                stacklevel=2,
            )

        self.include_path = include_path
        self.metadata = metadata

        index = TaskIndex()
        self._factory: TaskFactory = TaskFactory(meta=metadata)

        all_paths: list[Path] = []
        # Process defaults FIRST, then include_path (later paths can override earlier)
        if include_defaults:
            all_paths.append(Path(__file__).parent)
        if include_path:
            all_paths += [
                Path(p)
                for p in (
                    include_path
                    if isinstance(include_path, (list, tuple))
                    else [include_path]
                )
            ]

        self._index = index.build(all_paths)