Function evaluate

deeplabcut/benchmark/__init__.py:45–98

Run evaluation for all benchmarks and methods. Note that in order for your custom benchmark to be included during evaluation, the following conditions need to be met: - The benchmark subclassed one of the benchmark definitions in in ``benchmark.benchmarks`` - The

(
    include_benchmarks: Container[str] = None,
    results: ResultCollection = None,
    on_error="return",
)

def evaluate(
    include_benchmarks: Container[str] = None,
    results: ResultCollection = None,
    on_error="return",
) -> ResultCollection:
    """Run evaluation for all benchmarks and methods.

    Note that in order for your custom benchmark to be included during
    evaluation, the following conditions need to be met:

        - The benchmark subclassed one of the benchmark definitions in
          in ``benchmark.benchmarks``
        - The benchmark is registered by applying the ``@benchmark.register``
          decorator to the class
        - The benchmark was imported. This is done automatically for all
          benchmarks that are defined in submodules or subpackages of the
          ``benchmark.submissions`` module. For all other locations, make
          sure to manually import the packages **before** calling the
          ``evaluate()`` function.

    Args:
        include_benchmarks:
            If ``None``, run all benchmarks that were discovered. If a container
            is passed, only include methods that were defined on benchmarks with
            the specified names. E.g., ``include_benchmarks = ["trimouse"]`` would
            only evaluate methods of the trimouse benchmark dataset.
        on_error:
            see documentation in ``benchmark.base.Benchmark.evaluate()``

    Returns:
        A collection of all results, which can be printed or exported to
        ``pd.DataFrame`` or ``json`` file formats.
    """
    if results is None:
        results = ResultCollection()
    for benchmark_cls in __registry:
        if include_benchmarks is not None:
            if benchmark_cls.name not in include_benchmarks:
                continue
        benchmark = benchmark_cls()
        for name in benchmark.names():
            if (
                Result(
                    code=benchmark.code,
                    method_name=name,
                    benchmark_name=benchmark_cls.name,
                )
                in results
            ):
                continue
            else:
                result = benchmark.evaluate(name, on_error=on_error)
                results.add(result)
    return results


def get_filepath(basename: str):