Class LocalModuleTest

tests/test_metric_common.py:97–161

@for_all_test_methods(skip_if_metric_requires_fairseq, skip_on_windows_if_not_windows_compatible, skip_slow_metrics)
@local
class LocalModuleTest(parameterized.TestCase):
    INTENSIVE_CALLS_PATCHER = {}
    evaluation_module_name = None
    evaluation_module_type = None

    def test_load(self, evaluation_module_name, evaluation_module_type):
        doctest.ELLIPSIS_MARKER = "[...]"
        evaluation_module = importlib.import_module(
            evaluate.loading.evaluation_module_factory(
                os.path.join(evaluation_module_type + "s", evaluation_module_name), module_type=evaluation_module_type
            ).module_path
        )
        evaluation_instance = evaluate.loading.import_main_class(evaluation_module.__name__)
        # check parameters
        parameters = inspect.signature(evaluation_instance._compute).parameters
        self.assertTrue(all([p.kind != p.VAR_KEYWORD for p in parameters.values()]))  # no **kwargs
        # run doctest
        with self.patch_intensive_calls(evaluation_module_name, evaluation_module.__name__):
            with self.use_local_metrics(evaluation_module_type):
                try:
                    results = doctest.testmod(evaluation_module, verbose=True, raise_on_error=True)
                except doctest.UnexpectedException as e:
                    raise e.exc_info[1]  # raise the exception that doctest caught
        self.assertEqual(results.failed, 0)
        self.assertGreater(results.attempted, 1)

    @slow
    def test_load_real_metric(self, evaluation_module_name, evaluation_module_type):
        doctest.ELLIPSIS_MARKER = "[...]"
        metric_module = importlib.import_module(
            evaluate.loading.evaluation_module_factory(
                os.path.join(evaluation_module_type, evaluation_module_name)
            ).module_path
        )
        # run doctest
        with self.use_local_metrics():
            results = doctest.testmod(metric_module, verbose=True, raise_on_error=True)
        self.assertEqual(results.failed, 0)
        self.assertGreater(results.attempted, 1)

    @contextmanager
    def patch_intensive_calls(self, evaluation_module_name, module_name):
        if evaluation_module_name in self.INTENSIVE_CALLS_PATCHER:
            with self.INTENSIVE_CALLS_PATCHER[evaluation_module_name](module_name):
                yield
        else:
            yield

    @contextmanager
    def use_local_metrics(self, evaluation_module_type):
        def load_local_metric(evaluation_module_name, *args, **kwargs):
            return load(os.path.join(evaluation_module_type + "s", evaluation_module_name), *args, **kwargs)

        with patch("evaluate.load") as mock_load:
            mock_load.side_effect = load_local_metric
            yield

    @classmethod