Function evaluate_predictions

utils/evaluate/utils/eval_utils.py:109–136

Calculates and returns metrics.

(
        references_path,
        predictions_path,
        is_regex,
        answer_field="answer")

def evaluate_predictions(
        references_path,
        predictions_path,
        is_regex,
        answer_field="answer"):
    """Calculates and returns metrics."""
    if is_regex != ("CuratedTrec" in references_path):
        print("Warning: regex evaluation should (only) be applied to CuratedTrec.")

    references = {}
    with open(references_path, 'r', encoding='utf-8') as f:
        for line in f:
            example = json.loads(line)
            references[example["question"]] = example[answer_field]
    print("Found {} references in {}".format(len(references), references_path))

    predictions = {}
    with open(predictions_path, 'r', encoding='utf-8') as f:
        for line in f:
            example = json.loads(line)
            predictions[example["question"]] = example["prediction"]
            # predictions[example["question"]] = find_last_uppercase_abcd(example["fuse_generation"])
            # predictions[example["question"]] = example["fuse_generation"]
            # predictions[example["question"]] = example["best"]
    print("Found {} predictions in {}".format(len(predictions), predictions_path))

    return evaluate_predictions_impl(
        references=references, predictions=predictions, is_regex=is_regex)