MCPcopy Create free account
hub / github.com/google-research/language / evaluate_predictions

Function evaluate_predictions

language/orqa/utils/eval_utils.py:97–121  ·  view source on GitHub ↗

Calculates and returns metrics.

(
    references_path,
    predictions_path,
    is_regex,
    answer_field = "answer")

Source from the content-addressed store, hash-verified

95
96
97def evaluate_predictions(
98 references_path,
99 predictions_path,
100 is_regex,
101 answer_field = "answer"):
102 """Calculates and returns metrics."""
103 if is_regex != ("CuratedTrec" in references_path):
104 print("Warning: regex evaluation should (only) be applied to CuratedTrec.")
105
106 references = {}
107 with tf.io.gfile.GFile(references_path) as f:
108 for line in f:
109 example = json.loads(line)
110 references[example["question"]] = example[answer_field]
111 print("Found {} references in {}".format(len(references), references_path))
112
113 predictions = {}
114 with tf.io.gfile.GFile(predictions_path) as f:
115 for line in f:
116 example = json.loads(line)
117 predictions[example["question"]] = example["prediction"]
118 print("Found {} predictions in {}".format(len(predictions), predictions_path))
119
120 return evaluate_predictions_impl(
121 references=references, predictions=predictions, is_regex=is_regex)

Callers

nothing calls this directly

Calls 1

Tested by

no test coverage detected

Used in the wild real call sites across dependent graphs

searching dependent graphs…