Calculates and returns metrics.
(
references_path,
predictions_path,
is_regex,
answer_field = "answer")
| 95 | |
| 96 | |
| 97 | def evaluate_predictions( |
| 98 | references_path, |
| 99 | predictions_path, |
| 100 | is_regex, |
| 101 | answer_field = "answer"): |
| 102 | """Calculates and returns metrics.""" |
| 103 | if is_regex != ("CuratedTrec" in references_path): |
| 104 | print("Warning: regex evaluation should (only) be applied to CuratedTrec.") |
| 105 | |
| 106 | references = {} |
| 107 | with tf.io.gfile.GFile(references_path) as f: |
| 108 | for line in f: |
| 109 | example = json.loads(line) |
| 110 | references[example["question"]] = example[answer_field] |
| 111 | print("Found {} references in {}".format(len(references), references_path)) |
| 112 | |
| 113 | predictions = {} |
| 114 | with tf.io.gfile.GFile(predictions_path) as f: |
| 115 | for line in f: |
| 116 | example = json.loads(line) |
| 117 | predictions[example["question"]] = example["prediction"] |
| 118 | print("Found {} predictions in {}".format(len(predictions), predictions_path)) |
| 119 | |
| 120 | return evaluate_predictions_impl( |
| 121 | references=references, predictions=predictions, is_regex=is_regex) |
nothing calls this directly
no test coverage detected
searching dependent graphs…