| 106 | "rouge-l": functools.partial(rouge_metric, metric="rouge-l", dataset=dataset)}) |
| 107 | |
| 108 | def output_func(predictions, examples, output_file): |
| 109 | if args.task.lower() in ['squad', 'squad_v1']: |
| 110 | with open(output_file, "w", encoding='utf-8') as output: |
| 111 | res = {} |
| 112 | for prediction, example in zip(predictions, examples): |
| 113 | idx = example.idx |
| 114 | if prediction.lower().replace(' ', '') == 'n/a': |
| 115 | prediction = '' |
| 116 | if idx not in res or res[idx] == '': |
| 117 | res[idx] = prediction |
| 118 | json.dump(res, output) |
| 119 | with open(output_file + ".refs", "w", encoding='utf-8') as output: |
| 120 | for prediction, example in zip(predictions, examples): |
| 121 | res = {'id': example.idx, 'pred': prediction, 'gold': example.meta['answers']} |
| 122 | output.write(json.dumps(res) + '\n') |
| 123 | return |
| 124 | with open(output_file + ".hyps", "w", encoding='utf-8') as output: |
| 125 | for prediction in predictions: |
| 126 | output.write(prediction) |
| 127 | output.write("\n") |
| 128 | with open(output_file + ".refs", "w", encoding='utf-8') as output: |
| 129 | for example in examples: |
| 130 | output.write(example.meta["ref"]) |
| 131 | output.write("\n") |
| 132 | if args.task.lower() == 'squad_generation': |
| 133 | with open(output_file + ".source", "w", encoding='utf-8') as output: |
| 134 | for example in examples: |
| 135 | output.write(example.text_a.replace("\n", " ") + " Answer: " + example.meta["answer"]) |
| 136 | output.write("\n") |
| 137 | |
| 138 | return accuracy_func_provider(single_dataset_provider, metric_dict, args, is_test=is_test, eval_func=eval_func, |
| 139 | output_func=output_func, only_rank0=False) |