Evaluate on all_dataset
(model, tokenizer, context_builder, all_dataset)
| 5 | |
| 6 | |
| 7 | def evaluate(model, tokenizer, context_builder, all_dataset): |
| 8 | ''' |
| 9 | Evaluate on all_dataset |
| 10 | ''' |
| 11 | all_dataset_pred = copy.deepcopy(all_dataset) |
| 12 | |
| 13 | do_verbose = True |
| 14 | for dataset_name, dataset in all_dataset_pred.items(): |
| 15 | for question in dataset: |
| 16 | if do_verbose: |
| 17 | question['pred'] = get_pred(model, tokenizer, context_builder, question, do_verbose) |
| 18 | do_verbose = False |
| 19 | else: |
| 20 | question['pred'] = get_pred(model, tokenizer, context_builder, question, do_verbose) |
| 21 | return all_dataset_pred |
| 22 | |
| 23 | def get_pred(model, tokenizer, context_builder, question: dict, verbose: bool = False): |
| 24 | ''' |