Provide function that calculates accuracies.
(single_dataset_provider, metric_dict, args, is_test=False, eval_func=None, output_func=None,
only_rank0=True, tokenizer=None)
| 52 | |
| 53 | |
| 54 | def accuracy_func_provider(single_dataset_provider, metric_dict, args, is_test=False, eval_func=None, output_func=None, |
| 55 | only_rank0=True, tokenizer=None): |
| 56 | """Provide function that calculates accuracies.""" |
| 57 | # Build dataloaders. |
| 58 | global global_tokenizer |
| 59 | global_tokenizer = tokenizer |
| 60 | if only_rank0 and torch.distributed.is_initialized() and torch.distributed.get_rank() != 0: |
| 61 | return None |
| 62 | if is_test and not args.eval_valid: |
| 63 | datapaths = args.test_data if args.test_data is not None else ['test'] |
| 64 | else: |
| 65 | datapaths = args.valid_data if args.valid_data is not None else ['dev'] |
| 66 | if eval_func is None: |
| 67 | eval_func = multichoice_evaluate |
| 68 | dataloaders = [] |
| 69 | eval_batch_size = args.eval_batch_size if args.eval_batch_size else args.batch_size |
| 70 | for datapath in datapaths: |
| 71 | dataset = single_dataset_provider(datapath) |
| 72 | dataloader = build_data_loader( |
| 73 | dataset, eval_batch_size, num_workers=args.num_workers, |
| 74 | drop_last=False, shuffle=False, only_rank0=only_rank0) |
| 75 | dataloaders.append((dataset.dataset_name, dataloader)) |
| 76 | |
| 77 | def metrics_func(model, epoch, output_predictions=False, summary_writer=None): |
| 78 | print_rank_0('calculating metrics ...') |
| 79 | score_dict = OrderedDict([(key, 0.0) for key in metric_dict]) if isinstance(metric_dict, dict) else { |
| 80 | metric_dict: 0.0} |
| 81 | total = 0 |
| 82 | for name, dataloader in dataloaders: |
| 83 | example_dict = None |
| 84 | if hasattr(dataloader.dataset, "examples"): |
| 85 | example_dict = dataloader.dataset.examples |
| 86 | start_time = time.time() |
| 87 | predictions, labels, examples = eval_func(model, dataloader, example_dict, args) |
| 88 | elapsed_time = time.time() - start_time |
| 89 | if output_predictions and torch.distributed.get_rank() == 0: |
| 90 | filename = os.path.join(args.log_dir, name + '.jsonl') |
| 91 | output_func(predictions, examples, filename) |
| 92 | total_count = len(predictions) |
| 93 | single_dict = {key: metric(predictions, labels, examples) for key, metric in metric_dict.items()} |
| 94 | output_str = ' > |epoch: {}| metrics for {}: total {}'.format(epoch, name, total_count) |
| 95 | for key, value in single_dict.items(): |
| 96 | output_str += " {} = {:.4f} %".format(key, value) |
| 97 | if summary_writer is not None and epoch >= 0 and not is_test and len(dataloaders) > 1: |
| 98 | summary_writer.add_scalar(f'Train/valid_{name}_{key}', value, epoch) |
| 99 | output_str += ' elapsed time (sec): {:.3f}'.format(elapsed_time) |
| 100 | if len(dataloaders) > 1: |
| 101 | print_rank_0(output_str) |
| 102 | for key in score_dict: |
| 103 | score_dict[key] += single_dict[key] * total_count |
| 104 | total += total_count |
| 105 | score_dict = {key: score / float(total) for key, score in score_dict.items()} |
| 106 | output_str = ' >> |epoch: {}| overall: total = {}'.format(epoch, total) |
| 107 | for key, score in score_dict.items(): |
| 108 | output_str += " {} = {:.4f}".format(key, score) |
| 109 | if summary_writer is not None and epoch >= 0 and not is_test: |
| 110 | summary_writer.add_scalar(f'Train/valid_{key}', score, epoch) |
| 111 | print_rank_0(output_str) |
no test coverage detected