(model, epoch, output_predictions=False, summary_writer=None)
| 75 | dataloaders.append((dataset.dataset_name, dataloader)) |
| 76 | |
| 77 | def metrics_func(model, epoch, output_predictions=False, summary_writer=None): |
| 78 | print_rank_0('calculating metrics ...') |
| 79 | score_dict = OrderedDict([(key, 0.0) for key in metric_dict]) if isinstance(metric_dict, dict) else { |
| 80 | metric_dict: 0.0} |
| 81 | total = 0 |
| 82 | for name, dataloader in dataloaders: |
| 83 | example_dict = None |
| 84 | if hasattr(dataloader.dataset, "examples"): |
| 85 | example_dict = dataloader.dataset.examples |
| 86 | start_time = time.time() |
| 87 | predictions, labels, examples = eval_func(model, dataloader, example_dict, args) |
| 88 | elapsed_time = time.time() - start_time |
| 89 | if output_predictions and torch.distributed.get_rank() == 0: |
| 90 | filename = os.path.join(args.log_dir, name + '.jsonl') |
| 91 | output_func(predictions, examples, filename) |
| 92 | total_count = len(predictions) |
| 93 | single_dict = {key: metric(predictions, labels, examples) for key, metric in metric_dict.items()} |
| 94 | output_str = ' > |epoch: {}| metrics for {}: total {}'.format(epoch, name, total_count) |
| 95 | for key, value in single_dict.items(): |
| 96 | output_str += " {} = {:.4f} %".format(key, value) |
| 97 | if summary_writer is not None and epoch >= 0 and not is_test and len(dataloaders) > 1: |
| 98 | summary_writer.add_scalar(f'Train/valid_{name}_{key}', value, epoch) |
| 99 | output_str += ' elapsed time (sec): {:.3f}'.format(elapsed_time) |
| 100 | if len(dataloaders) > 1: |
| 101 | print_rank_0(output_str) |
| 102 | for key in score_dict: |
| 103 | score_dict[key] += single_dict[key] * total_count |
| 104 | total += total_count |
| 105 | score_dict = {key: score / float(total) for key, score in score_dict.items()} |
| 106 | output_str = ' >> |epoch: {}| overall: total = {}'.format(epoch, total) |
| 107 | for key, score in score_dict.items(): |
| 108 | output_str += " {} = {:.4f}".format(key, score) |
| 109 | if summary_writer is not None and epoch >= 0 and not is_test: |
| 110 | summary_writer.add_scalar(f'Train/valid_{key}', score, epoch) |
| 111 | print_rank_0(output_str) |
| 112 | return score_dict |
| 113 | |
| 114 | return metrics_func |
| 115 |
nothing calls this directly
no test coverage detected