Evaluation.
(model, dataloader, eval_metric, args)
| 136 | |
| 137 | |
| 138 | def evaluate(model, dataloader, eval_metric, args): |
| 139 | """Evaluation.""" |
| 140 | # Turn on evaluation mode which disables dropout. |
| 141 | model.eval() |
| 142 | total_output, total_count = 0.0, 0 |
| 143 | total_tokens = 0 |
| 144 | with torch.no_grad(): |
| 145 | # For all the batches in the dataset. |
| 146 | for iteration, batch in enumerate(dataloader): |
| 147 | if (iteration + 1) % args.log_interval == 0: |
| 148 | print_rank_0('> working on iteration: {}'.format(iteration)) |
| 149 | # Forward evaluation. |
| 150 | output, _, _ = lm_forward_step(batch, model, args, None, [], eval_metric=eval_metric) |
| 151 | count = batch['text'].size(0) |
| 152 | |
| 153 | total_output += output.item() |
| 154 | total_count += count |
| 155 | total_tokens += batch['loss_mask'].sum().item() |
| 156 | totals = torch.cuda.FloatTensor([total_output, total_count, total_tokens]) |
| 157 | torch.distributed.all_reduce(totals, group=mpu.get_data_parallel_group()) |
| 158 | total_output, total_count, total_tokens = totals.tolist() |
| 159 | return {eval_metric: total_output}, total_count |
| 160 | |
| 161 | |
| 162 | def evaluate_and_print_results(data_loader, model, eval_metric, args): |
no test coverage detected