Evaluation.
(data_iterator, model, eval_iters, args, timers, split, verbose=False, has_last=True, hooks={})
| 547 | return |
| 548 | |
| 549 | def evaluate(data_iterator, model, eval_iters, args, timers, split, verbose=False, has_last=True, hooks={}): |
| 550 | """Evaluation.""" |
| 551 | forward_step = hooks['forward_step_eval'] |
| 552 | # Turn on evaluation mode which disables dropout. |
| 553 | model.eval() |
| 554 | rank = torch.distributed.get_rank(group=mpu.get_data_parallel_group()) |
| 555 | total_lm_loss, metrics_total = 0, {} |
| 556 | if split=='val': |
| 557 | last_shape = args.val_last_shape |
| 558 | drop_number = args.val_drop_number |
| 559 | else: |
| 560 | assert split=='test' |
| 561 | last_shape = args.test_last_shape |
| 562 | drop_number = args.test_drop_number |
| 563 | is_scalar = {} |
| 564 | with torch.no_grad(): |
| 565 | iteration = 0 |
| 566 | while iteration < eval_iters: |
| 567 | iteration += 1 |
| 568 | if verbose and iteration % args.log_interval == 0: |
| 569 | print_rank0('Evaluating iter {}/{}'.format(iteration, eval_iters)) |
| 570 | # Forward evaluation. |
| 571 | # try: |
| 572 | lm_loss, metrics = forward_step(data_iterator, model, args, timers) |
| 573 | '''when contiguous memory optimizations are enabled, the buffers |
| 574 | allocated by the optimizations are deallocated during backward pass |
| 575 | in the absence of backward pass the buffers should be reset after each |
| 576 | forward pass''' |
| 577 | if args.deepspeed and args.deepspeed_activation_checkpointing: |
| 578 | deepspeed.checkpointing.reset() |
| 579 | total_lm_loss += lm_loss.data.detach().float().item() |
| 580 | is_last = True if iteration == eval_iters and args.strict_eval and len(last_shape)>0 else False |
| 581 | for name in metrics: |
| 582 | if name not in metrics_total: |
| 583 | metrics_total[name] = [] |
| 584 | is_scalar[name] = True if len(metrics[name].shape)==0 else False |
| 585 | shape = list(metrics[name].shape) |
| 586 | if not is_scalar[name] and is_last and metrics[name].shape[0] != last_shape[0]: |
| 587 | # pad tensor's first dim to args.batch_size |
| 588 | metrics[name] = torch.concat([metrics[name], torch.zeros([last_shape[0]-metrics[name].shape[0]] + shape[1:], dtype=metrics[name].dtype, device=metrics[name].device)]) |
| 589 | if rank==0: |
| 590 | metrics_gathered = [torch.zeros_like(metrics[name], dtype=metrics[name].dtype, device=metrics[name].device) for _ in range(args.world_size)] |
| 591 | else: |
| 592 | # metrics_gathered = None |
| 593 | metrics_gathered = [torch.zeros_like(metrics[name], dtype=metrics[name].dtype, device=metrics[name].device) for _ in range(args.world_size)] |
| 594 | # torch.distributed.gather(metrics[name], metrics_gathered, 0) |
| 595 | torch.distributed.all_gather(metrics_gathered, metrics[name]) |
| 596 | |
| 597 | if rank==0: |
| 598 | gathered_len = len(metrics_gathered) if not is_last else len(metrics_gathered) - drop_number * args.model_parallel_size |
| 599 | for i in range(gathered_len): |
| 600 | if is_scalar[name] or not is_last: |
| 601 | metrics_total[name].append(metrics_gathered[i].data.cpu()) |
| 602 | else: |
| 603 | metrics_total[name].append(metrics_gathered[i][:last_shape[i]].data.cpu()) |
| 604 | # Move model back to the train mode. |
| 605 | model.train() |
| 606 |
no test coverage detected