(args, pipe, device, test_data_loader)
| 19 | |
| 20 | |
| 21 | def test_loop(args, pipe, device, test_data_loader): |
| 22 | |
| 23 | if test_data_loader is None: |
| 24 | return |
| 25 | |
| 26 | print('testing starts.....') |
| 27 | |
| 28 | pipe.model.eval() |
| 29 | |
| 30 | if get_pipeline_parallel_rank() == args.pipeline_group_size - 1: |
| 31 | |
| 32 | def _lm_pred_func(x, y): |
| 33 | loss_fct = torch.nn.CrossEntropyLoss(reduction='none') |
| 34 | logits = x[:, :-1, :].contiguous().float() |
| 35 | labels = y[:, 1:].contiguous() |
| 36 | loss = loss_fct(logits.transpose(-1, -2), labels).mean(1).detach().cpu() |
| 37 | return loss |
| 38 | |
| 39 | loss_list = [] |
| 40 | for i, data in enumerate(test_data_loader): |
| 41 | |
| 42 | if args.evaluation_num_batch is not None and i >= args.evaluation_num_batch: |
| 43 | break |
| 44 | |
| 45 | input_ids = data['input_ids'].to(device) |
| 46 | labels = input_ids.clone() |
| 47 | pipe.infer_iter(input_ids, labels, output_=loss_list, pred_func=_lm_pred_func) |
| 48 | |
| 49 | loss = torch.tensor(loss_list).mean() |
| 50 | ppls = torch.exp(loss) |
| 51 | metric = {"valid.perplexity": ppls.item(), "valid.loss": loss.item()} |
| 52 | |
| 53 | print(metric) |
| 54 | train_log( |
| 55 | metric, |
| 56 | step=pipe.global_step, |
| 57 | ) |
| 58 | |
| 59 | else: |
| 60 | for i, data in enumerate(test_data_loader): |
| 61 | |
| 62 | if args.evaluation_num_batch is not None and i >= args.evaluation_num_batch: |
| 63 | break |
| 64 | |
| 65 | input_ids = data['input_ids'].to(device) |
| 66 | labels = input_ids.clone() |
| 67 | current_iter_time = pipe.infer_iter(input_ids, labels) |
| 68 | |
| 69 | pipe.model.train() |
| 70 | |
| 71 | |
| 72 |
no test coverage detected