(model, cfg, ctx, dev_path: str, max_batches: int = 50)
| 32 | |
| 33 | @torch.no_grad() |
| 34 | def eval_dev(model, cfg, ctx, dev_path: str, max_batches: int = 50) -> float: |
| 35 | model.eval() |
| 36 | it = get_sft_batch_iterator(dev_path, cfg.batch_size, device=ctx.device, |
| 37 | rank=ctx.rank, world_size=ctx.world_size, shuffle=False, infinite=False) |
| 38 | total, n = 0.0, 0 |
| 39 | for tokens, mask, _ in it: |
| 40 | with amp_autocast(cfg.amp_dtype, ctx.device): |
| 41 | logits, _ = model(tokens) |
| 42 | loss = sft_loss(logits, tokens, mask) |
| 43 | total += loss.item(); n += 1 |
| 44 | if n >= max_batches: |
| 45 | break |
| 46 | model.train() |
| 47 | return total / max(1, n) |
| 48 | |
| 49 | |
| 50 | def main(): |
no test coverage detected