(milestone: TimelineMilestone)
| 432 | target_epochs = max(args.epochs, required_epochs) |
| 433 | |
| 434 | def record_snapshot(milestone: TimelineMilestone) -> None: |
| 435 | nonlocal last_eval_accuracy, layer_metadata |
| 436 | accuracy = evaluate(model, test_loader, device) |
| 437 | last_eval_accuracy = accuracy |
| 438 | snapshots = capture_layer_snapshots(model, hidden_activations) |
| 439 | if not layer_metadata: |
| 440 | layer_metadata = [snapshot.metadata for snapshot in snapshots] |
| 441 | snapshot_path = write_snapshot_file(snapshots, snapshot_dir, len(timeline_entries), milestone.identifier) |
| 442 | weights_rel_path = to_posix_relative(snapshot_path, export_root) |
| 443 | entry: dict[str, Any] = { |
| 444 | "id": milestone.identifier, |
| 445 | "order": len(timeline_entries), |
| 446 | "label": milestone.label, |
| 447 | "kind": milestone.kind, |
| 448 | "target_images": milestone.threshold_images, |
| 449 | "images_seen": images_seen, |
| 450 | "batches_seen": global_step, |
| 451 | "dataset_passes": images_seen / dataset_size if dataset_size else 0.0, |
| 452 | "description": format_snapshot_description(milestone, images_seen, global_step, dataset_size), |
| 453 | "metrics": { |
| 454 | "test_accuracy": accuracy, |
| 455 | }, |
| 456 | "weights": { |
| 457 | "path": weights_rel_path, |
| 458 | "dtype": "float16", |
| 459 | "format": "layer_array_v1", |
| 460 | }, |
| 461 | } |
| 462 | if milestone.dataset_multiple is not None: |
| 463 | entry["dataset_multiple"] = milestone.dataset_multiple |
| 464 | if images_seen > 0: |
| 465 | entry["metrics"]["avg_training_loss"] = cumulative_loss / images_seen |
| 466 | timeline_entries.append(entry) |
| 467 | print( |
| 468 | f"[Timeline] Captured '{milestone.label}' at {images_seen:,} images " |
| 469 | f"({global_step:,} batches) – accuracy: {accuracy * 100:.2f}%" |
| 470 | ) |
| 471 | |
| 472 | def advance_milestones() -> None: |
| 473 | nonlocal milestone_index |
no test coverage detected