searches a directory for checkpoints following a pattern (e.g., sphericalharmonics-siren) and returns the one with lowest val_loss. checkpoint format example: sphericalharmonics-siren-val_lossval_loss=6.69.ckpt
(directory, pattern, verbose=False)
| 11 | |
| 12 | |
| 13 | def find_best_checkpoint(directory, pattern, verbose=False): |
| 14 | """searches a directory for checkpoints following a pattern (e.g., sphericalharmonics-siren) and returns |
| 15 | the one with lowest val_loss. |
| 16 | checkpoint format example: sphericalharmonics-siren-val_lossval_loss=6.69.ckpt |
| 17 | """ |
| 18 | checkpoints = [c for c in os.listdir(directory) if c.endswith("ckpt")] |
| 19 | checkpoints = [c for c in checkpoints if pattern in c] |
| 20 | |
| 21 | if len(checkpoints) == 0: |
| 22 | if verbose: |
| 23 | print("no suitable checkpoint found. returning None") |
| 24 | return None |
| 25 | else: |
| 26 | if verbose: |
| 27 | print(f"resuming from checkpoints in results-dir. Found candidates {' '.join(checkpoints)}") |
| 28 | val_loss = [float(c.split("val_loss=")[-1].replace(".ckpt", "")) for c in checkpoints] |
| 29 | |
| 30 | # this line sorts checkpoints according to their validation loss and takes first (lowest val loss) |
| 31 | resume_checkpoint = [c for _, c in sorted(zip(val_loss, checkpoints))][0] |
| 32 | if verbose: |
| 33 | print(f"taking: {resume_checkpoint}") |
| 34 | |
| 35 | return os.path.join(directory, resume_checkpoint) |
| 36 | |
| 37 | |
| 38 | def set_default_if_unset(hparams, key, value): |