| 258 | |
| 259 | |
| 260 | def auto_resume_helper(output_dir): |
| 261 | checkpoints = os.listdir(output_dir) |
| 262 | checkpoints = [ckpt for ckpt in checkpoints if ckpt.endswith('pth')] |
| 263 | print(f"All checkpoints founded in {output_dir}: {checkpoints}") |
| 264 | if len(checkpoints) > 0: |
| 265 | latest_checkpoint = max([os.path.join(output_dir, d) |
| 266 | for d in checkpoints], key=os.path.getmtime) |
| 267 | print(f"The latest checkpoint founded: {latest_checkpoint}") |
| 268 | resume_file = latest_checkpoint |
| 269 | else: |
| 270 | resume_file = None |
| 271 | return resume_file |
| 272 | |
| 273 | |
| 274 | def reduce_tensor(tensor, n=None): |