| 301 | |
| 302 | |
| 303 | def check_termination(epoch): |
| 304 | if AutoResume: |
| 305 | shouldterminate = AutoResume.termination_requested() |
| 306 | if shouldterminate: |
| 307 | if args.global_rank == 0: |
| 308 | progress = "Progress %d%% (epoch %d of %d)" % ( |
| 309 | (epoch * 100 / args.max_epoch), |
| 310 | epoch, |
| 311 | args.max_epoch |
| 312 | ) |
| 313 | AutoResume.request_resume( |
| 314 | user_dict={"RESUME_FILE": logx.save_ckpt_fn, |
| 315 | "TENSORBOARD_DIR": args.result_dir, |
| 316 | "EPOCH": str(epoch) |
| 317 | }, message=progress) |
| 318 | return 1 |
| 319 | else: |
| 320 | return 1 |
| 321 | return 0 |
| 322 | |
| 323 | |
| 324 | def main(): |