Get exception checkpoint based on restore ranks Args: args_param: training model parameters Returns: exception checkpoint list
(args_param)
| 374 | |
| 375 | |
| 376 | def get_exception_checkpoints(args_param): |
| 377 | """ |
| 378 | Get exception checkpoint based on restore ranks |
| 379 | Args: |
| 380 | args_param: training model parameters |
| 381 | Returns: exception checkpoint list |
| 382 | """ |
| 383 | print("======start exception checkpoint", flush=True) |
| 384 | restore_ranks = os.getenv("RESTORE_RANKS") |
| 385 | if not restore_ranks: |
| 386 | return None |
| 387 | |
| 388 | restore_rank_list = list(map(int, restore_ranks.split(","))) |
| 389 | ckpt_file_list = [] |
| 390 | ckpt_name = args_param.ckpt_name_prefix |
| 391 | for ckpt_rank in restore_rank_list: |
| 392 | ckpt_pattern = os.path.join(args_param.save_checkpoint_path, |
| 393 | f"rank_{ckpt_rank}", |
| 394 | f"{ckpt_name}*_breakpoint.ckpt") |
| 395 | ckpt_files = glob.glob(ckpt_pattern) |
| 396 | if not ckpt_files: |
| 397 | print( |
| 398 | f"There is no ckpt file in {args_param.save_checkpoint_path}, " |
| 399 | f"current ckpt_files found is {ckpt_files} " |
| 400 | f"with pattern {ckpt_pattern}, so skip the loading.") |
| 401 | return None |
| 402 | ckpt_files.sort(key=os.path.getmtime, reverse=True) |
| 403 | ckpt_file_list.append(ckpt_files[0]) |
| 404 | print(f"checkpoint file {ckpt_file_list}") |
| 405 | return ckpt_file_list |
| 406 | |
| 407 | |
| 408 | def check_exception_checkpoints(ckpt_file_list): |
no outgoing calls
no test coverage detected