(cfg)
| 32 | |
| 33 | |
| 34 | def create_working_directory(cfg): |
| 35 | file_name = "%s_working_dir.tmp" % os.environ["SLURM_JOB_ID"] |
| 36 | world_size = comm.get_world_size() |
| 37 | if world_size > 1 and not dist.is_initialized(): |
| 38 | comm.init_process_group("nccl", init_method="env://") |
| 39 | |
| 40 | working_dir = os.path.join(os.path.expanduser(cfg.output_dir), |
| 41 | cfg.task["class"], cfg.dataset["class"], cfg.task.model["class"], |
| 42 | time.strftime("%Y-%m-%d-%H-%M-%S")) |
| 43 | |
| 44 | # synchronize working directory |
| 45 | if comm.get_rank() == 0: |
| 46 | with open(file_name, "w") as fout: |
| 47 | fout.write(working_dir) |
| 48 | os.makedirs(working_dir) |
| 49 | comm.synchronize() |
| 50 | if comm.get_rank() != 0: |
| 51 | with open(file_name, "r") as fin: |
| 52 | working_dir = fin.read() |
| 53 | comm.synchronize() |
| 54 | if comm.get_rank() == 0: |
| 55 | os.remove(file_name) |
| 56 | |
| 57 | os.chdir(working_dir) |
| 58 | return working_dir |
| 59 | |
| 60 | |
| 61 | def detect_variables(cfg_file): |
nothing calls this directly
no outgoing calls
no test coverage detected