The main process.
(self)
| 287 | self.var_names_to_load.add(name) |
| 288 | |
| 289 | def run(self): |
| 290 | """The main process.""" |
| 291 | |
| 292 | # make sure to load config again |
| 293 | self.load_options(filename=self.process_info["config_filename"]) |
| 294 | |
| 295 | # initialize port for distribution run |
| 296 | if self.process_info["distributed_run"]: |
| 297 | multigpu_utils.init_distributed( |
| 298 | self.process_info["n_gpus"], |
| 299 | self.process_info["rank"], |
| 300 | auto_detect=self.process_info["use_torchrun"], |
| 301 | ) |
| 302 | print( |
| 303 | "Distributed training is enabled. " "Make sure all the modules will be wrapped by all_reduced or DDP!" |
| 304 | ) |
| 305 | |
| 306 | # determine output dir |
| 307 | if self.process_info["output_dir"] is None: |
| 308 | self.process_info["output_dir"] = "artifacts" |
| 309 | |
| 310 | # determine exp_tag |
| 311 | if self.process_info["rank"] == 0: |
| 312 | ori_exp_tag = self.process_info["exp_tag"] |
| 313 | self.process_info["exp_tag"] = self.determine_exp_tag( |
| 314 | exp_tag=self.process_info["exp_tag"], |
| 315 | artifact_root_dir=self.process_info["output_dir"], |
| 316 | continuing=self.process_info["trainer_filename"] is not None, |
| 317 | ) |
| 318 | if self.process_info["distributed_run"] and self.process_info["exp_tag"] != ori_exp_tag: |
| 319 | raise RuntimeError( |
| 320 | f"distributed enable, " |
| 321 | f'but final exp_tag {self.process_info["exp_tag"]} != ' |
| 322 | f"given exp_tag {ori_exp_tag}" |
| 323 | ) |
| 324 | |
| 325 | if self.process_info["distributed_run"]: |
| 326 | torch.distributed.barrier() |
| 327 | |
| 328 | # create folder to save checkpoints, log, plots |
| 329 | dir_dict = self._create_folders( |
| 330 | root_dir=self.process_info["output_dir"], |
| 331 | exp_tag=self.process_info["exp_tag"], |
| 332 | exp_tag_first=self.process_info["exp_tag_first"], |
| 333 | ) |
| 334 | self.process_info["dir_dict"] = dir_dict |
| 335 | |
| 336 | if self.process_info["exclude_dirs"] is None: |
| 337 | self.process_info["exclude_dirs"] = [] |
| 338 | |
| 339 | self.process_info["exclude_dirs"] += list(dir_dict.values()) |
| 340 | |
| 341 | # save the code for future reference |
| 342 | if self.process_info["rank"] == 0 and self.process_info["save_code"]: |
| 343 | print_and_save.save_code( |
| 344 | src_dir=os.getcwd(), |
| 345 | dest_dir=dir_dict["code"], |
| 346 | excluded_folders=self.process_info["exclude_dirs"], |