(self)
| 95 | self.exps = [] |
| 96 | |
| 97 | def assert_gpu_not_overlap(self): |
| 98 | all_gpus = set() |
| 99 | for exp in self.exps: |
| 100 | gpus = exp.runtime['env']['CUDA_VISIBLE_DEVICES'].split(',') |
| 101 | if all_gpus & set(gpus): |
| 102 | raise ValueError(f'GPU overlap: {self.exps}!') |
| 103 | all_gpus.update(gpus) |
| 104 | |
| 105 | def run(self, exp: Experiment): |
| 106 | if os.path.exists(os.path.join(exp.input_args.save_dir, exp.name + '.json')): |