(gpu, config, args)
| 222 | |
| 223 | |
| 224 | def _launch_experiment(gpu, config, args): |
| 225 | |
| 226 | command, result_path, log_path = get_command(args.model, args.task, args.n_gpu, config, args.overwrite) |
| 227 | |
| 228 | shell_cmd = f"CUDA_VISIBLE_DEVICES={gpu} " + command |
| 229 | if not args.debug: |
| 230 | shell_cmd += f" > {log_path} 2>&1; " |
| 231 | |
| 232 | print("Time {}, launched exp: {}".format(str(datetime.datetime.now()), log_path)) |
| 233 | |
| 234 | # if experiment has already been run, skip |
| 235 | if not os.path.exists(result_path) or args.overwrite: |
| 236 | return_code = subprocess.call(shell_cmd, shell=True) |
| 237 | |
| 238 | if not os.path.exists(result_path): |
| 239 | # running this process failed, alert me |
| 240 | print("Dispatcher, Alert! Job has crashed! Check logfile at:[{}]".format(log_path)) |
| 241 | |
| 242 | return result_path, config |
| 243 | |
| 244 | |
| 245 |
no test coverage detected