Instantiates the job object and runs it.
(
config: om.DictConfig,
gpu_queue: Optional[mp.Queue] = None,
process_to_gpu: Optional[DictProxy] = None,
)
| 334 | |
| 335 | |
| 336 | def run_job_worker( |
| 337 | config: om.DictConfig, |
| 338 | gpu_queue: Optional[mp.Queue] = None, |
| 339 | process_to_gpu: Optional[DictProxy] = None, |
| 340 | ) -> Any: |
| 341 | """Instantiates the job object and runs it.""" |
| 342 | # need to set seed before model initialization for determinism |
| 343 | reproducibility.configure_deterministic_mode() |
| 344 | reproducibility.seed_all(config.seed) |
| 345 | task_cls = TASK_NAME_TO_CLASS[config.task] |
| 346 | |
| 347 | model = build_model( |
| 348 | config.model, |
| 349 | num_labels=task_cls.num_labels, |
| 350 | multiple_choice=task_cls.multiple_choice, |
| 351 | custom_eval_metrics=task_cls.custom_eval_metrics, |
| 352 | ) |
| 353 | |
| 354 | instantiated_job = task_cls( |
| 355 | job_name=config.job_name, |
| 356 | seed=config.seed, |
| 357 | model=model, |
| 358 | tokenizer_name=config.tokenizer_name, |
| 359 | scheduler=build_scheduler(config.scheduler), |
| 360 | optimizer=build_optimizer(config.optimizer, model), |
| 361 | load_path=config.load_path, |
| 362 | save_folder=config.save_folder, |
| 363 | loggers=[build_logger(name, logger_config) for name, logger_config in config.get("loggers", {}).items()], |
| 364 | callbacks=[ |
| 365 | build_callback(name, callback_config) for name, callback_config in config.get("callbacks", {}).items() |
| 366 | ], |
| 367 | algorithms=[ |
| 368 | build_algorithm(name, algorithm_config) for name, algorithm_config in config.get("algorithms", {}).items() |
| 369 | ], |
| 370 | precision=config.precision, |
| 371 | **config.trainer_kwargs, |
| 372 | ) |
| 373 | results = instantiated_job.run(gpu_queue, process_to_gpu, config) |
| 374 | |
| 375 | # Extract W&B run ID from the logger |
| 376 | results["wandb_name"] = None |
| 377 | results["wandb_project"] = None |
| 378 | results["wandb_entity"] = None |
| 379 | |
| 380 | if results["loggers"] is None: |
| 381 | results["loggers"] = [] |
| 382 | for logger in results["loggers"]: |
| 383 | if isinstance(logger, WandBLogger): |
| 384 | results["wandb_run_url"] = logger.run_url |
| 385 | break |
| 386 | |
| 387 | # Clean up: delete the job so that the optimizer and anything else on the gpu gets deleted |
| 388 | del instantiated_job |
| 389 | torch.cuda.empty_cache() |
| 390 | gc.collect() |
| 391 | return results |
| 392 | |
| 393 |
no test coverage detected