(self, args: Any)
| 348 | break |
| 349 | |
| 350 | def begin(self, args: Any): |
| 351 | exps = self.prepare_experiments(args) |
| 352 | logger.info(f'all exps: {exps}') |
| 353 | exps.sort(key=lambda e: e.priority) |
| 354 | self.exp_queue = deque() |
| 355 | for exp in exps: |
| 356 | self.exp_queue.append(exp) |
| 357 | |
| 358 | while len(self.exp_queue) or len(self.exps) > 0: |
| 359 | while len(self.exp_queue): |
| 360 | try: |
| 361 | logger.info(f'Running exp: {self.exp_queue[0].name}') |
| 362 | self.run(self.exp_queue[0]) |
| 363 | except Exception as e: |
| 364 | if not isinstance(e, AssertionError): |
| 365 | logger.error(f'Adding exp {self.exp_queue[0].name} error because of:') |
| 366 | logger.error(e) |
| 367 | self.exp_queue.popleft() |
| 368 | else: |
| 369 | logger.info(f'Adding exp {self.exp_queue[0].name} error because of:', str(e)) |
| 370 | if 'no free gpu' in str(e).lower(): |
| 371 | break |
| 372 | else: |
| 373 | continue |
| 374 | else: |
| 375 | self.exp_queue.popleft() |
| 376 | self._poll() |
| 377 | logger.info(f'Run task finished because of exp queue: {self.exp_queue} and exps: {self.exps}') |
| 378 | |
| 379 | |
| 380 | def find_all_config(dir_or_file: str): |
no test coverage detected