(self)
| 506 | raise |
| 507 | |
| 508 | def run(self): |
| 509 | |
| 510 | if DEBUG_MODE: |
| 511 | self.run_dummy() |
| 512 | return |
| 513 | |
| 514 | config = self.config |
| 515 | ginfo = self.ginfo |
| 516 | tmp = self.tmp |
| 517 | |
| 518 | self.pre_run() |
| 519 | |
| 520 | end = time.time() |
| 521 | |
| 522 | load_flag = False |
| 523 | |
| 524 | for i, tmp.input in enumerate(self.loader): |
| 525 | tmp.vdata_time.update(time.time() - end) |
| 526 | |
| 527 | self.prepare_data() |
| 528 | |
| 529 | # TODO currently a work around for gpu memory leak when recovering |
| 530 | if load_flag: |
| 531 | tmp.current_step = 0 |
| 532 | self.forward() |
| 533 | self.backward() |
| 534 | dist.barrier() |
| 535 | #self.update() |
| 536 | self.load(self.load_args) |
| 537 | load_flag = False |
| 538 | |
| 539 | tmp.current_step = self.last_iter + i + 1 |
| 540 | self.lr_scheduler.step(tmp.current_step) |
| 541 | tmp.current_lr = self.lr_scheduler.get_lr()[0] |
| 542 | |
| 543 | # import pdb; pdb.set_trace(); |
| 544 | self.forward() |
| 545 | |
| 546 | if self.config.get('expand_bs', False): |
| 547 | self.backward_expand_bs() |
| 548 | if (tmp.current_step + 1) % 2 == 0: |
| 549 | if self.autodenan: |
| 550 | is_nan = self.auto_denan() |
| 551 | if is_nan: |
| 552 | continue |
| 553 | |
| 554 | self.model.reduce_gradients() |
| 555 | self.optimizer.step() |
| 556 | self.optimizer.zero_grad() |
| 557 | else: |
| 558 | self.backward() |
| 559 | if self.autodenan: |
| 560 | is_nan = self.auto_denan() |
| 561 | if is_nan: |
| 562 | continue |
| 563 | |
| 564 | self.model.reduce_gradients() |
| 565 | self.optimizer.step() |
nothing calls this directly
no test coverage detected