| 579 | pass |
| 580 | |
| 581 | def run(self): |
| 582 | config = self.config |
| 583 | ginfo = self.ginfo |
| 584 | tmp = self.tmp |
| 585 | |
| 586 | self.pre_run() |
| 587 | |
| 588 | end = time.time() |
| 589 | |
| 590 | load_flag = True |
| 591 | |
| 592 | for i, tmp.input in enumerate(self.loader): |
| 593 | tmp.vdata_time.update(time.time() - end) |
| 594 | self.prepare_data() |
| 595 | # TODO currently a work around for gpu memory leak when recovering |
| 596 | if load_flag: |
| 597 | tmp.current_step = 0 |
| 598 | self.forward() |
| 599 | self.model.module.decoder_module.ignore_this_iter = True |
| 600 | self.backward() |
| 601 | self.model.module.decoder_module.ignore_this_iter = False |
| 602 | dist.barrier() |
| 603 | #self.update() |
| 604 | self.load(self.load_args) |
| 605 | load_flag = False |
| 606 | |
| 607 | tmp.current_step = self.last_iter + i + 1 |
| 608 | self.lr_scheduler.step(tmp.current_step) |
| 609 | tmp.current_lr = self.lr_scheduler.get_lr()[0] |
| 610 | |
| 611 | self.forward() |
| 612 | self.backward() |
| 613 | |
| 614 | if self.autodenan: |
| 615 | self.auto_denan() |
| 616 | |
| 617 | #self.play_with_grads() |
| 618 | self.update() |
| 619 | self.gather_result() |
| 620 | |
| 621 | tmp.vbatch_time.update(time.time() - end) |
| 622 | end = time.time() |
| 623 | |
| 624 | if tmp.current_step % config.print_freq == 0 and ginfo.task_rank == 0: |
| 625 | if ginfo.task_id == 0: |
| 626 | self.tb_logging() |
| 627 | self.logging() |
| 628 | |
| 629 | self.save() |
| 630 | |
| 631 | self.post_run() |