(self)
| 468 | self.dataloader['train_iterations'] = itr + 1 |
| 469 | |
| 470 | def validate_epoch(self): |
| 471 | if 'validation_loader' not in self.dataloader: |
| 472 | val = False |
| 473 | else: |
| 474 | if isinstance(self.validation_epochs, int): |
| 475 | val = (self.last_epoch + 1) % self.validation_epochs == 0 |
| 476 | else: |
| 477 | val = (self.last_epoch + 1) in self.validation_epochs |
| 478 | |
| 479 | if val: |
| 480 | if self.args.distributed: |
| 481 | self.dataloader['validation_loader'].sampler.set_epoch(self.last_epoch) |
| 482 | |
| 483 | self.model.eval() |
| 484 | overall_loss = None |
| 485 | epoch_start = time.time() |
| 486 | itr_start = time.time() |
| 487 | itr = -1 |
| 488 | for itr, batch in enumerate(self.dataloader['validation_loader']): |
| 489 | data_time = time.time() - itr_start |
| 490 | step_start = time.time() |
| 491 | loss = self.step(batch, phase='val') |
| 492 | |
| 493 | for loss_n, loss_dict in loss.items(): |
| 494 | loss[loss_n] = reduce_dict(loss_dict) |
| 495 | if overall_loss is None: |
| 496 | overall_loss = loss |
| 497 | else: |
| 498 | for loss_n, loss_dict in loss.items(): |
| 499 | for k, v in loss_dict.items(): |
| 500 | overall_loss[loss_n][k] = (overall_loss[loss_n][k] * itr + loss[loss_n][k]) / (itr + 1) |
| 501 | |
| 502 | if self.logger is not None and (itr+1) % self.args.log_frequency == 0: |
| 503 | info = '{}: val'.format(self.args.name) |
| 504 | info = info + ': Epoch {}/{} | iter {}/{}'.format(self.last_epoch, self.max_epochs, itr, self.dataloader['validation_iterations']) |
| 505 | for loss_n, loss_dict in loss.items(): |
| 506 | info += ' ||' |
| 507 | info += '' if loss_n == 'none' else ' {}'.format(loss_n) |
| 508 | # info = info + ': Epoch {}/{} | iter {}/{}'.format(self.last_epoch, self.max_epochs, itr, self.dataloader['validation_iterations']) |
| 509 | for k in loss_dict: |
| 510 | info += ' | {}: {:.4f}'.format(k, float(loss_dict[k])) |
| 511 | |
| 512 | itr_time_avg = (time.time() - epoch_start) / (itr + 1) |
| 513 | info += ' || data_time: {dt}s | fbward_time: {fbt}s | iter_time: {it}s | epoch_time: {et} | left_time: {lt}'.format( |
| 514 | dt=round(data_time, 1), |
| 515 | fbt=round(time.time() - step_start, 1), |
| 516 | it=round(time.time() - itr_start, 1), |
| 517 | et=format_seconds(time.time() - epoch_start), |
| 518 | lt=format_seconds(itr_time_avg*(self.dataloader['train_iterations']-itr-1)) |
| 519 | ) |
| 520 | |
| 521 | self.logger.log_info(info) |
| 522 | itr_start = time.time() |
| 523 | # modify here to make sure dataloader['validation_iterations'] is correct |
| 524 | assert itr >= 0, "The data is too less to form one iteration!" |
| 525 | self.dataloader['validation_iterations'] = itr + 1 |
| 526 | |
| 527 | if self.logger is not None: |
no test coverage detected