| 504 | self.tb_logger.add_scalar('lr', tmp.current_lr, tmp.current_step) |
| 505 | |
| 506 | def logging(self): |
| 507 | tmp = self.tmp |
| 508 | config = self.config |
| 509 | ginfo = self.ginfo |
| 510 | |
| 511 | vlosses = tmp.vlosses |
| 512 | |
| 513 | log_msg = '\t'.join([ |
| 514 | 'Iter: [{0}/{1}] ', |
| 515 | 'task{task_id:<2}: {task_name}\t' |
| 516 | 'Time: {batch_time.avg:.3f} (ETA:{eta:.2f}h) ({data_time.avg:.3f}) ', |
| 517 | 'Loss: {loss.avg:.4f} ' |
| 518 | 'Prec@1: {top1.avg:.3f} ' |
| 519 | 'LR: {current_lr} ' |
| 520 | '{meters} ', |
| 521 | 'max mem: {memory:.0f}' |
| 522 | ]) |
| 523 | |
| 524 | MB = 1024.0 * 1024.0 |
| 525 | |
| 526 | loss_str = [] |
| 527 | for name, meter in vlosses.items(): |
| 528 | loss_str.append( |
| 529 | "{}: {} ".format(name, str(meter.item())) |
| 530 | ) |
| 531 | |
| 532 | loss_str = '\t'.join(loss_str) |
| 533 | log_msg = log_msg.format(tmp.current_step, config.max_iter, \ |
| 534 | task_id=ginfo.task_id, task_name=ginfo.task_name, \ |
| 535 | batch_time=tmp.vbatch_time, \ |
| 536 | eta=(config.max_iter-tmp.current_step)*tmp.vbatch_time.avg/3600, \ |
| 537 | data_time=tmp.vdata_time, \ |
| 538 | loss=tmp.vloss, \ |
| 539 | top1=tmp.vtop1, \ |
| 540 | current_lr=tmp.current_lr, \ |
| 541 | meters=loss_str, \ |
| 542 | memory=torch.cuda.max_memory_allocated() / MB) |
| 543 | |
| 544 | self.logger.info(log_msg) |
| 545 | |
| 546 | def save(self): |
| 547 | config = self.config |