| 362 | self.tb_logger.add_scalar('lr', tmp.current_lr, tmp.current_step) |
| 363 | |
| 364 | def logging(self): |
| 365 | tmp = self.tmp |
| 366 | config = self.config |
| 367 | ginfo = self.ginfo |
| 368 | |
| 369 | vlosses = tmp.vlosses |
| 370 | |
| 371 | log_msg = '\t'.join([ |
| 372 | 'Iter: [{0}/{1}] ', |
| 373 | 'task{task_id:<2}: {task_name}', |
| 374 | 'TaskFBTime: {task_time.avg:.3f}', |
| 375 | 'Time: {batch_time.avg:.3f} (ETA:{eta:.2f}h) ({data_time.avg:.3f}) ', |
| 376 | 'Loss: {loss.avg:.4f} ', |
| 377 | 'Prec@1: {top1.avg:.3f} ', |
| 378 | 'LR: {current_lr} ', |
| 379 | '{meters} ', |
| 380 | 'max mem: {memory:.0f}' |
| 381 | ]) |
| 382 | |
| 383 | MB = 1024.0 * 1024.0 |
| 384 | |
| 385 | loss_str = [] |
| 386 | for name, meter in vlosses.items(): |
| 387 | loss_str.append( |
| 388 | "{}: {} ".format(name, str(meter.item())) |
| 389 | ) |
| 390 | |
| 391 | loss_str = '\t'.join(loss_str) |
| 392 | log_msg = log_msg.format(tmp.current_step, config.max_iter, \ |
| 393 | task_id=ginfo.task_id, task_name=ginfo.task_name, \ |
| 394 | task_time=tmp.vtask_time, \ |
| 395 | batch_time=tmp.vbatch_time, \ |
| 396 | eta=(config.max_iter-tmp.current_step)*tmp.vbatch_time.avg/3600, \ |
| 397 | data_time=tmp.vdata_time, \ |
| 398 | loss=tmp.vloss, \ |
| 399 | top1=tmp.vtop1, \ |
| 400 | current_lr=tmp.current_lr, \ |
| 401 | meters=loss_str, \ |
| 402 | memory=torch.cuda.max_memory_allocated() / MB) |
| 403 | |
| 404 | self.logger.info(log_msg) |
| 405 | |
| 406 | def save(self): |
| 407 | if ((self.tmp.current_step + 1) % self.config.get('ckpt_interval', 1000) == 0 or |