finetune pretrained PoseNet using NeRF
(args, model, optimizer, i_split, hwf, near, far, device, early_stopping, images=None, poses_train=None, train_dl=None, val_dl=None, test_dl=None)
| 361 | |
| 362 | |
| 363 | def train_nerf_tracking(args, model, optimizer, i_split, hwf, near, far, device, early_stopping, images=None, poses_train=None, train_dl=None, val_dl=None, test_dl=None): |
| 364 | ''' finetune pretrained PoseNet using NeRF ''' |
| 365 | half_res = False # direct-pn paper settings |
| 366 | # half_res = True # This half_res is to further downsample the output image of nerf to 100x100 |
| 367 | # Prepare dataloaders for PoseNet, each batch contains (image, pose) |
| 368 | if args.dataset_type != '7Scenes' and args.dataset_type != 'Cambridge': # blender dataset |
| 369 | train_dl, val_dl, test_dl = prepare_data(args, images, poses_train, i_split) |
| 370 | _, render_kwargs_test, _, grad_vars, _ = create_nerf(args) # llff nerf |
| 371 | else: |
| 372 | # load NeRF model |
| 373 | _, render_kwargs_test, start, grad_vars, _ = create_nerf(args) |
| 374 | global_step = start |
| 375 | if args.reduce_embedding==2: |
| 376 | render_kwargs_test['i_epoch'] = global_step |
| 377 | |
| 378 | # # only freeze 150MB? effectiveness of this towards training result is yet unknown |
| 379 | render_kwargs_test['embedding_a'] = disable_model_grad(render_kwargs_test['embedding_a']) |
| 380 | render_kwargs_test['embedding_t'] = disable_model_grad(render_kwargs_test['embedding_t']) |
| 381 | render_kwargs_test['network_fn'] = disable_model_grad(render_kwargs_test['network_fn']) |
| 382 | render_kwargs_test['network_fine'] = disable_model_grad(render_kwargs_test['network_fine']) |
| 383 | |
| 384 | data_loaders = [train_dl, val_dl, test_dl] |
| 385 | bds_dict = { |
| 386 | 'near' : near, |
| 387 | 'far' : far, |
| 388 | } |
| 389 | # render_kwargs_train.update(bds_dict) |
| 390 | render_kwargs_test.update(bds_dict) |
| 391 | i_train, i_val, i_test = i_split |
| 392 | |
| 393 | N_epoch = 2001 |
| 394 | print('Begin') |
| 395 | print('TRAIN views are', i_train) |
| 396 | print('TEST views are', i_test) |
| 397 | print('VAL views are', i_val) |
| 398 | time0 = time.time() |
| 399 | |
| 400 | model_log = tqdm(total=0, position = 1, bar_format='{desc}') |
| 401 | for epoch in tqdm(range(N_epoch), desc='epochs'): |
| 402 | #train 1 epoch with batch_size = 1 |
| 403 | loss, psnr = train_on_epoch(args, data_loaders, model, hwf, optimizer, half_res, device, **render_kwargs_test) |
| 404 | |
| 405 | val_loss, val_psnr = eval_on_epoch(args, data_loaders, model, hwf, half_res, device, **render_kwargs_test) |
| 406 | |
| 407 | tqdm.write('At epoch {0:4d} : train loss: {1:.4f}, train psnr: {2:.4f}, val loss: {3:.4f}, val psnr: {4:.4f}'.format(epoch, loss, psnr, val_loss, val_psnr)) |
| 408 | |
| 409 | # check wether to early stop |
| 410 | early_stopping(val_loss, model, epoch=epoch, save_multiple=(not args.no_save_multiple), save_all=args.save_all_ckpt) |
| 411 | if early_stopping.early_stop: |
| 412 | print("Early stopping") |
| 413 | break |
| 414 | model_log.set_description_str(f'Best val loss: {early_stopping.val_loss_min:.4f}') |
| 415 | |
| 416 | if (epoch % 1 == 0) and (args.pose_only != 1): |
| 417 | ### run one single image save the result |
| 418 | if args.dataset_type == '7Scenes' or args.dataset_type == 'llff' or args.dataset_type == 'Cambridge': |
| 419 | save_val_result_7Scenes(args, epoch, val_dl, model, hwf, half_res, device, **render_kwargs_test) |
| 420 |
nothing calls this directly
no test coverage detected