| 394 | |
| 395 | |
| 396 | def evaluate(model_paths, eval_name, visible_count=None, wandb=None, tb_writer=None, dataset_name=None, logger=None): |
| 397 | |
| 398 | full_dict = {} |
| 399 | per_view_dict = {} |
| 400 | full_dict_polytopeonly = {} |
| 401 | per_view_dict_polytopeonly = {} |
| 402 | print("") |
| 403 | |
| 404 | scene_dir = model_paths |
| 405 | full_dict[scene_dir] = {} |
| 406 | per_view_dict[scene_dir] = {} |
| 407 | full_dict_polytopeonly[scene_dir] = {} |
| 408 | per_view_dict_polytopeonly[scene_dir] = {} |
| 409 | |
| 410 | test_dir = Path(scene_dir) / eval_name |
| 411 | |
| 412 | for method in os.listdir(test_dir): |
| 413 | |
| 414 | full_dict[scene_dir][method] = {} |
| 415 | per_view_dict[scene_dir][method] = {} |
| 416 | full_dict_polytopeonly[scene_dir][method] = {} |
| 417 | per_view_dict_polytopeonly[scene_dir][method] = {} |
| 418 | |
| 419 | method_dir = test_dir / method |
| 420 | gt_dir = method_dir/ "gt" |
| 421 | renders_dir = method_dir / "renders" |
| 422 | renders, gts, image_names = readImages(renders_dir, gt_dir) |
| 423 | |
| 424 | ssims = [] |
| 425 | psnrs = [] |
| 426 | lpipss = [] |
| 427 | |
| 428 | for idx in tqdm(range(len(renders)), desc="Metric evaluation progress"): |
| 429 | ssims.append(ssim(renders[idx], gts[idx])) |
| 430 | psnrs.append(psnr(renders[idx], gts[idx])) |
| 431 | lpipss.append(lpips_fn(renders[idx], gts[idx]).detach()) |
| 432 | |
| 433 | if wandb is not None: |
| 434 | wandb.log({"test_SSIMS":torch.stack(ssims).mean().item(), }) |
| 435 | wandb.log({"test_PSNR_final":torch.stack(psnrs).mean().item(), }) |
| 436 | wandb.log({"test_LPIPS":torch.stack(lpipss).mean().item(), }) |
| 437 | |
| 438 | logger.info(f"model_paths: \033[1;35m{model_paths}\033[0m") |
| 439 | logger.info(" SSIM : \033[1;35m{:>12.7f}\033[0m".format(torch.tensor(ssims).mean(), ".5")) |
| 440 | logger.info(" PSNR : \033[1;35m{:>12.7f}\033[0m".format(torch.tensor(psnrs).mean(), ".5")) |
| 441 | logger.info(" LPIPS: \033[1;35m{:>12.7f}\033[0m".format(torch.tensor(lpipss).mean(), ".5")) |
| 442 | print("") |
| 443 | |
| 444 | |
| 445 | if tb_writer: |
| 446 | tb_writer.add_scalar(f'{dataset_name}/SSIM', torch.tensor(ssims).mean().item(), 0) |
| 447 | tb_writer.add_scalar(f'{dataset_name}/PSNR', torch.tensor(psnrs).mean().item(), 0) |
| 448 | tb_writer.add_scalar(f'{dataset_name}/LPIPS', torch.tensor(lpipss).mean().item(), 0) |
| 449 | |
| 450 | tb_writer.add_scalar(f'{dataset_name}/VISIBLE_NUMS', torch.tensor(visible_count).mean().item(), 0) |
| 451 | |
| 452 | full_dict[scene_dir][method].update({"SSIM": torch.tensor(ssims).mean().item(), |
| 453 | "PSNR": torch.tensor(psnrs).mean().item(), |