Given camera poses, return the captured RGBD images. Args: camera: (b, q) already on device render_normal_w: whether to ray trace to get surface normal in world coordinate render_method: 'rasteriza
(
self,
camera: Camera,
render_normal_w: bool = True,
device: torch.device = torch.device('cpu'),
render_method: str = 'rasterization',
camera_for_normal: T.Optional[Camera] = None,
)
| 3293 | self.mesh.textures = new_textures |
| 3294 | |
| 3295 | def get_rgbd_image( |
| 3296 | self, |
| 3297 | camera: Camera, |
| 3298 | render_normal_w: bool = True, |
| 3299 | device: torch.device = torch.device('cpu'), |
| 3300 | render_method: str = 'rasterization', |
| 3301 | camera_for_normal: T.Optional[Camera] = None, |
| 3302 | ) -> RGBDImage: |
| 3303 | """ |
| 3304 | Given camera poses, return the captured RGBD images. |
| 3305 | |
| 3306 | Args: |
| 3307 | camera: |
| 3308 | (b, q) already on device |
| 3309 | render_normal_w: |
| 3310 | whether to ray trace to get surface normal in world coordinate |
| 3311 | render_method: |
| 3312 | 'rasterization': use o3d rasterization (may have anti-aliasing applied) |
| 3313 | 'ray_cast': use ray_casting to sample rgb |
| 3314 | camera_for_normal: |
| 3315 | (b, q) camera for computing normal (in case intrinsic is negative at (2,2)) |
| 3316 | used only when using rasterization. |
| 3317 | |
| 3318 | Returns: |
| 3319 | rgbdimage: (b, q) on device |
| 3320 | """ |
| 3321 | |
| 3322 | if render_method == 'rasterization': |
| 3323 | return self._rasterize_rendering( |
| 3324 | camera=camera, |
| 3325 | render_normal_w=render_normal_w, |
| 3326 | device=device, |
| 3327 | camera_for_normal=camera_for_normal, |
| 3328 | ) |
| 3329 | |
| 3330 | elif render_method == 'ray_cast': |
| 3331 | # run ray intersection to get normal |
| 3332 | ray = camera.generate_camera_rays(device=device) # (b, q, h, w) |
| 3333 | out_dict = self.get_ray_intersection( |
| 3334 | ray=ray, |
| 3335 | device=device, |
| 3336 | ) |
| 3337 | rgb = out_dict['ray_rgbs'] # (b, q, h, w, 3) |
| 3338 | normal_w = out_dict['surface_normals_w'] # (b, q, h, w, 3) |
| 3339 | hit_map = out_dict['hit_map'] # (b, q, h, w) |
| 3340 | ray_ts = out_dict['ray_ts'] # (b, q, h, w) |
| 3341 | |
| 3342 | # convert ray_ts to depth |
| 3343 | xyz_w = ray.origins_w + ray_ts.unsqueeze(-1) * ray.directions_w # (b, q, h, w, 3) |
| 3344 | xyz1_w = torch.cat((xyz_w, torch.ones_like(xyz_w[..., 0:1])), dim=-1) # (b, q, h, w, 4) |
| 3345 | H_w2c = camera.get_H_w2c() # (b, q, 4, 4) |
| 3346 | xyz1_c = (H_w2c.unsqueeze(2).unsqueeze(2) @ xyz1_w.unsqueeze(-1)).squeeze(-1) # (b, q, h, w, 4) |
| 3347 | z_map = xyz1_c[..., 2] # (b, q, h, w) |
| 3348 | |
| 3349 | valid_mask = torch.logical_and(hit_map > 0.5, z_map.isfinite()) |
| 3350 | z_map[valid_mask.logical_not()] = INF |
| 3351 | |
| 3352 | return RGBDImage( |
no test coverage detected