Function sample_point_cloud_with_ray_tracing

plib/render.py:24–92

Sample a set of points (with color) on the mesh. Args: mesh: a triangle mesh representing the surface. We assume it only have one texture map cam_pose: a dictionary containing: - intrinsic_matrix: the (3,3) intrinsic matri

(
        mesh: o3d.geometry.TriangleMesh,
        cam_pose: T.Dict[str, T.Any],
        texture_maps: T.List[np.ndarray] = None,
)

def sample_point_cloud_with_ray_tracing(
        mesh: o3d.geometry.TriangleMesh,
        cam_pose: T.Dict[str, T.Any],
        texture_maps: T.List[np.ndarray] = None,
) -> T.Dict[str, T.Any]:
    """
    Sample a set of points (with color) on the mesh.

    Args:
        mesh:
            a triangle mesh representing the surface.
            We assume it only have one texture map
        cam_pose:
            a dictionary containing:
                - intrinsic_matrix: the (3,3) intrinsic matrix
                - extrinsic_matrix: the (4,4) homogeneous matrix (from world to local)
                - width_px: number of pixels of the sensor
                - height_px: number of pixels of the sensor
        texture_maps:
            a list of texture maps (h, w, c) that we want to get the values using uv_mapping
            For example, if it is the rgb albedo, it is be retrieved with
            :py:`texture = np.asarray(mesh.textures[0]) / 255.  # (h,w,3)`.
            It can also be surface normals, features, etc.
            If `None`, not uv interpolation is performed.
    Returns:
        raycast_results:
            the output of `RaycastingScene`
        uv_outputs:
            a list of uv-interpolated values from each texture map.
    """

    # set up the raycasting scene
    mesh_t = o3d.t.geometry.TriangleMesh.from_legacy(mesh)
    scene = o3d.t.geometry.RaycastingScene()
    mesh_id = scene.add_triangles(mesh_t)

    # create the pinhole camera rays
    rays = o3d.t.geometry.RaycastingScene.create_rays_pinhole(
        intrinsic_matrix=cam_pose['intrinsic_matrix'],
        extrinsic_matrix=cam_pose['extrinsic_matrix'],
        width_px=cam_pose['width_px'],
        height_px=cam_pose['height_px'],
    )

    # cast the rays, get the intersections
    raycast_results = scene.cast_rays(rays)
    # if the ray does not hit the mesh, it goes to inf
    hit_map = 1 - np.isinf(raycast_results['t_hit'].numpy())  # (h',w')

    if texture_maps is None or len(texture_maps) == 0:
        return dict(
            hits=hit_map,  # (h', w')  1: hit, 0: not hit
            rays=rays,  # (h', w', 6)  x, y, z, dx, dy, dz
            raycast_results=raycast_results,
            uv_outputs=[],
        )

    uv_outputs = interp_texture_map_from_ray_tracing_results(