Warp kpts0 from I0 to I1 with depth, K and Rt Also check covisibility and depth consistency. Depth is consistent if relative error < 0.2 (hard-coded). # https://github.com/zju3dv/LoFTR/blob/94e98b695be18acb43d5d3250f52226a8e36f839/src/loftr/utils/geometry.py adapted from here Args:
(kpts0, depth0, depth1, T_0to1, K0, K1, smooth_mask = False, return_relative_depth_error = False, depth_interpolation_mode = "bilinear", relative_depth_error_threshold = 0.05)
| 153 | |
| 154 | @torch.no_grad() |
| 155 | def warp_kpts(kpts0, depth0, depth1, T_0to1, K0, K1, smooth_mask = False, return_relative_depth_error = False, depth_interpolation_mode = "bilinear", relative_depth_error_threshold = 0.05): |
| 156 | """Warp kpts0 from I0 to I1 with depth, K and Rt |
| 157 | Also check covisibility and depth consistency. |
| 158 | Depth is consistent if relative error < 0.2 (hard-coded). |
| 159 | # https://github.com/zju3dv/LoFTR/blob/94e98b695be18acb43d5d3250f52226a8e36f839/src/loftr/utils/geometry.py adapted from here |
| 160 | Args: |
| 161 | kpts0 (torch.Tensor): [N, L, 2] - <x, y>, should be normalized in (-1,1) |
| 162 | depth0 (torch.Tensor): [N, H, W], |
| 163 | depth1 (torch.Tensor): [N, H, W], |
| 164 | T_0to1 (torch.Tensor): [N, 3, 4], |
| 165 | K0 (torch.Tensor): [N, 3, 3], |
| 166 | K1 (torch.Tensor): [N, 3, 3], |
| 167 | Returns: |
| 168 | calculable_mask (torch.Tensor): [N, L] |
| 169 | warped_keypoints0 (torch.Tensor): [N, L, 2] <x0_hat, y1_hat> |
| 170 | """ |
| 171 | ( |
| 172 | n, |
| 173 | h, |
| 174 | w, |
| 175 | ) = depth0.shape |
| 176 | if depth_interpolation_mode == "combined": |
| 177 | # Inspired by approach in inloc, try to fill holes from bilinear interpolation by nearest neighbour interpolation |
| 178 | if smooth_mask: |
| 179 | raise NotImplementedError("Combined bilinear and NN warp not implemented") |
| 180 | valid_bilinear, warp_bilinear = warp_kpts(kpts0, depth0, depth1, T_0to1, K0, K1, |
| 181 | smooth_mask = smooth_mask, |
| 182 | return_relative_depth_error = return_relative_depth_error, |
| 183 | depth_interpolation_mode = "bilinear", |
| 184 | relative_depth_error_threshold = relative_depth_error_threshold) |
| 185 | valid_nearest, warp_nearest = warp_kpts(kpts0, depth0, depth1, T_0to1, K0, K1, |
| 186 | smooth_mask = smooth_mask, |
| 187 | return_relative_depth_error = return_relative_depth_error, |
| 188 | depth_interpolation_mode = "nearest-exact", |
| 189 | relative_depth_error_threshold = relative_depth_error_threshold) |
| 190 | nearest_valid_bilinear_invalid = (~valid_bilinear).logical_and(valid_nearest) |
| 191 | warp = warp_bilinear.clone() |
| 192 | warp[nearest_valid_bilinear_invalid] = warp_nearest[nearest_valid_bilinear_invalid] |
| 193 | valid = valid_bilinear | valid_nearest |
| 194 | return valid, warp |
| 195 | |
| 196 | |
| 197 | kpts0_depth = F.grid_sample(depth0[:, None], kpts0[:, :, None], mode = depth_interpolation_mode, align_corners=False)[ |
| 198 | :, 0, :, 0 |
| 199 | ] |
| 200 | kpts0 = torch.stack( |
| 201 | (w * (kpts0[..., 0] + 1) / 2, h * (kpts0[..., 1] + 1) / 2), dim=-1 |
| 202 | ) # [-1+1/h, 1-1/h] -> [0.5, h-0.5] |
| 203 | # Sample depth, get calculable_mask on depth != 0 |
| 204 | # nonzero_mask = kpts0_depth != 0 |
| 205 | # Sample depth, get calculable_mask on depth > 0 |
| 206 | nonzero_mask = kpts0_depth > 0 |
| 207 | |
| 208 | # Unproject |
| 209 | kpts0_h = ( |
| 210 | torch.cat([kpts0, torch.ones_like(kpts0[:, :, [0]])], dim=-1) |
| 211 | * kpts0_depth[..., None] |
| 212 | ) # (N, L, 3) |