(case: PromptCase,
prompt_weights: Dict[str, torch.Tensor],
mask_weights: Dict[str, torch.Tensor],
no_mem_embed: torch.Tensor,
neck_trk_0: torch.Tensor,
neck_trk_1: torch.Tensor,
neck_trk_2: torch.Tensor,
outdir: str)
| 254 | |
| 255 | |
| 256 | def dump_case(case: PromptCase, |
| 257 | prompt_weights: Dict[str, torch.Tensor], |
| 258 | mask_weights: Dict[str, torch.Tensor], |
| 259 | no_mem_embed: torch.Tensor, |
| 260 | neck_trk_0: torch.Tensor, |
| 261 | neck_trk_1: torch.Tensor, |
| 262 | neck_trk_2: torch.Tensor, |
| 263 | outdir: str) -> None: |
| 264 | os.makedirs(outdir, exist_ok=True) |
| 265 | |
| 266 | sparse_embeddings, dense_embeddings, image_pe = build_prompt_encoder_outputs(prompt_weights, case) |
| 267 | |
| 268 | image_embeddings = neck_trk_2 + no_mem_embed.view(1, -1, 1, 1) |
| 269 | feat_s0 = F.conv2d( |
| 270 | neck_trk_0, |
| 271 | mask_weights["conv_s0.weight"].float(), |
| 272 | mask_weights["conv_s0.bias"].float(), |
| 273 | ) |
| 274 | feat_s1 = F.conv2d( |
| 275 | neck_trk_1, |
| 276 | mask_weights["conv_s1.weight"].float(), |
| 277 | mask_weights["conv_s1.bias"].float(), |
| 278 | ) |
| 279 | |
| 280 | save_ggml_bnd(os.path.join(outdir, "sam_pe_sparse"), sparse_embeddings) |
| 281 | save_ggml_nchw(os.path.join(outdir, "sam_pe_dense"), dense_embeddings) |
| 282 | save_ggml_nchw(os.path.join(outdir, "sam_pe_image_pe"), image_pe) |
| 283 | save_ggml_nchw(os.path.join(outdir, "sam_dec_image_feats"), image_embeddings) |
| 284 | |
| 285 | output_tokens = torch.cat( |
| 286 | [ |
| 287 | mask_weights["obj_score_token.weight"].float(), |
| 288 | mask_weights["iou_token.weight"].float(), |
| 289 | mask_weights["mask_tokens.weight"].float(), |
| 290 | ], |
| 291 | dim=0, |
| 292 | ).unsqueeze(0).expand(sparse_embeddings.size(0), -1, -1) |
| 293 | tokens = torch.cat((output_tokens, sparse_embeddings), dim=1) |
| 294 | save_ggml_bnd(os.path.join(outdir, "sam_dec_tokens_initial"), tokens) |
| 295 | |
| 296 | src = image_embeddings + dense_embeddings |
| 297 | pos_src = image_pe.repeat(tokens.shape[0], 1, 1, 1) |
| 298 | bsz, dim, h, w = src.shape |
| 299 | keys = src.flatten(2).permute(0, 2, 1) |
| 300 | key_pe = pos_src.flatten(2).permute(0, 2, 1) |
| 301 | queries = tokens |
| 302 | query_pe = tokens |
| 303 | |
| 304 | for idx in range(2): |
| 305 | prefix = f"transformer.layers.{idx}" |
| 306 | if idx == 0: |
| 307 | queries = attention_forward( |
| 308 | queries, queries, queries, prefix + ".self_attn", mask_weights, 8 |
| 309 | ) |
| 310 | else: |
| 311 | q = queries + query_pe |
| 312 | attn_out = attention_forward(q, q, queries, prefix + ".self_attn", mask_weights, 8) |
| 313 | queries = queries + attn_out |
no test coverage detected