| 329 | self.proj = DProjector(text_dim=token_dim, in_dim=token_dim) |
| 330 | |
| 331 | def forward(self, vis, text, pad_mask): |
| 332 | x_c4, x_c3, x_c2, x_c1 = vis |
| 333 | tokens = self.tokens.weight[None,...].expand(x_c1.shape[0], -1, -1) |
| 334 | maps = [] |
| 335 | v = x_c4 |
| 336 | for load, layer, fuse, v_ in zip(self.layers,[self.cgattention1,self.cgattention2,self.cgattention2], self.fuses, [x_c3, x_c2, x_c1]): |
| 337 | v = fuse(v, v_) |
| 338 | tokens, pe = load(tokens, text, pad_mask) |
| 339 | tokens, hitmap = layer(tokens, v, pe=pe) |
| 340 | maps.append(hitmap) |
| 341 | out = self.proj(v, tokens[:,-1]) |
| 342 | return out, maps |