| 136 | |
| 137 | |
| 138 | class MemoryEncoder(nn.Module): |
| 139 | def __init__( |
| 140 | self, |
| 141 | out_dim, |
| 142 | mask_downsampler, |
| 143 | fuser, |
| 144 | position_encoding, |
| 145 | in_dim=256, # in_dim of pix_feats |
| 146 | ): |
| 147 | super().__init__() |
| 148 | |
| 149 | self.mask_downsampler = mask_downsampler |
| 150 | |
| 151 | self.pix_feat_proj = nn.Conv2d(in_dim, in_dim, kernel_size=1) |
| 152 | self.fuser = fuser |
| 153 | self.position_encoding = position_encoding |
| 154 | self.out_proj = nn.Identity() |
| 155 | if out_dim != in_dim: |
| 156 | self.out_proj = nn.Conv2d(in_dim, out_dim, kernel_size=1) |
| 157 | |
| 158 | def forward( |
| 159 | self, |
| 160 | pix_feat: torch.Tensor, |
| 161 | masks: torch.Tensor, |
| 162 | skip_mask_sigmoid: bool = False, |
| 163 | ) -> Tuple[torch.Tensor, torch.Tensor]: |
| 164 | ## Process masks |
| 165 | # sigmoid, so that less domain shift from gt masks which are bool |
| 166 | if not skip_mask_sigmoid: |
| 167 | masks = F.sigmoid(masks) |
| 168 | masks = self.mask_downsampler(masks) |
| 169 | |
| 170 | ## Fuse pix_feats and downsampled masks |
| 171 | # in case the visual features are on CPU, cast them to CUDA |
| 172 | pix_feat = pix_feat.to(masks.device) |
| 173 | |
| 174 | x = self.pix_feat_proj(pix_feat) |
| 175 | x = x + masks |
| 176 | x = self.fuser(x) |
| 177 | x = self.out_proj(x) |
| 178 | |
| 179 | pos = self.position_encoding(x).to(x.dtype) |
| 180 | |
| 181 | return {"vision_features": x, "vision_pos_enc": [pos]} |
nothing calls this directly
no outgoing calls
no test coverage detected