Method forward

script/feature/efficientnet.py:88–147

inference DFNet. It can regress camera pose as well as extract intermediate layer features. :param x: image blob (2B x C x H x W) two stream or (B x C x H x W) single stream :param return_feature: whether to return features as output :param isSingleStream

(self, x, return_feature=False, isSingleStream=False, upsampleH=120, upsampleW=213)

        self.fc_pose = nn.Linear(1536, feat_dim)

    def forward(self, x, return_feature=False, isSingleStream=False, upsampleH=120, upsampleW=213):
        '''
        inference DFNet. It can regress camera pose as well as extract intermediate layer features.
            :param x: image blob (2B x C x H x W) two stream or (B x C x H x W) single stream
            :param return_feature: whether to return features as output
            :param isSingleStream: whether it's an single stream inference or siamese network inference
            :param upsampleH: feature upsample size H
            :param upsampleW: feature upsample size W
            :return feature_maps: (2, [B, C, H, W]) or (1, [B, C, H, W]) or None
            :return predict: [2B, 12] or [B, 12]
        '''
        # normalize input data
        mean, std = x.new_tensor(self.mean), x.new_tensor(self.std)
        x = (x - mean[:, None, None]) / std[:, None, None]

        ### encoder ###
        feature_maps = []
        list_x = self.feature_extractor(x)

        x = list_x['reduction_6'] # features to save
        for i in self.feature_block_index:
            fe = list_x['reduction_'+str(i)].clone()
            feature_maps.append(fe)

        ### extract and process intermediate features ###
        if return_feature:
            feature_maps = self.adaptation_layers(feature_maps) # (3, [B, C, H', W']), H', W' are different in each layer

            pdb.set_trace()
            if isSingleStream: # not siamese network style inference
                feature_stacks = []

                for f in feature_maps:
                    feature_stacks.append(torch.nn.UpsamplingBilinear2d(size=(upsampleH, upsampleW))(f))
                feature_maps = [torch.stack(feature_stacks)] # (1, [3, B, C, H, W])
            else: # siamese network style inference
                feature_stacks_t = []
                feature_stacks_r = []

                for f in feature_maps:
                    # split real and nerf batches
                    batch = f.shape[0] # should be target batch_size + rgb batch_size
                    feature_t = f[:batch//2]
                    feature_r = f[batch//2:]

                    feature_stacks_t.append(torch.nn.UpsamplingBilinear2d(size=(upsampleH, upsampleW))(feature_t)) # GT img
                    feature_stacks_r.append(torch.nn.UpsamplingBilinear2d(size=(upsampleH, upsampleW))(feature_r)) # render img
                feature_stacks_t = torch.stack(feature_stacks_t) # [3, B, C, H, W]
                feature_stacks_r = torch.stack(feature_stacks_r) # [3, B, C, H, W]
                feature_maps = [feature_stacks_t, feature_stacks_r] # (2, [3, B, C, H, W])

        else:
            feature_maps = None

        ### pose regression head ###
        x = self.avgpool(x)
        x = x.reshape(x.size(0), -1)
        predict = self.fc_pose(x)