Method forward

preprocess/auxiliary/AutoShot.py:272–306

(self, inputs)

            self.device = "cpu"

    def forward(self, inputs):
        # pt version [BS, C, N, H, W], so [3, 4] means apply avg on spatial dim, out dim is [BS, C, N]
        x = torch.cat([torch.mean(x, dim=[3, 4]) for x in inputs], dim=1)

        if self.stop_gradient:
            x = x.detach()

        x = x.permute(dims=[0, 2, 1])  # out is [BS, N ,C]
        batch_size, time_window, old_channels = x.shape
        x = x.reshape(shape=[batch_size * time_window, old_channels])  # [BS X N, C]
        x = self.projection(x)
        x = F.normalize(x, p=2, dim=1)  # norm at C dim

        _, new_channels = x.shape
        x = x.reshape(shape=[batch_size, time_window, new_channels])
        y = x.permute(dims=[0, 2, 1])
        similarities = torch.matmul(x, y)  # [batch_size, time_window, time_window]
        # note that it operates on dimensions of the input tensor in a backward fashion (from last dimension to the first dimension)
        similarities_padded = F.pad(similarities,
                                    pad=[(self.lookup_window - 1) // 2, (self.lookup_window - 1) // 2, 0, 0, 0, 0])

        batch_indices = torch.arange(0, batch_size, device=self.device). \
            reshape(shape=[batch_size, 1, 1]). \
            repeat([1, time_window, self.lookup_window])
        time_indices = torch.arange(0, time_window, device=self.device). \
            reshape(shape=[1, time_window, 1]). \
            repeat([batch_size, 1, self.lookup_window])
        lookup_indices = torch.arange(0, self.lookup_window, device=self.device). \
                             reshape(shape=[1, 1, self.lookup_window]). \
                             repeat([batch_size, time_window, 1]) + time_indices

        indices = torch.stack([batch_indices, time_indices, lookup_indices], dim=-1)

        similarities = gather_nd(similarities_padded, indices)
        return self.fc(similarities)


class ColorHistograms(nn.Module):