Post-processes VGGish embeddings. Returns a torch.Tensor instead of a numpy array in order to preserve the gradient. "The initial release of AudioSet included 128-D VGGish embeddings for each segment of AudioSet. These released embeddings were produced by applying a PCA transformati
| 40 | |
| 41 | |
| 42 | class Postprocessor(nn.Module): |
| 43 | """Post-processes VGGish embeddings. Returns a torch.Tensor instead of a |
| 44 | numpy array in order to preserve the gradient. |
| 45 | |
| 46 | "The initial release of AudioSet included 128-D VGGish embeddings for each |
| 47 | segment of AudioSet. These released embeddings were produced by applying |
| 48 | a PCA transformation (technically, a whitening transform is included as well) |
| 49 | and 8-bit quantization to the raw embedding output from VGGish, in order to |
| 50 | stay compatible with the YouTube-8M project which provides visual embeddings |
| 51 | in the same format for a large set of YouTube videos. This class implements |
| 52 | the same PCA (with whitening) and quantization transformations." |
| 53 | """ |
| 54 | |
| 55 | def __init__(self): |
| 56 | """Constructs a postprocessor.""" |
| 57 | super(Postprocessor, self).__init__() |
| 58 | # Create empty matrix, for user's state_dict to load |
| 59 | self.pca_eigen_vectors = torch.empty( |
| 60 | (vggish_params.EMBEDDING_SIZE, vggish_params.EMBEDDING_SIZE,), |
| 61 | dtype=torch.float, |
| 62 | ) |
| 63 | self.pca_means = torch.empty( |
| 64 | (vggish_params.EMBEDDING_SIZE, 1), dtype=torch.float |
| 65 | ) |
| 66 | |
| 67 | self.pca_eigen_vectors = nn.Parameter(self.pca_eigen_vectors, requires_grad=False) |
| 68 | self.pca_means = nn.Parameter(self.pca_means, requires_grad=False) |
| 69 | |
| 70 | def postprocess(self, embeddings_batch): |
| 71 | """Applies tensor postprocessing to a batch of embeddings. |
| 72 | |
| 73 | Args: |
| 74 | embeddings_batch: An tensor of shape [batch_size, embedding_size] |
| 75 | containing output from the embedding layer of VGGish. |
| 76 | |
| 77 | Returns: |
| 78 | A tensor of the same shape as the input, containing the PCA-transformed, |
| 79 | quantized, and clipped version of the input. |
| 80 | """ |
| 81 | assert len(embeddings_batch.shape) == 2, "Expected 2-d batch, got %r" % ( |
| 82 | embeddings_batch.shape, |
| 83 | ) |
| 84 | assert ( |
| 85 | embeddings_batch.shape[1] == vggish_params.EMBEDDING_SIZE |
| 86 | ), "Bad batch shape: %r" % (embeddings_batch.shape,) |
| 87 | |
| 88 | # Apply PCA. |
| 89 | # - Embeddings come in as [batch_size, embedding_size]. |
| 90 | # - Transpose to [embedding_size, batch_size]. |
| 91 | # - Subtract pca_means column vector from each column. |
| 92 | # - Premultiply by PCA matrix of shape [output_dims, input_dims] |
| 93 | # where both are are equal to embedding_size in our case. |
| 94 | # - Transpose result back to [batch_size, embedding_size]. |
| 95 | pca_applied = torch.mm(self.pca_eigen_vectors, (embeddings_batch.t() - self.pca_means)).t() |
| 96 | |
| 97 | # Quantize by: |
| 98 | # - clipping to [min, max] range |
| 99 | clipped_embeddings = torch.clamp( |