Applies tensor postprocessing to a batch of embeddings. Args: embeddings_batch: An tensor of shape [batch_size, embedding_size] containing output from the embedding layer of VGGish. Returns: A tensor of the same shape as the input, containing the PCA
(self, embeddings_batch)
| 68 | self.pca_means = nn.Parameter(self.pca_means, requires_grad=False) |
| 69 | |
| 70 | def postprocess(self, embeddings_batch): |
| 71 | """Applies tensor postprocessing to a batch of embeddings. |
| 72 | |
| 73 | Args: |
| 74 | embeddings_batch: An tensor of shape [batch_size, embedding_size] |
| 75 | containing output from the embedding layer of VGGish. |
| 76 | |
| 77 | Returns: |
| 78 | A tensor of the same shape as the input, containing the PCA-transformed, |
| 79 | quantized, and clipped version of the input. |
| 80 | """ |
| 81 | assert len(embeddings_batch.shape) == 2, "Expected 2-d batch, got %r" % ( |
| 82 | embeddings_batch.shape, |
| 83 | ) |
| 84 | assert ( |
| 85 | embeddings_batch.shape[1] == vggish_params.EMBEDDING_SIZE |
| 86 | ), "Bad batch shape: %r" % (embeddings_batch.shape,) |
| 87 | |
| 88 | # Apply PCA. |
| 89 | # - Embeddings come in as [batch_size, embedding_size]. |
| 90 | # - Transpose to [embedding_size, batch_size]. |
| 91 | # - Subtract pca_means column vector from each column. |
| 92 | # - Premultiply by PCA matrix of shape [output_dims, input_dims] |
| 93 | # where both are are equal to embedding_size in our case. |
| 94 | # - Transpose result back to [batch_size, embedding_size]. |
| 95 | pca_applied = torch.mm(self.pca_eigen_vectors, (embeddings_batch.t() - self.pca_means)).t() |
| 96 | |
| 97 | # Quantize by: |
| 98 | # - clipping to [min, max] range |
| 99 | clipped_embeddings = torch.clamp( |
| 100 | pca_applied, vggish_params.QUANTIZE_MIN_VAL, vggish_params.QUANTIZE_MAX_VAL |
| 101 | ) |
| 102 | # - convert to 8-bit in range [0.0, 255.0] |
| 103 | quantized_embeddings = torch.round( |
| 104 | (clipped_embeddings - vggish_params.QUANTIZE_MIN_VAL) |
| 105 | * ( |
| 106 | 255.0 |
| 107 | / (vggish_params.QUANTIZE_MAX_VAL - vggish_params.QUANTIZE_MIN_VAL) |
| 108 | ) |
| 109 | ) |
| 110 | return torch.squeeze(quantized_embeddings) |
| 111 | |
| 112 | def forward(self, x): |
| 113 | return self.postprocess(x) |