Build a resized Embedding Module from a provided token Embedding Module. Increasing the size will add newly initialized vectors at the end Reducing the size will remove vectors from the end Args: old_embeddings: ``torch.nn.Embedding`` Old
(
self, old_embeddings: torch.nn.Embedding, new_num_tokens: Optional[int] = None
)
| 407 | return self.get_input_embeddings() |
| 408 | |
| 409 | def _get_resized_embeddings( |
| 410 | self, old_embeddings: torch.nn.Embedding, new_num_tokens: Optional[int] = None |
| 411 | ) -> torch.nn.Embedding: |
| 412 | """ Build a resized Embedding Module from a provided token Embedding Module. |
| 413 | Increasing the size will add newly initialized vectors at the end |
| 414 | Reducing the size will remove vectors from the end |
| 415 | |
| 416 | Args: |
| 417 | old_embeddings: ``torch.nn.Embedding`` |
| 418 | Old embeddings to be resized. |
| 419 | new_num_tokens: (`optional`) int |
| 420 | New number of tokens in the embedding matrix. |
| 421 | Increasing the size will add newly initialized vectors at the end |
| 422 | Reducing the size will remove vectors from the end |
| 423 | If not provided or None: return the provided token Embedding Module. |
| 424 | Return: ``torch.nn.Embedding`` |
| 425 | Pointer to the resized Embedding Module or the old Embedding Module if new_num_tokens is None |
| 426 | """ |
| 427 | if new_num_tokens is None: |
| 428 | return old_embeddings |
| 429 | |
| 430 | old_num_tokens, old_embedding_dim = old_embeddings.weight.size() |
| 431 | if old_num_tokens == new_num_tokens: |
| 432 | return old_embeddings |
| 433 | |
| 434 | # Build new embeddings |
| 435 | new_embeddings = nn.Embedding(new_num_tokens, old_embedding_dim) |
| 436 | new_embeddings.to(old_embeddings.weight.device) |
| 437 | |
| 438 | # initialize all new embeddings (in particular added tokens) |
| 439 | self._init_weights(new_embeddings) |
| 440 | |
| 441 | # Copy token embeddings from the previous weights |
| 442 | num_tokens_to_copy = min(old_num_tokens, new_num_tokens) |
| 443 | new_embeddings.weight.data[:num_tokens_to_copy, :] = old_embeddings.weight.data[:num_tokens_to_copy, :] |
| 444 | |
| 445 | return new_embeddings |
| 446 | |
| 447 | def init_weights(self): |
| 448 | """ Initialize and prunes weights if needed. """ |
no test coverage detected