Resize tokenizer and embedding. Note: This is the unoptimized version that may make your embedding size not be divisible by 64.
(
special_tokens_dict: Dict,
tokenizer: transformers.PreTrainedTokenizer,
model: transformers.PreTrainedModel,
)
| 111 | |
| 112 | |
| 113 | def smart_tokenizer_and_embedding_resize( |
| 114 | special_tokens_dict: Dict, |
| 115 | tokenizer: transformers.PreTrainedTokenizer, |
| 116 | model: transformers.PreTrainedModel, |
| 117 | ): |
| 118 | """Resize tokenizer and embedding. |
| 119 | |
| 120 | Note: This is the unoptimized version that may make your embedding size not be divisible by 64. |
| 121 | """ |
| 122 | num_new_tokens = tokenizer.add_special_tokens(special_tokens_dict) |
| 123 | model.resize_token_embeddings(len(tokenizer)) |
| 124 | |
| 125 | if num_new_tokens > 0: |
| 126 | input_embeddings = model.get_input_embeddings().weight.data |
| 127 | output_embeddings = model.get_output_embeddings().weight.data |
| 128 | |
| 129 | input_embeddings_avg = input_embeddings[:-num_new_tokens].mean(dim=0, keepdim=True) |
| 130 | output_embeddings_avg = output_embeddings[:-num_new_tokens].mean(dim=0, keepdim=True) |
| 131 | |
| 132 | input_embeddings[-num_new_tokens:] = input_embeddings_avg |
| 133 | output_embeddings[-num_new_tokens:] = output_embeddings_avg |
| 134 | |
| 135 | |
| 136 | def _tokenize_fn(strings: Sequence[str], tokenizer: transformers.PreTrainedTokenizer) -> Dict: |
no test coverage detected