Resize tokenizer and embedding. Note: This is the unoptimized version that may make your embedding size not be divisible by 64.
(
special_tokens_dict: Dict,
tokenizer: transformers.PreTrainedTokenizer,
model: transformers.PreTrainedModel,
)
| 37 | return gathered_s |
| 38 | |
| 39 | def smart_tokenizer_and_embedding_resize( |
| 40 | special_tokens_dict: Dict, |
| 41 | tokenizer: transformers.PreTrainedTokenizer, |
| 42 | model: transformers.PreTrainedModel, |
| 43 | ): |
| 44 | """Resize tokenizer and embedding. |
| 45 | |
| 46 | Note: This is the unoptimized version that may make your embedding size not be divisible by 64. |
| 47 | """ |
| 48 | num_new_tokens = tokenizer.add_special_tokens(special_tokens_dict) |
| 49 | model.resize_token_embeddings(len(tokenizer)) |
| 50 | |
| 51 | if num_new_tokens > 0: |
| 52 | input_embeddings = model.get_input_embeddings().weight.data |
| 53 | output_embeddings = model.get_output_embeddings().weight.data |
| 54 | |
| 55 | input_embeddings_avg = input_embeddings[:-num_new_tokens].mean(dim=0, keepdim=True) |
| 56 | output_embeddings_avg = output_embeddings[:-num_new_tokens].mean(dim=0, keepdim=True) |
| 57 | |
| 58 | input_embeddings[-num_new_tokens:] = input_embeddings_avg |
| 59 | output_embeddings[-num_new_tokens:] = output_embeddings_avg |
| 60 | |
| 61 | def _tokenize_fn(strings, tokenizer: transformers.PreTrainedTokenizer): |
| 62 | """Tokenize a list of strings.""" |
no test coverage detected