Tokenize a list of strings.
(strings, tokenizer: transformers.PreTrainedTokenizer)
| 59 | output_embeddings[-num_new_tokens:] = output_embeddings_avg |
| 60 | |
| 61 | def _tokenize_fn(strings, tokenizer: transformers.PreTrainedTokenizer): |
| 62 | """Tokenize a list of strings.""" |
| 63 | tokenized_list = [ |
| 64 | tokenizer( |
| 65 | text, |
| 66 | return_tensors="pt", |
| 67 | padding="longest", |
| 68 | max_length=tokenizer.model_max_length, |
| 69 | truncation=True, |
| 70 | ) |
| 71 | for text in strings |
| 72 | ] |
| 73 | input_ids = labels = [tokenized.input_ids[0] for tokenized in tokenized_list] |
| 74 | input_ids_lens = labels_lens = [ |
| 75 | tokenized.input_ids.ne(tokenizer.pad_token_id).sum().item() for tokenized in tokenized_list |
| 76 | ] |
| 77 | return dict( |
| 78 | input_ids=input_ids, |
| 79 | labels=labels, |
| 80 | input_ids_lens=input_ids_lens, |
| 81 | labels_lens=labels_lens, |
| 82 | ) |
| 83 | |
| 84 | def preprocess( |
| 85 | sources, |