Tokenize a list of strings.
(strings: Sequence[str], tokenizer: transformers.PreTrainedTokenizer)
| 134 | |
| 135 | |
| 136 | def _tokenize_fn(strings: Sequence[str], tokenizer: transformers.PreTrainedTokenizer) -> Dict: |
| 137 | """Tokenize a list of strings.""" |
| 138 | tokenized_list = [ |
| 139 | tokenizer( |
| 140 | text, |
| 141 | return_tensors="pt", |
| 142 | padding="longest", |
| 143 | max_length=tokenizer.model_max_length, |
| 144 | truncation=True, |
| 145 | ) |
| 146 | for text in strings |
| 147 | ] |
| 148 | input_ids = labels = [tokenized.input_ids[0] for tokenized in tokenized_list] |
| 149 | input_ids_lens = labels_lens = [ |
| 150 | tokenized.input_ids.ne(tokenizer.pad_token_id).sum().item() for tokenized in tokenized_list |
| 151 | ] |
| 152 | return dict( |
| 153 | input_ids=input_ids, |
| 154 | labels=labels, |
| 155 | input_ids_lens=input_ids_lens, |
| 156 | labels_lens=labels_lens, |
| 157 | ) |
| 158 | |
| 159 | |
| 160 | def preprocess( |