Tokenize a list of strings.
(strings: Sequence[str], tokenizer: transformers.PreTrainedTokenizer)
| 170 | ) |
| 171 | |
| 172 | def _tokenize_fn(strings: Sequence[str], tokenizer: transformers.PreTrainedTokenizer) -> Dict: |
| 173 | """Tokenize a list of strings.""" |
| 174 | tokenized_list = [ |
| 175 | tokenizer( |
| 176 | text, |
| 177 | return_tensors="pt", |
| 178 | padding="longest", |
| 179 | max_length=tokenizer.model_max_length, |
| 180 | truncation=True, |
| 181 | ) |
| 182 | for text in strings |
| 183 | ] |
| 184 | input_ids = labels = [tokenized.input_ids[0] for tokenized in tokenized_list] |
| 185 | input_ids_lens = labels_lens = [ |
| 186 | tokenized.input_ids.ne(tokenizer.pad_token_id).sum().item() for tokenized in tokenized_list |
| 187 | ] |
| 188 | return dict( |
| 189 | input_ids=input_ids, |
| 190 | labels=labels, |
| 191 | input_ids_lens=input_ids_lens, |
| 192 | labels_lens=labels_lens, |
| 193 | ) |
| 194 | |
| 195 | def preprocess( |
| 196 | sources: Sequence[str], |