Tokenize a list of strings.
(strings, tokenizer: transformers.PreTrainedTokenizer)
| 70 | |
| 71 | |
| 72 | def _tokenize_fn(strings, tokenizer: transformers.PreTrainedTokenizer): |
| 73 | """Tokenize a list of strings.""" |
| 74 | tokenized_list = [ |
| 75 | tokenizer( |
| 76 | text, |
| 77 | return_tensors="pt", |
| 78 | padding="longest", |
| 79 | max_length=tokenizer.model_max_length, |
| 80 | truncation=True, |
| 81 | ) |
| 82 | for text in strings |
| 83 | ] |
| 84 | input_ids = labels = [tokenized.input_ids[0] for tokenized in tokenized_list] |
| 85 | input_ids_lens = labels_lens = [ |
| 86 | tokenized.input_ids.ne(tokenizer.pad_token_id).sum().item() for tokenized in tokenized_list |
| 87 | ] |
| 88 | return dict( |
| 89 | input_ids=input_ids, |
| 90 | labels=labels, |
| 91 | input_ids_lens=input_ids_lens, |
| 92 | labels_lens=labels_lens, |
| 93 | ) |
| 94 | |
| 95 | |
| 96 | def preprocess( |