| 481 | self.pad_sequences = pad_sequences |
| 482 | |
| 483 | def _tokenize(self, text_sample): |
| 484 | assert self.tokenizer is not None, "Tokenizer required if data is not pretokenized" |
| 485 | if self.tokenizer._pad_token is None: |
| 486 | # Some tokenizers (e.g. GPT2 tokenizer) have no padding token which causes bugs |
| 487 | raise RuntimeError("If tokenizing on-the-fly, tokenizer must have a pad_token_id") |
| 488 | |
| 489 | return self.tokenizer( |
| 490 | text_sample["text"], |
| 491 | truncation=True, |
| 492 | padding="max_length" if self.pad_sequences else False, |
| 493 | max_length=self.max_seq_len, |
| 494 | ) |
| 495 | |
| 496 | def __getitem__(self, index: int): |
| 497 | shard_id, shard_sample_id = self.spanner[index] |