Method _tokenize

src/text_data.py:205–210

(self, text_sample)

    # How to tokenize a text sample to a token sample
    def _tokenize(self, text_sample):
        if self.tokenizer._pad_token is None:
            # Some tokenizers (e.g. GPT2 tokenizer) have no padding token which causes bugs
            raise RuntimeError("If tokenizing on-the-fly, tokenizer must have a pad_token_id")

        return self.tokenizer(text_sample["text"], truncation=True, padding="max_length", max_length=self.max_seq_len)

    def _read_binary_tokenized_sample(self, sample: BatchEncoding):
        seq_len = sample["len"] if "len" in sample else len(sample["input_ids"])