Method process_line

data_utils/corpora.py:355–358

(self, data, tokenizer, tokenize)

    assert_str = "make sure to set PATH for wikipedia data_utils/corpora.py"

    def process_line(self, data, tokenizer, tokenize):
        prompt, text = data['prompt'], data['text']
        prompt, text = self.process_sample(prompt, tokenizer, tokenize), self.process_sample(text, tokenizer, tokenize)
        return [prompt], [text]


class OpenWebText(PromptReader):