Method process_line

data_utils/corpora.py:345–348

(self, data, tokenizer, tokenize)

    assert_str = "make sure to set PATH for wikipedia data_utils/corpora.py"

    def process_line(self, data, tokenizer, tokenize):
        text = data['text']
        prompt, text = self.process_sample("", tokenizer, tokenize), self.process_sample(text, tokenizer, tokenize)
        return [prompt], [text]


class TestDataset(PromptReader):