Method process_line

data_utils/corpora.py:473–480

(self, data, tokenizer, tokenize)

    PATH = "/dataset/fd5061f6/english_data/stories_31G.jsonl"

    def process_line(self, data, tokenizer, tokenize):
        text = data.get("text", None)
        if text:
            prompt, text = self.process_sample("", tokenizer, tokenize), self.process_sample(text, tokenizer,
                                                                                             tokenize)
            return [prompt], [text]
        else:
            return [], []


class BertBaseData(BertData):