(self, data, tokenizer, tokenize)
| 323 | assert_str = "make sure to set PATH for baike data_utils/corpora.py" |
| 324 | |
| 325 | def process_line(self, data, tokenizer, tokenize): |
| 326 | prompts, texts = [], [] |
| 327 | text = data.get("title", "") + data.get("abstract", "") + data.get("content", "") |
| 328 | if text: |
| 329 | p, t = self.process_sample("", tokenizer, tokenize), self.process_sample(text, tokenizer, tokenize) |
| 330 | prompts.append(p) |
| 331 | texts.append(t) |
| 332 | return prompts, texts |
| 333 | |
| 334 | |
| 335 | class wikipedia(PromptReader): |
nothing calls this directly
no test coverage detected