(self, data, tokenizer, tokenize)
| 384 | assert_str = "make sure to set PATH for cc-news data_utils/corpora.py" |
| 385 | |
| 386 | def process_line(self, data, tokenizer, tokenize): |
| 387 | text = "" |
| 388 | title = data.get("title", None) |
| 389 | description = data.get("description", None) |
| 390 | maintext = data.get("maintext", None) |
| 391 | if title: |
| 392 | text += title.strip() + " " |
| 393 | if description and (not maintext or not maintext.startswith(description)): |
| 394 | text += description.strip() + " " |
| 395 | if maintext: |
| 396 | text += maintext |
| 397 | if len(text) > 100: |
| 398 | prompt, text = self.process_sample("", tokenizer, tokenize), self.process_sample(text, tokenizer, tokenize) |
| 399 | return [prompt], [text] |
| 400 | else: |
| 401 | return [], [] |
| 402 | |
| 403 | |
| 404 | class BertData(PromptReader): |
nothing calls this directly
no test coverage detected