(self, data, tokenizer, tokenize)
| 451 | info.put(source_dict) |
| 452 | |
| 453 | def process_line(self, data, tokenizer, tokenize): |
| 454 | source = data["meta"].get("pile_set_name", None) |
| 455 | text = data.get("text", None) |
| 456 | if source and text: |
| 457 | if source in self.filtered_sources: |
| 458 | return [], [], None |
| 459 | elif source in self.downsample_sources and random.random() > self.downsample_sources[source]: |
| 460 | return [], [], None |
| 461 | else: |
| 462 | prompt, text = self.process_sample("", tokenizer, tokenize), self.process_sample(text, tokenizer, |
| 463 | tokenize) |
| 464 | return [prompt], [text], source |
| 465 | else: |
| 466 | return [], [], None |
| 467 | |
| 468 | |
| 469 | class Stories(PromptReader): |
no test coverage detected