(self, input, output, info, tokenizer, tokenize)
| 434 | print_rank_0(total_dict) |
| 435 | |
| 436 | def tokenize_worker(self, input, output, info, tokenizer, tokenize): |
| 437 | source_dict = defaultdict(int) |
| 438 | for row in iter(input.get, 'STOP'): |
| 439 | row = row.rstrip() |
| 440 | if row: |
| 441 | if self.is_json: |
| 442 | row = json.loads(row) |
| 443 | prompts, texts, source = self.process_line(row, tokenizer, tokenize) |
| 444 | length = 0 |
| 445 | for prompt, text in zip(prompts, texts): |
| 446 | length += len(text) |
| 447 | output.put((prompt, text)) |
| 448 | if source: |
| 449 | source_dict[source] += length |
| 450 | output.put("COMPLETE") |
| 451 | info.put(source_dict) |
| 452 | |
| 453 | def process_line(self, data, tokenizer, tokenize): |
| 454 | source = data["meta"].get("pile_set_name", None) |
nothing calls this directly
no test coverage detected