(self)
| 614 | yield batch |
| 615 | |
| 616 | def parallel_example_iterator(self): |
| 617 | if self.config.tokenizer_processes == 1: |
| 618 | for example, loc, index in self.json_iterator(): |
| 619 | self._file_loc = loc |
| 620 | self._index = index |
| 621 | yield self.text_processor((example, loc, index), has_aux=True) |
| 622 | else: |
| 623 | process_pool = Pool(self.config.tokenizer_processes) |
| 624 | batched_iterator = self.batched( |
| 625 | self.json_iterator(), self.config.tokenizer_parallel_batch_size |
| 626 | ) |
| 627 | with process_pool as pool: |
| 628 | map_fn = partial(self.text_processor, has_aux=True) |
| 629 | next_batch = pool.map_async( |
| 630 | map_fn, next(batched_iterator), |
| 631 | chunksize=self.config.tokenizer_parallel_chunk_size |
| 632 | ) |
| 633 | while True: |
| 634 | current_batch = next_batch |
| 635 | next_batch = pool.map_async( |
| 636 | map_fn, next(batched_iterator), |
| 637 | chunksize=self.config.tokenizer_parallel_chunk_size |
| 638 | ) |
| 639 | for example in current_batch.get(): |
| 640 | yield example |
| 641 | |
| 642 | def __iter__(self): |
| 643 | if self.config.mode == 'pad': |
no test coverage detected