(self, iterator, batch_size)
| 604 | index += 1 |
| 605 | |
| 606 | def batched(self, iterator, batch_size): |
| 607 | batch = [] |
| 608 | for example in iterator: |
| 609 | batch.append(example) |
| 610 | if len(batch) == batch_size: |
| 611 | yield batch |
| 612 | batch = [] |
| 613 | if len(batch) > 0: |
| 614 | yield batch |
| 615 | |
| 616 | def parallel_example_iterator(self): |
| 617 | if self.config.tokenizer_processes == 1: |
no outgoing calls
no test coverage detected