(self, iterator, batch_size)
| 396 | index += 1 |
| 397 | |
| 398 | def batched(self, iterator, batch_size): |
| 399 | batch = [] |
| 400 | for example in iterator: |
| 401 | batch.append(example) |
| 402 | if len(batch) == batch_size: |
| 403 | yield batch |
| 404 | batch = [] |
| 405 | if len(batch) > 0: |
| 406 | yield batch |
| 407 | |
| 408 | def parallel_example_iterator(self): |
| 409 | if self.config.tokenizer_processes == 1: |
no outgoing calls
no test coverage detected