Read streaming batches from a Parquet file. Parameters ---------- batch_size : int, default 64K Maximum number of records to yield per batch. Batches may be smaller if there aren't enough rows in the file. row_groups : list
(self, batch_size=65536, row_groups=None, columns=None,
use_threads=True, use_pandas_metadata=False)
| 532 | use_threads=use_threads) |
| 533 | |
| 534 | def iter_batches(self, batch_size=65536, row_groups=None, columns=None, |
| 535 | use_threads=True, use_pandas_metadata=False): |
| 536 | """ |
| 537 | Read streaming batches from a Parquet file. |
| 538 | |
| 539 | Parameters |
| 540 | ---------- |
| 541 | batch_size : int, default 64K |
| 542 | Maximum number of records to yield per batch. Batches may be |
| 543 | smaller if there aren't enough rows in the file. |
| 544 | row_groups : list |
| 545 | Only these row groups will be read from the file. |
| 546 | columns : list |
| 547 | If not None, only these columns will be read from the file. A |
| 548 | column name may be a prefix of a nested field, e.g. 'a' will select |
| 549 | 'a.b', 'a.c', and 'a.d.e'. |
| 550 | use_threads : boolean, default True |
| 551 | Perform multi-threaded column reads. |
| 552 | use_pandas_metadata : boolean, default False |
| 553 | If True and file has custom pandas schema metadata, ensure that |
| 554 | index columns are also loaded. |
| 555 | |
| 556 | Yields |
| 557 | ------ |
| 558 | pyarrow.RecordBatch |
| 559 | Contents of each batch as a record batch |
| 560 | |
| 561 | Examples |
| 562 | -------- |
| 563 | Generate an example Parquet file: |
| 564 | |
| 565 | >>> import pyarrow as pa |
| 566 | >>> table = pa.table({'n_legs': [2, 2, 4, 4, 5, 100], |
| 567 | ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", |
| 568 | ... "Brittle stars", "Centipede"]}) |
| 569 | >>> import pyarrow.parquet as pq |
| 570 | >>> pq.write_table(table, 'example.parquet') |
| 571 | >>> parquet_file = pq.ParquetFile('example.parquet') |
| 572 | >>> for i in parquet_file.iter_batches(): |
| 573 | ... print("RecordBatch") |
| 574 | ... print(i.to_pandas()) |
| 575 | ... |
| 576 | RecordBatch |
| 577 | n_legs animal |
| 578 | 0 2 Flamingo |
| 579 | 1 2 Parrot |
| 580 | 2 4 Dog |
| 581 | 3 4 Horse |
| 582 | 4 5 Brittle stars |
| 583 | 5 100 Centipede |
| 584 | """ |
| 585 | if batch_size <= 0: |
| 586 | raise ValueError("batch_size must be greater than zero") |
| 587 | |
| 588 | if row_groups is None: |
| 589 | row_groups = range(0, self.metadata.num_row_groups) |
| 590 | column_indices = self._get_column_indices( |
| 591 | columns, use_pandas_metadata=use_pandas_metadata) |