Read streaming batches from a Parquet file. Parameters ---------- batch_size : int, default 64K Maximum number of records to yield per batch. Batches may be smaller if there aren't enough rows in the file. row_groups : list
(self, batch_size=65536, row_groups=None, columns=None,
use_threads=True, use_pandas_metadata=False)
| 538 | use_threads=use_threads) |
| 539 | |
| 540 | def iter_batches(self, batch_size=65536, row_groups=None, columns=None, |
| 541 | use_threads=True, use_pandas_metadata=False): |
| 542 | """ |
| 543 | Read streaming batches from a Parquet file. |
| 544 | |
| 545 | Parameters |
| 546 | ---------- |
| 547 | batch_size : int, default 64K |
| 548 | Maximum number of records to yield per batch. Batches may be |
| 549 | smaller if there aren't enough rows in the file. |
| 550 | row_groups : list |
| 551 | Only these row groups will be read from the file. |
| 552 | columns : list |
| 553 | If not None, only these columns will be read from the file. A |
| 554 | column name may be a prefix of a nested field, e.g. 'a' will select |
| 555 | 'a.b', 'a.c', and 'a.d.e'. |
| 556 | use_threads : boolean, default True |
| 557 | Perform multi-threaded column reads. |
| 558 | use_pandas_metadata : boolean, default False |
| 559 | If True and file has custom pandas schema metadata, ensure that |
| 560 | index columns are also loaded. |
| 561 | |
| 562 | Yields |
| 563 | ------ |
| 564 | pyarrow.RecordBatch |
| 565 | Contents of each batch as a record batch |
| 566 | |
| 567 | Examples |
| 568 | -------- |
| 569 | Generate an example Parquet file: |
| 570 | |
| 571 | >>> import pyarrow as pa |
| 572 | >>> table = pa.table({'n_legs': [2, 2, 4, 4, 5, 100], |
| 573 | ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", |
| 574 | ... "Brittle stars", "Centipede"]}) |
| 575 | >>> import pyarrow.parquet as pq |
| 576 | >>> pq.write_table(table, 'example.parquet') |
| 577 | >>> parquet_file = pq.ParquetFile('example.parquet') |
| 578 | >>> for i in parquet_file.iter_batches(): |
| 579 | ... print("RecordBatch") |
| 580 | ... print(i.to_pandas()) |
| 581 | ... |
| 582 | RecordBatch |
| 583 | n_legs animal |
| 584 | 0 2 Flamingo |
| 585 | 1 2 Parrot |
| 586 | 2 4 Dog |
| 587 | 3 4 Horse |
| 588 | 4 5 Brittle stars |
| 589 | 5 100 Centipede |
| 590 | """ |
| 591 | if batch_size <= 0: |
| 592 | raise ValueError("batch_size must be greater than zero") |
| 593 | |
| 594 | if row_groups is None: |
| 595 | row_groups = range(0, self.metadata.num_row_groups) |
| 596 | column_indices = self._get_column_indices( |
| 597 | columns, use_pandas_metadata=use_pandas_metadata) |