Read (multiple) Parquet files as a single pyarrow.Table. Parameters ---------- columns : List[str] Names of columns to read from the dataset. The partition fields are not automatically included. use_threads : bool, default True
(self, columns=None, use_threads=True, use_pandas_metadata=False)
| 1538 | return self._dataset.schema |
| 1539 | |
| 1540 | def read(self, columns=None, use_threads=True, use_pandas_metadata=False): |
| 1541 | """ |
| 1542 | Read (multiple) Parquet files as a single pyarrow.Table. |
| 1543 | |
| 1544 | Parameters |
| 1545 | ---------- |
| 1546 | columns : List[str] |
| 1547 | Names of columns to read from the dataset. The partition fields |
| 1548 | are not automatically included. |
| 1549 | use_threads : bool, default True |
| 1550 | Perform multi-threaded column reads. |
| 1551 | use_pandas_metadata : bool, default False |
| 1552 | If True and file has custom pandas schema metadata, ensure that |
| 1553 | index columns are also loaded. |
| 1554 | |
| 1555 | Returns |
| 1556 | ------- |
| 1557 | pyarrow.Table |
| 1558 | Content of the file as a table (of columns). |
| 1559 | |
| 1560 | Examples |
| 1561 | -------- |
| 1562 | Generate an example dataset: |
| 1563 | |
| 1564 | >>> import pyarrow as pa |
| 1565 | >>> table = pa.table({'year': [2020, 2022, 2021, 2022, 2019, 2021], |
| 1566 | ... 'n_legs': [2, 2, 4, 4, 5, 100], |
| 1567 | ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", |
| 1568 | ... "Brittle stars", "Centipede"]}) |
| 1569 | >>> import pyarrow.parquet as pq |
| 1570 | >>> pq.write_to_dataset(table, root_path='dataset_v2_read', |
| 1571 | ... partition_cols=['year']) |
| 1572 | >>> dataset = pq.ParquetDataset('dataset_v2_read/') |
| 1573 | |
| 1574 | Read the dataset: |
| 1575 | |
| 1576 | >>> dataset.read(columns=["n_legs"]) |
| 1577 | pyarrow.Table |
| 1578 | n_legs: int64 |
| 1579 | ---- |
| 1580 | n_legs: [[5],[2],[4,100],[2,4]] |
| 1581 | """ |
| 1582 | # if use_pandas_metadata, we need to include index columns in the |
| 1583 | # column selection, to be able to restore those in the pandas DataFrame |
| 1584 | metadata = self.schema.metadata or {} |
| 1585 | |
| 1586 | common_metadata = None |
| 1587 | if use_pandas_metadata: |
| 1588 | # if the dataset schema metadata itself doesn't have pandas |
| 1589 | # then try to get this from common file (for backwards compat) |
| 1590 | if b"pandas" not in metadata: |
| 1591 | common_metadata = self._get_common_pandas_metadata() |
| 1592 | if common_metadata: |
| 1593 | metadata = common_metadata |
| 1594 | |
| 1595 | if columns is not None and use_pandas_metadata: |
| 1596 | if metadata and b'pandas' in metadata: |
| 1597 | # RangeIndex can be represented as dict instead of column name |