Read (multiple) Parquet files as a single pyarrow.Table. Parameters ---------- columns : List[str] Names of columns to read from the dataset. The partition fields are not automatically included. use_threads : bool, default True
(self, columns=None, use_threads=True, use_pandas_metadata=False)
| 1523 | return self._dataset.schema |
| 1524 | |
| 1525 | def read(self, columns=None, use_threads=True, use_pandas_metadata=False): |
| 1526 | """ |
| 1527 | Read (multiple) Parquet files as a single pyarrow.Table. |
| 1528 | |
| 1529 | Parameters |
| 1530 | ---------- |
| 1531 | columns : List[str] |
| 1532 | Names of columns to read from the dataset. The partition fields |
| 1533 | are not automatically included. |
| 1534 | use_threads : bool, default True |
| 1535 | Perform multi-threaded column reads. |
| 1536 | use_pandas_metadata : bool, default False |
| 1537 | If True and file has custom pandas schema metadata, ensure that |
| 1538 | index columns are also loaded. |
| 1539 | |
| 1540 | Returns |
| 1541 | ------- |
| 1542 | pyarrow.Table |
| 1543 | Content of the file as a table (of columns). |
| 1544 | |
| 1545 | Examples |
| 1546 | -------- |
| 1547 | Generate an example dataset: |
| 1548 | |
| 1549 | >>> import pyarrow as pa |
| 1550 | >>> table = pa.table({'year': [2020, 2022, 2021, 2022, 2019, 2021], |
| 1551 | ... 'n_legs': [2, 2, 4, 4, 5, 100], |
| 1552 | ... 'animal': ["Flamingo", "Parrot", "Dog", "Horse", |
| 1553 | ... "Brittle stars", "Centipede"]}) |
| 1554 | >>> import pyarrow.parquet as pq |
| 1555 | >>> pq.write_to_dataset(table, root_path='dataset_v2_read', |
| 1556 | ... partition_cols=['year']) |
| 1557 | >>> dataset = pq.ParquetDataset('dataset_v2_read/') |
| 1558 | |
| 1559 | Read the dataset: |
| 1560 | |
| 1561 | >>> dataset.read(columns=["n_legs"]) |
| 1562 | pyarrow.Table |
| 1563 | n_legs: int64 |
| 1564 | ---- |
| 1565 | n_legs: [[5],[2],[4,100],[2,4]] |
| 1566 | """ |
| 1567 | # if use_pandas_metadata, we need to include index columns in the |
| 1568 | # column selection, to be able to restore those in the pandas DataFrame |
| 1569 | metadata = self.schema.metadata or {} |
| 1570 | |
| 1571 | if use_pandas_metadata: |
| 1572 | # if the dataset schema metadata itself doesn't have pandas |
| 1573 | # then try to get this from common file (for backwards compat) |
| 1574 | if b"pandas" not in metadata: |
| 1575 | common_metadata = self._get_common_pandas_metadata() |
| 1576 | if common_metadata: |
| 1577 | metadata = common_metadata |
| 1578 | |
| 1579 | if columns is not None and use_pandas_metadata: |
| 1580 | if metadata and b'pandas' in metadata: |
| 1581 | # RangeIndex can be represented as dict instead of column name |
| 1582 | index_columns = [ |