Function from_arrow

python/ray/data/read_api.py:3632–3716

Create a :class:`~ray.data.Dataset` from a list of PyArrow tables. Examples: >>> import pyarrow as pa >>> import ray >>> table = pa.table({"x": [1]}) >>> ray.data.from_arrow(table) # doctest: +ELLIPSIS shape: (1, 1) ╭───────╮ │ x │

(
    tables: Union["pyarrow.Table", bytes, List[Union["pyarrow.Table", bytes]]],
    *,
    override_num_blocks: Optional[int] = None,
)

@PublicAPI
def from_arrow(
    tables: Union["pyarrow.Table", bytes, List[Union["pyarrow.Table", bytes]]],
    *,
    override_num_blocks: Optional[int] = None,
) -> MaterializedDataset:
    """Create a :class:`~ray.data.Dataset` from a list of PyArrow tables.

    Examples:
        >>> import pyarrow as pa
        >>> import ray
        >>> table = pa.table({"x": [1]})
        >>> ray.data.from_arrow(table)  # doctest: +ELLIPSIS
        shape: (1, 1)
        ╭───────╮
        │ x     │
        │ ---   │
        │ int64 │
        ╞═══════╡
        │ 1     │
        ╰───────╯
        (Showing 1 of 1 rows)

        Create a Ray Dataset from a list of PyArrow tables.

        >>> ray.data.from_arrow([table, table])  # doctest: +ELLIPSIS
        shape: (2, 1)
        ╭───────╮
        │ x     │
        │ ---   │
        │ int64 │
        ╞═══════╡
        │ 1     │
        │ 1     │
        ╰───────╯
        (Showing 2 of 2 rows)


    Args:
        tables: A PyArrow table, or a list of PyArrow tables,
                or its streaming format in bytes.
        override_num_blocks: Override the number of output blocks from all read tasks.
            By default, the number of output blocks is dynamically decided based on
            input data size and available resources. You shouldn't manually set this
            value in most cases.

    Returns:
        :class:`~ray.data.Dataset` holding data from the PyArrow tables.
    """
    import builtins

    import pyarrow as pa

    if isinstance(tables, (pa.Table, bytes)):
        tables = [tables]

    if override_num_blocks is not None:
        if override_num_blocks <= 0:
            raise ValueError("override_num_blocks must be > 0")