Function from_arrow_refs

python/ray/data/read_api.py:3720–3777

Create a :class:`~ray.data.Dataset` from a list of Ray object references to PyArrow tables. Examples: >>> import pyarrow as pa >>> import ray >>> table_ref = ray.put(pa.table({"x": [1]})) >>> ray.data.from_arrow_refs(table_ref) # doctest: +ELLIPSIS s

(
    tables: Union[
        ObjectRef[Union["pyarrow.Table", bytes]],
        List[ObjectRef[Union["pyarrow.Table", bytes]]],
    ],
)

@DeveloperAPI
def from_arrow_refs(
    tables: Union[
        ObjectRef[Union["pyarrow.Table", bytes]],
        List[ObjectRef[Union["pyarrow.Table", bytes]]],
    ],
) -> MaterializedDataset:
    """Create a :class:`~ray.data.Dataset` from a list of Ray object references to
    PyArrow tables.

    Examples:
        >>> import pyarrow as pa
        >>> import ray
        >>> table_ref = ray.put(pa.table({"x": [1]}))
        >>> ray.data.from_arrow_refs(table_ref)  # doctest: +ELLIPSIS
        shape: (1, 1)
        ╭───────╮
        │ x     │
        │ ---   │
        │ int64 │
        ╞═══════╡
        │ 1     │
        ╰───────╯
        (Showing 1 of 1 rows)

        Create a Ray Dataset from a list of PyArrow table references

        >>> ray.data.from_arrow_refs([table_ref, table_ref])  # doctest: +ELLIPSIS
        shape: (2, 1)
        ╭───────╮
        │ x     │
        │ ---   │
        │ int64 │
        ╞═══════╡
        │ 1     │
        │ 1     │
        ╰───────╯
        (Showing 2 of 2 rows)


    Args:
        tables: A Ray object reference to Arrow table, or list of Ray object
                references to Arrow tables, or its streaming format in bytes.

    Returns:
         :class:`~ray.data.Dataset` holding data read from the tables.
    """
    if isinstance(tables, ray.ObjectRef):
        tables = [tables]

    get_metadata_schema = cached_remote_fn(get_table_block_metadata_schema)
    label_selector = DataContext.get_current().execution_options.label_selector
    if label_selector:
        get_metadata_schema = get_metadata_schema.options(label_selector=label_selector)
    metadata_schema = ray.get([get_metadata_schema.remote(t) for t in tables])
    stats = DatasetStats(metadata={"FromArrow": metadata_schema}, parent=None)
    context = DataContext.get_current().copy()
    logical_plan = LogicalPlan(FromArrow(tables, metadata_schema), context)
    return MaterializedDataset(logical_plan, context, stats)