Function embed_array_storage

src/datasets/table.py:2153–2230

Embed data into an arrays's storage. For custom features like Audio or Image, it takes into account the "embed_storage" methods they define to embed external data (e.g. an image file) into an array. Args: array (`pa.Array`): The PyArrow

(
    array: pa.Array,
    feature: "FeatureType",
    token_per_repo_id=None,
    local_files: bool = True,
    remote_files: bool = True,
)

@_wrap_for_chunked_arrays
def embed_array_storage(
    array: pa.Array,
    feature: "FeatureType",
    token_per_repo_id=None,
    local_files: bool = True,
    remote_files: bool = True,
):
    """Embed data into an arrays's storage.
    For custom features like Audio or Image, it takes into account the "embed_storage" methods
    they define to embed external data (e.g. an image file) into an array.

    <Added version="2.4.0"/>

    Args:
        array (`pa.Array`):
            The PyArrow array in which to embed data.
        feature (`datasets.features.FeatureType`):
            Array features.
        local_files (`bool`, defaults to `True`)
            Whether to embed local files data in the array

            <Added version="4.8.5"/>
        remote_files (`bool`, defaults to `True`)
            Whether to embed remote files data in the array.
            E.g. files with paths that start with hf:// or https://

            <Added version="4.8.5"/>

    Raises:
        `TypeError`: if the target type is not supported according, e.g.

            - if a field is missing

    Returns:
         array (`pyarrow.Array`): the casted array
    """
    if not local_files and not remote_files:
        return array

    from .features import LargeList, List

    _e = partial(
        embed_array_storage, token_per_repo_id=token_per_repo_id, local_files=local_files, remote_files=remote_files
    )

    if isinstance(array, pa.ExtensionArray):
        array = array.storage
    if hasattr(feature, "embed_storage"):
        return feature.embed_storage(
            array, token_per_repo_id=token_per_repo_id, local_files=local_files, remote_files=remote_files
        )
    elif pa.types.is_struct(array.type):
        # feature must be a dict
        if isinstance(feature, dict):
            arrays = [_e(array.field(name), subfeature) for name, subfeature in feature.items()]
            return pa.StructArray.from_arrays(arrays, names=list(feature), mask=array.is_null())
    elif pa.types.is_list(array.type):
        # feature must be either List(subfeature)