Function fake_target_chunksize

xarray/namedarray/utils.py:267–300

The `normalize_chunks` algorithm takes a size `limit` in bytes, but will not work for object dtypes. So we rescale the `limit` to an appropriate one based on `float64` dtype, and pass that to `normalize_chunks`. Arguments --------- data : Variable or ChunkedArray T

(
    data: DuckArray[Any],
    limit: int,
)

def fake_target_chunksize(
    data: DuckArray[Any],
    limit: int,
) -> tuple[int, np.dtype[Any]]:
    """
    The `normalize_chunks` algorithm takes a size `limit` in bytes, but will not
    work for object dtypes.  So we rescale the `limit` to an appropriate one based
    on `float64` dtype, and pass that to `normalize_chunks`.

    Arguments
    ---------
    data : Variable or ChunkedArray
        The data for which we want to determine chunk sizes.
    limit : int
        The target chunk size in bytes. Passed to the chunk manager's `normalize_chunks` method.
    """

    # Short circuit for non-object dtypes
    from xarray.core.common import _contains_cftime_datetimes

    if not _contains_cftime_datetimes(data):
        return limit, data.dtype

    from xarray.core.formatting import first_n_items

    output_dtype = np.dtype(np.float64)

    nbytes_approx: int = sys.getsizeof(first_n_items(data, 1))  # type: ignore[no-untyped-call]

    f64_nbytes = output_dtype.itemsize

    limit = int(limit * (f64_nbytes / nbytes_approx))

    return limit, output_dtype


class ReprObject: