Determine automatic chunks This takes in a chunks value that contains ``"auto"`` values in certain dimensions and replaces those values with concrete dimension sizes that try to get chunks to be of a certain size in bytes, provided by the ``limit=`` keyword. If multiple dimensions
(chunks, shape, limit, dtype, previous_chunks=None)
| 3253 | |
| 3254 | |
| 3255 | def auto_chunks(chunks, shape, limit, dtype, previous_chunks=None): |
| 3256 | """Determine automatic chunks |
| 3257 | |
| 3258 | This takes in a chunks value that contains ``"auto"`` values in certain |
| 3259 | dimensions and replaces those values with concrete dimension sizes that try |
| 3260 | to get chunks to be of a certain size in bytes, provided by the ``limit=`` |
| 3261 | keyword. If multiple dimensions are marked as ``"auto"`` then they will |
| 3262 | all respond to meet the desired byte limit, trying to respect the aspect |
| 3263 | ratio of their dimensions in ``previous_chunks=``, if given. |
| 3264 | |
| 3265 | Parameters |
| 3266 | ---------- |
| 3267 | chunks: Tuple |
| 3268 | A tuple of either dimensions or tuples of explicit chunk dimensions |
| 3269 | Some entries should be "auto" |
| 3270 | shape: Tuple[int] |
| 3271 | limit: int, str |
| 3272 | The maximum allowable size of a chunk in bytes |
| 3273 | previous_chunks: Tuple[Tuple[int]] |
| 3274 | |
| 3275 | See also |
| 3276 | -------- |
| 3277 | normalize_chunks: for full docstring and parameters |
| 3278 | """ |
| 3279 | if previous_chunks is not None: |
| 3280 | # rioxarray is passing ((1, ), (x,)) for shapes like (100, 5x), |
| 3281 | # so add this compat code for now |
| 3282 | # https://github.com/corteva/rioxarray/pull/820 |
| 3283 | previous_chunks = ( |
| 3284 | c[0] if isinstance(c, tuple) and len(c) == 1 else c for c in previous_chunks |
| 3285 | ) |
| 3286 | previous_chunks = _convert_int_chunk_to_tuple(shape, previous_chunks) |
| 3287 | chunks = list(chunks) |
| 3288 | |
| 3289 | autos = {i for i, c in enumerate(chunks) if c == "auto"} |
| 3290 | if not autos: |
| 3291 | return tuple(chunks) |
| 3292 | |
| 3293 | if limit is None: |
| 3294 | limit = config.get("array.chunk-size") |
| 3295 | if isinstance(limit, str): |
| 3296 | limit = parse_bytes(limit) |
| 3297 | |
| 3298 | if dtype is None: |
| 3299 | raise TypeError("dtype must be known for auto-chunking") |
| 3300 | |
| 3301 | if dtype.hasobject: |
| 3302 | raise NotImplementedError( |
| 3303 | "Can not use auto rechunking with object dtype. " |
| 3304 | "We are unable to estimate the size in bytes of object data" |
| 3305 | ) |
| 3306 | |
| 3307 | for x in tuple(chunks) + tuple(shape): |
| 3308 | if ( |
| 3309 | isinstance(x, Number) |
| 3310 | and np.isnan(x) |
| 3311 | or isinstance(x, tuple) |
| 3312 | and np.isnan(x).any() |
no test coverage detected