Function tokenize

dask/tokenize.py:47–80

Deterministic token >>> tokenize([1, 2, '3']) # doctest: +SKIP '06961e8de572e73c2e74b51348177918' >>> tokenize('Hello') == tokenize('Hello') True Parameters ---------- args, kwargs: objects to tokenize ensure_deterministic: bool, optional If True,

(
    *args: object, ensure_deterministic: bool | None = None, **kwargs: object
)

def tokenize(
    *args: object, ensure_deterministic: bool | None = None, **kwargs: object
) -> str:
    """Deterministic token

    >>> tokenize([1, 2, '3'])  # doctest: +SKIP
    '06961e8de572e73c2e74b51348177918'

    >>> tokenize('Hello') == tokenize('Hello')
    True

    Parameters
    ----------
    args, kwargs:
        objects to tokenize
    ensure_deterministic: bool, optional
        If True, raise TokenizationError if the objects cannot be deterministically
        tokenized, e.g. two identical objects will return different tokens.
        Defaults to the `tokenize.ensure-deterministic` configuration parameter.
    """
    global _SEEN
    with tokenize_lock:
        seen_before, _SEEN = _SEEN, {}
        token = None
        try:
            _ENSURE_DETERMINISTIC.get()
        except LookupError:
            token = _ENSURE_DETERMINISTIC.set(ensure_deterministic)
        try:
            return _tokenize(*args, **kwargs)
        finally:
            if token:
                _ENSURE_DETERMINISTIC.reset(token)
            _SEEN = seen_before


def _maybe_raise_nondeterministic(msg: str) -> None: