MCPcopy
hub / github.com/dask/dask / _normalize_pickle

Function _normalize_pickle

dask/tokenize.py:233–263  ·  view source on GitHub ↗
(o: object)

Source from the content-addressed store, hash-verified

231
232
233def _normalize_pickle(o: object) -> tuple:
234 buffers: list[pickle.PickleBuffer] = []
235 pik: int | None = None
236 pik2: int | None = None
237 for _ in range(3):
238 buffers.clear()
239 try:
240 out = pickle.dumps(o, protocol=5, buffer_callback=buffers.append)
241 if b"__main__" in out:
242 # Use `cloudpickle` for objects defined in `__main__`
243 buffers.clear()
244 out = cloudpickle.dumps(o, protocol=5, buffer_callback=buffers.append)
245 pickle.loads(out, buffers=buffers)
246 pik2 = hash_buffer_hex(out)
247 except Exception:
248 buffers.clear()
249 try:
250 out = cloudpickle.dumps(o, protocol=5, buffer_callback=buffers.append)
251 pickle.loads(out, buffers=buffers)
252 pik2 = hash_buffer_hex(out)
253 except Exception:
254 break
255 if pik and pik2 and pik == pik2:
256 break
257 pik = pik2
258 else:
259 _maybe_raise_nondeterministic("Failed to tokenize deterministically")
260 if pik is None:
261 _maybe_raise_nondeterministic("Failed to tokenize deterministically")
262 pik = int(uuid.uuid4())
263 return pik, [hash_buffer_hex(buf) for buf in buffers]
264
265
266def _normalize_dataclass(obj):

Callers 2

normalize_objectFunction · 0.85
normalize_ufuncFunction · 0.85

Calls 3

hash_buffer_hexFunction · 0.90
clearMethod · 0.45

Tested by

no test coverage detected

Used in the wild real call sites across dependent graphs

searching dependent graphs…