Function object_size

dask/sizeof.py:150–175

(*xs)

    OBJECT_DTYPES = (object, pd.StringDtype("python"))

    def object_size(*xs):
        if not xs:
            return 0
        ncells = sum(len(x) for x in xs)
        if not ncells:
            return 0

        # Deduplicate Series of references to the same objects,
        # e.g. as produced by read_parquet
        unique_samples = {}
        for x in xs:
            sample = np.random.choice(x, size=100, replace=True)
            for i in sample.tolist():
                unique_samples[id(i)] = i

        nsamples = 100 * len(xs)
        sample_nbytes = sum(sizeof(i) for i in unique_samples.values())
        if len(unique_samples) / nsamples > 0.5:
            # Less than half of the references are duplicated.
            # Assume that, if we were to analyze twice the amount of random references,
            # we would get twice the amount of unique objects too.
            return int(sample_nbytes * ncells / nsamples)
        else:
            # Assume we've already found all unique objects and that all references that
            # we have not yet analyzed are going to point to the same data.
            return sample_nbytes

    @sizeof.register(pd.DataFrame)
    def sizeof_pandas_dataframe(df):