Class BigArray

lib/core/bigarray.py:65–315

List-like class used for storing large amounts of data (disk cached) >>> _ = BigArray(xrange(100000), chunk_size=500 * 1024) >>> _[20] = 0 >>> _[-1] = 999 >>> _[99999] 999 >>> _[100000] Traceback (most recent call last): ... IndexError: BigArray index out of

        self.dirty = dirty

class BigArray(list):
    """
    List-like class used for storing large amounts of data (disk cached)

    >>> _ = BigArray(xrange(100000), chunk_size=500 * 1024)
    >>> _[20] = 0
    >>> _[-1] = 999
    >>> _[99999]
    999
    >>> _[100000]
    Traceback (most recent call last):
    ...
    IndexError: BigArray index out of range
    >>> _ += [0]
    >>> sum(_)
    4999850980
    >>> _[len(_) // 2] = 17
    >>> sum(_)
    4999800997
    >>> _[100000]
    0
    >>> _[0] = [None]
    >>> _.index(0)
    20
    >>> import pickle; __ = pickle.loads(pickle.dumps(_))
    >>> __.append(1)
    >>> len(_)
    100001
    >>> _ = __
    >>> _[-1]
    1
    >>> len([_ for _ in BigArray(xrange(100000))])
    100000
    """

    def __init__(self, items=None, chunk_size=BIGARRAY_CHUNK_SIZE):
        self.chunks = [[]]
        self.chunk_length = sys.maxsize
        self.cache = None
        self.filenames = set()
        self._lock = threading.Lock()
        self._os_remove = os.remove
        self._size_counter = 0
        self._chunk_size = chunk_size

        for item in (items or []):
            self.append(item)

    def __add__(self, value):
        retval = BigArray(self)

        for _ in value:
            retval.append(_)

        return retval

    def __iadd__(self, value):
        for _ in value: