Method add

utils/cache.py:109–133

(self, item)

    def add(self, item):
        if self.shard_file is None:
            self.create_new_shard()
        buffer = io.BytesIO()
        torch.save(item, buffer)
        bytes_view = buffer.getbuffer()
        self.shard_file.write(bytes_view)

        # update items metadata
        item = (self.shard, self.shard_index)
        self.items.append(item)
        self.con.execute('INSERT INTO items VALUES(?, ?)', item)
        self.shard_index += 1

        # update shard metadata
        size = len(bytes_view)
        entry = (self.offset, size)
        self.shard_metadata[self.shard].append(entry)
        self.con.execute(f'INSERT INTO {self.shard_table} VALUES (?, ?)', entry)
        self.offset += size

        # create new shard when existing one is large enough
        current_size_gb = self.shard_file.tell() / 1_000_000_000
        if current_size_gb >= self.shard_size_gb:
            self.finalize_current_shard()


# for testing