| 37 | |
| 38 | |
| 39 | def init(self): |
| 40 | print('[CACHE] Initializing') |
| 41 | # create database |
| 42 | self.con = sqlite3.connect(self.metadata_db, autocommit=False) |
| 43 | |
| 44 | # check fingerprint, clear cache if different |
| 45 | self.con.execute('CREATE TABLE IF NOT EXISTS fingerprint(value)') |
| 46 | existing_fingerprint = self.con.execute('SELECT value FROM fingerprint').fetchone() |
| 47 | if existing_fingerprint is not None: |
| 48 | existing_fingerprint = existing_fingerprint[0] |
| 49 | print(f'[CACHE] Existing cache has fingerprint {existing_fingerprint}') |
| 50 | if self.fingerprint != existing_fingerprint: |
| 51 | print('[CACHE] Fingerprint changed, deleting existing cache files') |
| 52 | self.clear() |
| 53 | return |
| 54 | else: |
| 55 | print(f'[CACHE] Storing new fingerprint: {self.fingerprint}') |
| 56 | self.con.execute('INSERT INTO fingerprint VALUES(?)', (self.fingerprint,)) |
| 57 | |
| 58 | # items table, current length, next shard index |
| 59 | self.con.execute('CREATE TABLE IF NOT EXISTS items(shard, shard_index)') |
| 60 | self.items = self.con.execute('SELECT shard, shard_index FROM items').fetchall() or [] |
| 61 | max_existing_shard = -1 |
| 62 | for shard, _ in self.items: |
| 63 | max_existing_shard = max(max_existing_shard, shard) |
| 64 | self.shard = max_existing_shard + 1 # current shard to write to |
| 65 | self.shard_file = None |
| 66 | print(f'[CACHE] Existing cache length: {len(self)}') |
| 67 | |
| 68 | # shard metadata |
| 69 | self.shard_metadata = defaultdict(list) |
| 70 | for table_name, in self.con.execute('SELECT name FROM sqlite_master').fetchall(): |
| 71 | if table_name.startswith('shard_'): |
| 72 | shard_id = int(table_name.split('_')[-1]) |
| 73 | for entry in self.con.execute(f'SELECT offset, size FROM {table_name}').fetchall(): |
| 74 | self.shard_metadata[shard_id].append(entry) |
| 75 | self.open_files = {} |
| 76 | |
| 77 | # commit |
| 78 | self.con.commit() |
| 79 | |
| 80 | |
| 81 | def clear(self): |