| 94 | self._close_lmdb() |
| 95 | |
| 96 | def _set_keys(self, keys=None): |
| 97 | def find_keys(txn, size): |
| 98 | logger.warn("Traversing the database to find keys is slow. Your should specify the keys.") |
| 99 | keys = [] |
| 100 | with timed_operation("Loading LMDB keys ...", log_start=True), \ |
| 101 | get_tqdm(total=size) as pbar: |
| 102 | for k in self._txn.cursor(): |
| 103 | assert k[0] != b'__keys__' |
| 104 | keys.append(k[0]) |
| 105 | pbar.update() |
| 106 | return keys |
| 107 | |
| 108 | self.keys = self._txn.get(b'__keys__') |
| 109 | if self.keys is not None: |
| 110 | self.keys = loads(self.keys) |
| 111 | self._size -= 1 # delete this item |
| 112 | |
| 113 | if self._shuffle: # keys are necessary when shuffle is True |
| 114 | if keys is None: |
| 115 | if self.keys is None: |
| 116 | self.keys = find_keys(self._txn, self._size) |
| 117 | else: |
| 118 | # check if key-format like '{:0>8d}' was given |
| 119 | if isinstance(keys, six.string_types): |
| 120 | self.keys = map(lambda x: keys.format(x), list(np.arange(self._size))) |
| 121 | else: |
| 122 | self.keys = keys |
| 123 | |
| 124 | def _open_lmdb(self): |
| 125 | self._lmdb = lmdb.open(self._lmdb_path, |