Method _get_env

datastudio/utils/database.py:195–234

Lazy-load shard environment with LRU eviction (thread-safe). Args: shard_id: Shard ID to load. Returns: lmdb.Environment for the shard.

(self, shard_id: int)

        return get_shard_id(image_path, self.num_shards)

    def _get_env(self, shard_id: int) -> lmdb.Environment:
        """Lazy-load shard environment with LRU eviction (thread-safe).

        Args:
            shard_id: Shard ID to load.

        Returns:
            lmdb.Environment for the shard.
        """
        with self._lock:
            if shard_id in self._envs:
                # Move to end to mark as recently used
                self._envs.move_to_end(shard_id)
                return self._envs[shard_id]

            # Evict oldest shard if at capacity
            while len(self._envs) >= self.max_open_shards:
                oldest_shard_id, oldest_env = self._envs.popitem(last=False)
                try:
                    oldest_env.close()
                except Exception:
                    pass

            shard_path = self._get_shard_path(shard_id)

            # Check existing file size, dynamically adjust map_size
            data_file = os.path.join(shard_path, "data.mdb")
            actual_map_size = self.map_size_per_shard
            if os.path.exists(data_file):
                actual_size = os.path.getsize(data_file)
                if actual_size > self.map_size_per_shard:
                    actual_map_size = actual_size * 2

            self._envs[shard_id] = init_sharded_lmdb(
                shard_path,
                map_size=actual_map_size,
                readonly=self.readonly,
                lock=self.lock,
            )
            return self._envs[shard_id]

    def _get_key(self, image_path: str) -> bytes:
        """Get LMDB key for an image path."""