Batch read multiple images, one transaction per shard. Opens temporary LMDB envs with readahead=True for better sequential read performance, and reads shards in parallel threads. Args: image_paths: List of image paths to read. Returns: Dict
(self, image_paths: List[str])
| 348 | return result |
| 349 | |
| 350 | def batch_get(self, image_paths: List[str]) -> Dict[str, Optional[bytes]]: |
| 351 | """Batch read multiple images, one transaction per shard. |
| 352 | |
| 353 | Opens temporary LMDB envs with readahead=True for better sequential |
| 354 | read performance, and reads shards in parallel threads. |
| 355 | |
| 356 | Args: |
| 357 | image_paths: List of image paths to read. |
| 358 | |
| 359 | Returns: |
| 360 | Dict mapping image_path -> bytes (or None if not found). |
| 361 | """ |
| 362 | # Group paths by shard using single MD5 per path |
| 363 | shard_groups: Dict[int, List[Tuple[str, bytes]]] = defaultdict(list) |
| 364 | for path in image_paths: |
| 365 | shard_id, key = self._get_shard_id_and_key(path) |
| 366 | shard_groups[shard_id].append((path, key)) |
| 367 | |
| 368 | total = len(image_paths) |
| 369 | show_progress = total >= 1000 and self.logger is not None |
| 370 | |
| 371 | result = {} |
| 372 | result_lock = threading.Lock() |
| 373 | |
| 374 | if show_progress: |
| 375 | pbar = tqdm( |
| 376 | total=total, |
| 377 | desc="LMDB read", |
| 378 | unit="img", |
| 379 | ) |
| 380 | |
| 381 | def _open_read_env(shard_id): |
| 382 | """Open a temporary env with readahead=True for batch reading.""" |
| 383 | shard_path = self._get_shard_path(shard_id) |
| 384 | data_file = os.path.join(shard_path, "data.mdb") |
| 385 | actual_map_size = self.map_size_per_shard |
| 386 | if os.path.exists(data_file): |
| 387 | actual_size = os.path.getsize(data_file) |
| 388 | if actual_size > self.map_size_per_shard: |
| 389 | actual_map_size = actual_size * 2 |
| 390 | return init_sharded_lmdb( |
| 391 | shard_path, |
| 392 | map_size=actual_map_size, |
| 393 | readonly=True, |
| 394 | lock=False, |
| 395 | readahead=True, |
| 396 | ) |
| 397 | |
| 398 | def _read_shard(shard_id_and_items): |
| 399 | shard_id, items = shard_id_and_items |
| 400 | env = _open_read_env(shard_id) |
| 401 | try: |
| 402 | shard_result = {} |
| 403 | with env.begin(write=False) as txn: |
| 404 | for path, key in items: |
| 405 | shard_result[path] = txn.get(key) |
| 406 | with result_lock: |
| 407 | result.update(shard_result) |
no test coverage detected