Method
load_from_dataset
(cls, dataset: Dataset, config: StorageConfig)
Source from the content-addressed store, hash-verified
| 392 | |
| 393 | @classmethod |
| 394 | async def load_from_dataset(cls, dataset: Dataset, config: StorageConfig) -> "SQLTaskStorage": |
| 395 | storage = cls(config) |
| 396 | await storage.prepare() |
| 397 | batch_size = config.batch_size |
| 398 | batch = [] |
| 399 | for item in dataset: |
| 400 | batch.append(item) |
| 401 | if len(batch) >= batch_size: |
| 402 | await storage.write(batch) |
| 403 | batch.clear() |
| 404 | if batch: |
| 405 | await storage.write(batch) |
| 406 | return storage |
| 407 | |
| 408 | def acquire(self) -> int: |
| 409 | self.ref_count += 1 |