| 68 | # put data into lmdb, and doubling the size if full. |
| 69 | # Ref: https://github.com/NVIDIA/DIGITS/pull/209/files |
| 70 | def put_or_grow(txn, key, value): |
| 71 | try: |
| 72 | txn.put(key, value) |
| 73 | return txn |
| 74 | except lmdb.MapFullError: |
| 75 | pass |
| 76 | txn.abort() |
| 77 | curr_size = db.info()['map_size'] |
| 78 | new_size = curr_size * 2 |
| 79 | logger.info("Doubling LMDB map_size to {:.2f}GB".format(new_size / 10**9)) |
| 80 | db.set_mapsize(new_size) |
| 81 | txn = db.begin(write=True) |
| 82 | txn = put_or_grow(txn, key, value) |
| 83 | return txn |
| 84 | |
| 85 | with get_tqdm(total=size) as pbar: |
| 86 | idx = -1 |