A dict of doubles, backed by an mmapped file. The file starts with a 4 byte int, indicating how much of it is used. Then 4 bytes of padding. There's then a number of entries, consisting of a 4 byte int which is the size of the next field, a utf-8 encoded string key, padding to a 8 b
| 48 | |
| 49 | |
| 50 | class MmapedDict: |
| 51 | """A dict of doubles, backed by an mmapped file. |
| 52 | |
| 53 | The file starts with a 4 byte int, indicating how much of it is used. |
| 54 | Then 4 bytes of padding. |
| 55 | There's then a number of entries, consisting of a 4 byte int which is the |
| 56 | size of the next field, a utf-8 encoded string key, padding to a 8 byte |
| 57 | alignment, and then a 8 byte float which is the value and a 8 byte float |
| 58 | which is a UNIX timestamp in seconds. |
| 59 | |
| 60 | Not thread safe. |
| 61 | """ |
| 62 | |
| 63 | def __init__(self, filename, read_mode=False): |
| 64 | self._f = open(filename, 'rb' if read_mode else 'a+b') |
| 65 | self._fname = filename |
| 66 | capacity = os.fstat(self._f.fileno()).st_size |
| 67 | if capacity == 0: |
| 68 | self._f.truncate(_INITIAL_MMAP_SIZE) |
| 69 | capacity = _INITIAL_MMAP_SIZE |
| 70 | self._capacity = capacity |
| 71 | self._m = mmap.mmap(self._f.fileno(), self._capacity, |
| 72 | access=mmap.ACCESS_READ if read_mode else mmap.ACCESS_WRITE) |
| 73 | |
| 74 | self._positions = {} |
| 75 | self._used = _unpack_integer(self._m, 0)[0] |
| 76 | if self._used == 0: |
| 77 | self._used = 8 |
| 78 | _pack_integer(self._m, 0, self._used) |
| 79 | else: |
| 80 | if not read_mode: |
| 81 | for key, _, _, pos in self._read_all_values(): |
| 82 | self._positions[key] = pos |
| 83 | |
| 84 | @staticmethod |
| 85 | def read_all_values_from_file(filename): |
| 86 | with open(filename, 'rb') as infp: |
| 87 | # Read the first block of data, including the first 4 bytes which tell us |
| 88 | # how much of the file (which is preallocated to _INITIAL_MMAP_SIZE bytes) is occupied. |
| 89 | data = infp.read(mmap.PAGESIZE) |
| 90 | used = _unpack_integer(data, 0)[0] |
| 91 | if used > len(data): # Then read in the rest, if needed. |
| 92 | data += infp.read(used - len(data)) |
| 93 | return _read_all_values(data, used) |
| 94 | |
| 95 | def _init_value(self, key): |
| 96 | """Initialize a value. Lock must be held by caller.""" |
| 97 | encoded = key.encode('utf-8') |
| 98 | # Pad to be 8-byte aligned. |
| 99 | padded = encoded + (b' ' * (8 - (len(encoded) + 4) % 8)) |
| 100 | value = struct.pack(f'i{len(padded)}sdd'.encode(), len(encoded), padded, 0.0, 0.0) |
| 101 | while self._used + len(value) > self._capacity: |
| 102 | self._capacity *= 2 |
| 103 | self._f.truncate(self._capacity) |
| 104 | self._m = mmap.mmap(self._f.fileno(), self._capacity) |
| 105 | self._m[self._used:self._used + len(value)] = value |
| 106 | |
| 107 | # Update how much space we've used. |