Adds (key, data) to bucket. Args: key (int): the key. data (binary): the data.
(self, key: type_utils.Key, data: bytes)
| 149 | return self._length |
| 150 | |
| 151 | def add(self, key: type_utils.Key, data: bytes): |
| 152 | """Adds (key, data) to bucket. |
| 153 | |
| 154 | Args: |
| 155 | key (int): the key. |
| 156 | data (binary): the data. |
| 157 | """ |
| 158 | if not self._fobj: |
| 159 | file_utils.makedirs_cached(os.path.dirname(self._path)) |
| 160 | self._fobj = tf.io.gfile.GFile(self._path, mode='wb') |
| 161 | data_size = len(data) |
| 162 | |
| 163 | try: |
| 164 | self._fobj.write(_hkey_to_bytes(key)) |
| 165 | except tf.errors.ResourceExhaustedError as error: |
| 166 | # catch "Too many open files" |
| 167 | if error.message.endswith('Too many open files'): |
| 168 | _increase_open_files_limit() |
| 169 | self._fobj.write(_hkey_to_bytes(key)) |
| 170 | else: |
| 171 | raise error |
| 172 | # http://docs.python.org/3/library/struct.html#byte-order-size-and-alignment |
| 173 | # The equal sign ("=") is important here, has it guarantees the standard |
| 174 | # size (Q: 8 bytes) is used, as opposed to native size, which can differ |
| 175 | # from one platform to the other. This way we know exactly 8 bytes have been |
| 176 | # written, and we can read that same amount of bytes later. |
| 177 | # We do not specify endianess (platform dependent), but this is OK since the |
| 178 | # temporary files are going to be written and read by the same platform. |
| 179 | self._fobj.write(struct.pack('=Q', data_size)) |
| 180 | self._fobj.write(data) |
| 181 | self._length += 1 |
| 182 | self._size += data_size |
| 183 | |
| 184 | def flush(self): |
| 185 | if self._fobj: |