read specified portion of file
(self, start=0, end=None)
| 227 | return len(self.ends) |
| 228 | |
| 229 | def file_read(self, start=0, end=None): |
| 230 | """read specified portion of file""" |
| 231 | data_type_size = np.dtype(self.array_data_type).itemsize |
| 232 | # atomic reads to avoid race conditions with multiprocess dataloader |
| 233 | self.read_lock.acquire() |
| 234 | if not self.mem_map and not self.load_memory: |
| 235 | # seek to start of file read |
| 236 | if self.is_array: |
| 237 | start = start * data_type_size |
| 238 | end = end * data_type_size if end is not None else None |
| 239 | self.file.seek(start) |
| 240 | # read to end of file if no end point provided |
| 241 | if end is None: |
| 242 | rtn = self.file.read() |
| 243 | # else read amount needed to reach end point |
| 244 | else: |
| 245 | rtn = self.file.read(end - start) |
| 246 | if self.is_array: |
| 247 | rtn = np.ndarray(shape=(len(rtn) // data_type_size,), dtype=self.array_data_type, buffer=rtn, order='C') |
| 248 | else: |
| 249 | rtn = rtn.decode('utf-8', 'ignore') |
| 250 | else: |
| 251 | rtn = self.file[start:end] |
| 252 | if self.is_array: |
| 253 | rtn = rtn.copy() |
| 254 | else: |
| 255 | rtn = rtn.decode('utf-8', 'strict') |
| 256 | self.read_lock.release() |
| 257 | # TODO: @raulp figure out mem map byte string bug |
| 258 | # if mem map'd need to decode byte string to string |
| 259 | # # rtn = str(rtn) |
| 260 | # if self.mem_map: |
| 261 | # rtn = rtn.decode('unicode_escape') |
| 262 | return rtn |