Method
__init__
(
self,
path,
process_fn,
length_per_sample=64 + 1024 + 4096,
dtype="int32",
preload=False,
**kwargs,
)
Source from the content-addressed store, hash-verified
| 71 | |
| 72 | class BinaryDataset(Dataset): |
| 73 | def __init__( |
| 74 | self, |
| 75 | path, |
| 76 | process_fn, |
| 77 | length_per_sample=64 + 1024 + 4096, |
| 78 | dtype="int32", |
| 79 | preload=False, |
| 80 | **kwargs, |
| 81 | ): # TODO ARGS |
| 82 | assert length_per_sample is not None |
| 83 | self.length_per_sample = length_per_sample |
| 84 | self.dtype = np.dtype(dtype) |
| 85 | self.process_fn = process_fn |
| 86 | if preload: |
| 87 | self.bin = np.fromfile(path, dtype=self.dtype).reshape( |
| 88 | -1, length_per_sample |
| 89 | ) |
| 90 | else: |
| 91 | with open(path, "r") as fid: |
| 92 | nbytes = fid.seek(0, 2) |
| 93 | flen = fid.tell() // self.dtype.itemsize |
| 94 | self.bin = np.memmap( |
| 95 | path, |
| 96 | dtype=self.dtype, |
| 97 | shape=(flen // length_per_sample, length_per_sample), |
| 98 | ) |
| 99 | |
| 100 | def __len__(self): |
| 101 | return self.bin.shape[0] |
Callers
nothing calls this directly
Tested by
no test coverage detected