(self, path, data_type='data', mem_map=False, map_fn=None, is_array=False, array_data_type=np.int32,
load_memory=False, half_load=False)
| 139 | """ |
| 140 | |
| 141 | def __init__(self, path, data_type='data', mem_map=False, map_fn=None, is_array=False, array_data_type=np.int32, |
| 142 | load_memory=False, half_load=False): |
| 143 | lazypath = get_lazy_path(path) |
| 144 | datapath = os.path.join(lazypath, data_type) |
| 145 | # get file where array entries are concatenated into one big string |
| 146 | self._file = open(datapath, 'rb') |
| 147 | self.file = self._file |
| 148 | self.is_array = is_array |
| 149 | self.array_data_type = array_data_type |
| 150 | # memory map file if necessary |
| 151 | lenpath = os.path.join(lazypath, data_type + '.len.pkl') |
| 152 | self.lens = pkl.load(open(lenpath, 'rb')) |
| 153 | if half_load: |
| 154 | self.lens = self.lens[:2 * len(self.lens) // 3] |
| 155 | self.ends = list(accumulate(self.lens)) |
| 156 | self.dumb_ends = list(self.ends) |
| 157 | self.mem_map = mem_map |
| 158 | self.load_memory = load_memory |
| 159 | if self.load_memory: |
| 160 | data_type_size = np.dtype(self.array_data_type).itemsize |
| 161 | if half_load: |
| 162 | self.file = self.file.read(sum(self.lens) * data_type_size) |
| 163 | else: |
| 164 | self.file = self.file.read() |
| 165 | self.file = np.ndarray(shape=(len(self.file) // data_type_size,), dtype=array_data_type, buffer=self.file, |
| 166 | order='C') |
| 167 | elif self.mem_map: |
| 168 | if is_array: |
| 169 | if self.ends[-1] == 0: |
| 170 | self.file = np.array([], dtype=array_data_type) |
| 171 | else: |
| 172 | self.file = np.memmap(self.file, dtype=array_data_type, mode='r', order='C') |
| 173 | else: |
| 174 | if self.ends[-1] == 0: |
| 175 | self.file = bytearray() |
| 176 | else: |
| 177 | self.file = mmap.mmap(self.file.fileno(), 0, prot=mmap.PROT_READ) |
| 178 | self.read_lock = Lock() |
| 179 | self.process_fn = map_fn |
| 180 | self.map_fn = map_fn |
| 181 | self._tokenizer = None |
| 182 | self.is_lazy = True |
| 183 | |
| 184 | def SetTokenizer(self, tokenizer): |
| 185 | """ |
nothing calls this directly
no test coverage detected