read file and splice strings based on string ending array `self.ends`
(self, index)
| 197 | return self._tokenizer |
| 198 | |
| 199 | def __getitem__(self, index): |
| 200 | """ |
| 201 | read file and splice strings based on string ending array `self.ends` |
| 202 | """ |
| 203 | if not isinstance(index, slice): |
| 204 | if index == 0: |
| 205 | start = 0 |
| 206 | else: |
| 207 | start = self.ends[index - 1] |
| 208 | end = self.ends[index] |
| 209 | rtn = self.file_read(start, end) |
| 210 | if self.map_fn is not None: |
| 211 | rtn = self.map_fn(rtn) |
| 212 | else: |
| 213 | # if slice, fetch strings with 1 diskread and then splice in memory |
| 214 | chr_lens = self.ends[index] |
| 215 | if index.start == 0 or index.start is None: |
| 216 | start = 0 |
| 217 | else: |
| 218 | start = self.ends[index.start - 1] |
| 219 | stop = chr_lens[-1] |
| 220 | strings = self.file_read(start, stop) |
| 221 | rtn = split_strings(strings, start, chr_lens) |
| 222 | if self.map_fn is not None: |
| 223 | rtn = [self.map_fn(s) for s in rtn] |
| 224 | return rtn |
| 225 | |
| 226 | def __len__(self): |
| 227 | return len(self.ends) |
nothing calls this directly
no test coverage detected