(self, idx)
| 187 | |
| 188 | # @lru_cache(maxsize=8) |
| 189 | def __getitem__(self, idx): |
| 190 | if not self.data_file: |
| 191 | self.read_data(self.path) |
| 192 | if isinstance(idx, int): |
| 193 | i = idx |
| 194 | self.check_index(i) |
| 195 | tensor_size = self.sizes[self.dim_offsets[i] : self.dim_offsets[i + 1]] |
| 196 | a = np.empty(tensor_size, dtype=self.dtype) |
| 197 | self.data_file.seek(self.data_offsets[i] * self.element_size) |
| 198 | self.data_file.readinto(a) |
| 199 | return a |
| 200 | elif isinstance(idx, slice): |
| 201 | start, stop, step = idx.indices(len(self)) |
| 202 | if step != 1: |
| 203 | raise ValueError("Slices into indexed_dataset must be contiguous") |
| 204 | sizes = self.sizes[self.dim_offsets[start] : self.dim_offsets[stop]] |
| 205 | size = sum(sizes) |
| 206 | a = np.empty(size, dtype=self.dtype) |
| 207 | self.data_file.seek(self.data_offsets[start] * self.element_size) |
| 208 | self.data_file.readinto(a) |
| 209 | offsets = list(accumulate(sizes)) |
| 210 | sents = np.split(a, offsets[:-1]) |
| 211 | return sents |
| 212 | |
| 213 | def __len__(self): |
| 214 | return self._len |
nothing calls this directly
no test coverage detected