Dataset wrapper to access a subset of another dataset. Purpose: useful to index into existing datasets, possibly large-scale datasets as the subindexing operation is done in an on-the-fly manner. Arguments: ds (Dataset or array-like): List of datasets to be subindexed
| 141 | |
| 142 | |
| 143 | class SplitDataset(data.Dataset): |
| 144 | """ |
| 145 | Dataset wrapper to access a subset of another dataset. |
| 146 | Purpose: useful to index into existing datasets, possibly |
| 147 | large-scale datasets as the subindexing operation is done in an |
| 148 | on-the-fly manner. |
| 149 | Arguments: |
| 150 | ds (Dataset or array-like): List of datasets to be subindexed |
| 151 | split_inds (1D array-like): List of indices part of subset |
| 152 | """ |
| 153 | |
| 154 | def __init__(self, ds, split_inds, **kwargs): |
| 155 | self.split_inds = list(split_inds) |
| 156 | self.wrapped_data = ds |
| 157 | self.is_lazy = isinstance(ds, LazyLoader) or (hasattr(ds, 'is_lazy') and ds.is_lazy) |
| 158 | self._X = None |
| 159 | self._Y = None |
| 160 | |
| 161 | def __len__(self): |
| 162 | return len(self.split_inds) |
| 163 | |
| 164 | def get_text_len(self, idx): |
| 165 | return self.wrapped_data.get_text_len(self.split_inds[idx]) |
| 166 | |
| 167 | def __getitem__(self, index): |
| 168 | return self.wrapped_data[self.split_inds[index]] |
| 169 | |
| 170 | def SetTokenizer(self, tokenizer): |
| 171 | self.wrapped_data.SetTokenizer(tokenizer) |
| 172 | |
| 173 | def GetTokenizer(self): |
| 174 | return self.wrapped_data.GetTokenizer() |
| 175 | |
| 176 | @property |
| 177 | def X(self): |
| 178 | if self._X is None: |
| 179 | self._X = itemgetter(*self.split_inds)(self.wrapped_data.X) |
| 180 | return self._X |
| 181 | |
| 182 | @property |
| 183 | def Y(self): |
| 184 | if self._Y is None: |
| 185 | self._Y = np.array(itemgetter(*self.split_inds)(self.wrapped_data.Y)) |
| 186 | return self._Y |
| 187 | |
| 188 | def __iter__(self): |
| 189 | for idx in self.split_inds: |
| 190 | yield self.wrapped_data[idx] |
| 191 | |
| 192 | |
| 193 | def split_ds(ds, split=None, shuffle=True, save_splits=None, load_splits=None): |