Set the input data and run deterministic transforms to generate cache content. Note: should call this func after an entire epoch and must set `persistent_workers=False` in PyTorch DataLoader, because it needs to create new worker processes based on new generated cac
(self, data: Sequence)
| 263 | self.set_data(data) # type: ignore |
| 264 | |
| 265 | def set_data(self, data: Sequence) -> None: |
| 266 | """ |
| 267 | Set the input data and run deterministic transforms to generate cache content. |
| 268 | |
| 269 | Note: should call this func after an entire epoch and must set `persistent_workers=False` |
| 270 | in PyTorch DataLoader, because it needs to create new worker processes based on new |
| 271 | generated cache content. |
| 272 | |
| 273 | """ |
| 274 | self.data = data |
| 275 | |
| 276 | # only compute cache for the unique items of dataset, and record the last index for duplicated items |
| 277 | mapping = {self.hash_func(v): i for i, v in enumerate(self.data)} |
| 278 | self.cache_num = min(int(self.set_num), int(len(mapping) * self.set_rate), len(mapping)) |
| 279 | self._hash_keys = list(mapping)[: self.cache_num] |
| 280 | indices = list(mapping.values())[: self.cache_num] |
| 281 | self._cache, self._cache_other = zip(*self._fill_cache(indices)) # type: ignore |
| 282 | |
| 283 | def _fill_cache(self, indices=None) -> list: |
| 284 | """ |