Set the input data and run deterministic transforms to generate cache content. Note: should call this func after an entire epoch and must set `persistent_workers=False` in PyTorch DataLoader, because it needs to create new worker processes based on new generated cac
(self, data: Sequence)
| 858 | self.set_data(data) |
| 859 | |
| 860 | def set_data(self, data: Sequence) -> None: |
| 861 | """ |
| 862 | Set the input data and run deterministic transforms to generate cache content. |
| 863 | |
| 864 | Note: should call this func after an entire epoch and must set `persistent_workers=False` |
| 865 | in PyTorch DataLoader, because it needs to create new worker processes based on new |
| 866 | generated cache content. |
| 867 | |
| 868 | """ |
| 869 | self.data = data |
| 870 | |
| 871 | def _compute_cache_num(data_len: int): |
| 872 | self.cache_num = min(int(self.set_num), int(data_len * self.set_rate), data_len) |
| 873 | |
| 874 | if self.hash_as_key: |
| 875 | # only compute cache for the unique items of dataset, and record the last index for duplicated items |
| 876 | mapping = {self.hash_func(v): i for i, v in enumerate(self.data)} |
| 877 | _compute_cache_num(len(mapping)) |
| 878 | self._hash_keys = list(mapping)[: self.cache_num] |
| 879 | indices = list(mapping.values())[: self.cache_num] |
| 880 | else: |
| 881 | _compute_cache_num(len(self.data)) |
| 882 | indices = list(range(self.cache_num)) |
| 883 | |
| 884 | if self.runtime_cache in (False, None): # prepare cache content immediately |
| 885 | self._cache = self._fill_cache(indices) |
| 886 | return |
| 887 | if isinstance(self.runtime_cache, str) and "process" in self.runtime_cache: |
| 888 | # this must be in the main process, not in dataloader's workers |
| 889 | self._cache = Manager().list([None] * self.cache_num) |
| 890 | return |
| 891 | if (self.runtime_cache is True) or (isinstance(self.runtime_cache, str) and "thread" in self.runtime_cache): |
| 892 | self._cache = [None] * self.cache_num |
| 893 | return |
| 894 | self._cache = self.runtime_cache # type: ignore |
| 895 | return |
| 896 | |
| 897 | def _fill_cache(self, indices=None) -> list: |
| 898 | """ |