Load data from multiple datasets.
(self,
configs: list,
partition: list,
num_data: Optional[Union[int, None]] = None)
| 19 | Default: None. |
| 20 | """ |
| 21 | def __init__(self, |
| 22 | configs: list, |
| 23 | partition: list, |
| 24 | num_data: Optional[Union[int, None]] = None): |
| 25 | """Load data from multiple datasets.""" |
| 26 | assert min(partition) >= 0 |
| 27 | datasets = [build_dataset(cfg) for cfg in configs] |
| 28 | self.dataset = ConcatDataset(datasets) |
| 29 | if num_data is not None: |
| 30 | self.length = num_data |
| 31 | else: |
| 32 | self.length = max(len(ds) for ds in datasets) |
| 33 | weights = [ |
| 34 | np.ones(len(ds)) * p / len(ds) |
| 35 | for (p, ds) in zip(partition, datasets) |
| 36 | ] |
| 37 | weights = np.concatenate(weights, axis=0) |
| 38 | self.sampler = WeightedRandomSampler(weights, 1) |
| 39 | |
| 40 | def __len__(self): |
| 41 | """Get the size of the dataset.""" |
nothing calls this directly
no test coverage detected