(self, idx)
| 74 | 'attention_mask': attention_mask} |
| 75 | |
| 76 | def __getitem__(self, idx): |
| 77 | if self.reweight: |
| 78 | rng = random.Random(idx) |
| 79 | rng = np.random.RandomState(seed=[rng.randint(0, 2 ** 32 - 1) for _ in range(16)]) |
| 80 | dataset_idx = rng.choice(np.arange(len(self.datasets)), p=self.weights) |
| 81 | dataset = self.datasets[dataset_idx] |
| 82 | sample_idx = rng.choice(np.arange(len(dataset))) |
| 83 | item = self.datasets[dataset_idx][sample_idx] |
| 84 | else: |
| 85 | dataset_idx = bisect_right(self.cumulative_lens, idx) |
| 86 | if dataset_idx == 0: |
| 87 | sample_idx = idx |
| 88 | else: |
| 89 | sample_idx = idx - self.cumulative_lens[dataset_idx - 1] |
| 90 | item = self.datasets[dataset_idx][sample_idx] |
| 91 | item = self.pet_wrapper(item) |
| 92 | return item |
| 93 | |
| 94 | |
| 95 | class DataConfig: |
nothing calls this directly
no test coverage detected