| 159 | return len(self.data) |
| 160 | |
| 161 | def __iter__(self): |
| 162 | idxs = np.random.permutation(len(self.data)) |
| 163 | sz = self.bs * 50 |
| 164 | ck_idx = [idxs[i : i + sz] for i in range(0, len(idxs), sz)] |
| 165 | sort_idx = np.concatenate([sorted(s, key=self.key, reverse=True) for s in ck_idx]) |
| 166 | sz = self.bs |
| 167 | ck_idx = [sort_idx[i : i + sz] for i in range(0, len(sort_idx), sz)] |
| 168 | max_ck = np.argmax([self.key(ck[0]) for ck in ck_idx]) # find the chunk with the largest key, |
| 169 | ck_idx[0], ck_idx[max_ck] = ck_idx[max_ck], ck_idx[0] # then make sure it goes first. |
| 170 | sort_idx = np.concatenate(np.random.permutation(ck_idx[1:])) if len(ck_idx) > 1 else np.array([], dtype=np.int) |
| 171 | sort_idx = np.concatenate((ck_idx[0], sort_idx)) |
| 172 | return iter(sort_idx) |
| 173 | |
| 174 | |
| 175 | def use_task_specific_params(model, task): |