Go through the text data by order of src length with a bit of randomness. From fastai repo.
| 331 | |
| 332 | |
| 333 | class SortishSampler(Sampler): |
| 334 | "Go through the text data by order of src length with a bit of randomness. From fastai repo." |
| 335 | |
| 336 | def __init__(self, data, batch_size, shuffle=True): |
| 337 | self.data, self.bs, self.shuffle = data, batch_size, shuffle |
| 338 | |
| 339 | def __len__(self) -> int: |
| 340 | return len(self.data) |
| 341 | |
| 342 | def __iter__(self): |
| 343 | return iter(sortish_sampler_indices(self.data, self.bs, shuffle=self.shuffle)) |
| 344 | |
| 345 | |
| 346 | def sortish_sampler_indices(data: List, bs: int, shuffle=True) -> np.array: |
no outgoing calls
no test coverage detected