(self, np_rng)
| 604 | self.weighting = list(accumulate(lens)) |
| 605 | |
| 606 | def get_weighted_samples(self, np_rng): |
| 607 | while True: |
| 608 | idx = np_rng.randint(self.total_len) |
| 609 | data_idx = bisect_right(self.weighting, idx) |
| 610 | tokens, loss_mask = self.getidx(data_idx) |
| 611 | if self.filter_english: |
| 612 | text = self.tokenizer.DecodeIds(tokens[:1024]) |
| 613 | lang = self.model.predict(text.replace('\n', ''))[0][0] |
| 614 | if lang == '__label__en': |
| 615 | break |
| 616 | else: |
| 617 | break |
| 618 | return tokens, loss_mask |
| 619 | |
| 620 | def __len__(self): |
| 621 | return self.num_samples |
no test coverage detected