(self)
| 594 | self.init_weighting() |
| 595 | |
| 596 | def init_weighting(self): |
| 597 | if self.is_lazy: |
| 598 | lens = np.array([self.ds.get_text_len(idx) for idx in range(len(self.ds))]) |
| 599 | else: |
| 600 | lens = np.array([len(d['text']) if isinstance(d, dict) else len(d) for d in self.ds]) |
| 601 | self.total_len = np.sum(lens) |
| 602 | print_rank_0( |
| 603 | f"Dataset document count {len(lens)}, token count {self.total_len}, non sentence start{self.non_sentence_start}") |
| 604 | self.weighting = list(accumulate(lens)) |
| 605 | |
| 606 | def get_weighted_samples(self, np_rng): |
| 607 | while True: |
no test coverage detected