(self)
| 873 | self.get_weighting() |
| 874 | |
| 875 | def get_weighting(self): |
| 876 | if self.weighted: |
| 877 | if hasattr(self.ds, 'is_lazy') and self.ds.is_lazy: |
| 878 | lens = np.array(self.ds.lens) |
| 879 | else: |
| 880 | lens = np.array([len(d['text']) if isinstance(d, dict) else len(d) for d in self.ds]) |
| 881 | self.total_len = np.sum(lens) |
| 882 | self.weighting = list(accumulate(lens)) |
| 883 | else: |
| 884 | self.weighting = None |
| 885 | |
| 886 | def get_weighted_samples(self, np_rng): |
| 887 | if self.weighting is not None: |