sentence_start: the stripped article must start with a complete sentence
(self, ds, tokenizer,
max_seq_len=1024,
sample_across_doc=True,
non_sentence_start=0.0, filter_english=False, **kwargs)
| 569 | |
| 570 | class BlockDataset(data.Dataset): |
| 571 | def __init__(self, ds, tokenizer, |
| 572 | max_seq_len=1024, |
| 573 | sample_across_doc=True, |
| 574 | non_sentence_start=0.0, filter_english=False, **kwargs): |
| 575 | """ |
| 576 | sentence_start: the stripped article must start with a complete sentence |
| 577 | """ |
| 578 | self.ds = ds |
| 579 | self.ds_len = len(self.ds) |
| 580 | self.num_samples = 1000 * self.ds_len |
| 581 | self.max_seq_len = max_seq_len |
| 582 | self.tokenizer = tokenizer |
| 583 | self.sample_across_doc = sample_across_doc |
| 584 | self.non_sentence_start = non_sentence_start |
| 585 | self.filter_english = filter_english |
| 586 | self.weighting, self.total_len = None, None |
| 587 | self.is_lazy = False |
| 588 | if self.filter_english: |
| 589 | import fasttext |
| 590 | self.model = fasttext.load_model('/mnt/lid.176.bin') |
| 591 | print_rank_0("Load language detection model") |
| 592 | if hasattr(self.ds, 'is_lazy') and self.ds.is_lazy: |
| 593 | self.is_lazy = True |
| 594 | self.init_weighting() |
| 595 | |
| 596 | def init_weighting(self): |
| 597 | if self.is_lazy: |
nothing calls this directly
no test coverage detected