Method
__init__
(self, ds, tokenizer, max_seq_len=1024, mem_len=None, sample_across_doc=True, **kwargs)
Source from the content-addressed store, hash-verified
| 500 | |
| 501 | class XLDataset(data.Dataset): |
| 502 | def __init__(self, ds, tokenizer, max_seq_len=1024, mem_len=None, sample_across_doc=True, **kwargs): |
| 503 | self.ds = ds |
| 504 | self.tokenizer = tokenizer |
| 505 | self.max_seq_len = max_seq_len |
| 506 | if mem_len is None: |
| 507 | mem_len = max_seq_len |
| 508 | self.mem_len = mem_len |
| 509 | self.sample_across_doc = sample_across_doc |
| 510 | self.indices, self.num_samples = None, None |
| 511 | if hasattr(self.ds, 'is_lazy') and self.ds.is_lazy: |
| 512 | self.is_lazy = True |
| 513 | self.init_indices() |
| 514 | |
| 515 | def init_indices(self): |
| 516 | if self.is_lazy: |
Tested by
no test coverage detected