(self, data_dir, tokenizer, data_type='train')
| 24 | |
| 25 | class SFTDataset(Dataset): |
| 26 | def __init__(self, data_dir, tokenizer, data_type='train'): |
| 27 | super().__init__() |
| 28 | |
| 29 | self.data_dir = data_dir |
| 30 | self.tokenizer = tokenizer |
| 31 | self.data_type = data_type |
| 32 | |
| 33 | self.data = [] |
| 34 | # We do not calculate losses for the meta instruction or results returned by plugins |
| 35 | # The token spans with label -100, [(span_start, span_end), ...] |
| 36 | self.no_loss_spans = [] |
| 37 | |
| 38 | self.load_data() |
| 39 | |
| 40 | def load_data(self): |
| 41 | logger.info("Loading data...") |