(self, raw_data, tokenizer: transformers.PreTrainedTokenizer)
| 206 | """Dataset for supervised fine-tuning.""" |
| 207 | |
| 208 | def __init__(self, raw_data, tokenizer: transformers.PreTrainedTokenizer): |
| 209 | super(LazySupervisedDataset, self).__init__() |
| 210 | self.tokenizer = tokenizer |
| 211 | |
| 212 | rank0_print("Formatting inputs...Skip in lazy mode") |
| 213 | self.tokenizer = tokenizer |
| 214 | self.raw_data = raw_data |
| 215 | self.cached_data_dict = {} |
| 216 | |
| 217 | def __len__(self): |
| 218 | return len(self.raw_data) |
nothing calls this directly
no test coverage detected