(self, raw_data, tokenizer: transformers.PreTrainedTokenizer, template="tool-llama")
| 195 | """Dataset for supervised fine-tuning.""" |
| 196 | |
| 197 | def __init__(self, raw_data, tokenizer: transformers.PreTrainedTokenizer, template="tool-llama"): |
| 198 | super(LazySupervisedDataset, self).__init__() |
| 199 | self.tokenizer = tokenizer |
| 200 | |
| 201 | rank0_print("Formatting inputs...Skip in lazy mode") |
| 202 | self.tokenizer = tokenizer |
| 203 | self.raw_data = raw_data |
| 204 | self.cached_data_dict = {} |
| 205 | self.template = template |
| 206 | |
| 207 | def __len__(self): |
| 208 | return len(self.raw_data) |
nothing calls this directly
no test coverage detected