(self, raw_data, tokenizer: transformers.PreTrainedTokenizer, conv_template = "vicuna-1.1", mask_user = True)
| 296 | """Dataset for supervised fine-tuning.""" |
| 297 | |
| 298 | def __init__(self, raw_data, tokenizer: transformers.PreTrainedTokenizer, conv_template = "vicuna-1.1", mask_user = True): |
| 299 | super(LazySupervisedDataset, self).__init__() |
| 300 | self.tokenizer = tokenizer |
| 301 | |
| 302 | rank0_print("Formatting inputs...Skip in lazy mode") |
| 303 | self.conv_template = conv_template |
| 304 | self.mask_user = mask_user |
| 305 | self.tokenizer = tokenizer |
| 306 | self.raw_data = raw_data |
| 307 | self.cached_data_dict = {} |
| 308 | |
| 309 | if mask_user: |
| 310 | rank0_print( |
| 311 | f"WARNING: The loss of user prompt will be masked" |
| 312 | ) |
| 313 | else: |
| 314 | rank0_print( |
| 315 | f"WARNING: The loss of user prompt will **NOT** be masked" |
| 316 | ) |
| 317 | |
| 318 | def __len__(self): |
| 319 | return len(self.raw_data) |
nothing calls this directly
no test coverage detected