(self, raw_data, tokenizer: transformers.PreTrainedTokenizer, conv_template = "vicuna-1.1", mask_user = True)
| 261 | """Dataset for supervised fine-tuning.""" |
| 262 | |
| 263 | def __init__(self, raw_data, tokenizer: transformers.PreTrainedTokenizer, conv_template = "vicuna-1.1", mask_user = True): |
| 264 | super(SupervisedDataset, self).__init__() |
| 265 | |
| 266 | rank0_print("Formatting inputs...") |
| 267 | sources = [example["conversations"] for example in raw_data] |
| 268 | data_dict = preprocess(sources, tokenizer, conv_template, mask_user) |
| 269 | |
| 270 | if mask_user: |
| 271 | rank0_print( |
| 272 | f"WARNING: The loss of user prompt will be masked" |
| 273 | ) |
| 274 | else: |
| 275 | rank0_print( |
| 276 | f"WARNING: The loss of user prompt will **NOT** be masked" |
| 277 | ) |
| 278 | |
| 279 | |
| 280 | self.input_ids = data_dict["input_ids"] |
| 281 | self.labels = data_dict["labels"] |
| 282 | self.attention_mask = data_dict["attention_mask"] |
| 283 | |
| 284 | def __len__(self): |
| 285 | return len(self.input_ids) |
no test coverage detected