Make dataset and collator for supervised fine-tuning.
(tokenizer: transformers.PreTrainedTokenizer, data_args)
| 252 | ) |
| 253 | |
| 254 | def make_supervised_data_module(tokenizer: transformers.PreTrainedTokenizer, data_args) -> Dict: |
| 255 | """Make dataset and collator for supervised fine-tuning.""" |
| 256 | train_dataset = SupervisedDataset(tokenizer=tokenizer, data_path=data_args.data_path, max_sample=data_args.max_train_samples, split="train") |
| 257 | eval_dataset = SupervisedDataset(tokenizer=tokenizer, data_path=data_args.data_path, max_sample=data_args.max_train_samples, split="eval") |
| 258 | data_collator = DataCollatorForSupervisedDataset(tokenizer=tokenizer) |
| 259 | return dict(train_dataset=train_dataset, eval_dataset=eval_dataset, data_collator=data_collator) |
| 260 | |
| 261 | |
| 262 | def train(): |
no test coverage detected