Make dataset and collator for supervised fine-tuning.
(tokenizer: transformers.PreTrainedTokenizer, data_path)
| 185 | |
| 186 | |
| 187 | def make_supervised_data_module(tokenizer: transformers.PreTrainedTokenizer, data_path) -> Dict: |
| 188 | """Make dataset and collator for supervised fine-tuning.""" |
| 189 | eval_dataset = SupervisedDataset(tokenizer=tokenizer, data_path=data_path) |
| 190 | data_collator = DataCollatorForSupervisedDataset(tokenizer=tokenizer) |
| 191 | return eval_dataset, data_collator |
| 192 | |
| 193 | |
| 194 | def main(rank, args): |
no test coverage detected