(data_args)
| 86 | ) |
| 87 | |
| 88 | def make_supervised_data_module(data_args) -> Dict: |
| 89 | print("loading data...") |
| 90 | if data_args.batch_method == "naive": |
| 91 | train_dataset = LMDataset(data_args.train_file) |
| 92 | data_collator = DataCollatorForLMDataset() |
| 93 | elif data_args.batch_method == "pack": |
| 94 | train_dataset = LMPackDataset(data_args.train_file) |
| 95 | data_collator = DataCollatorForLMPackDataset() |
| 96 | elif data_args.batch_method == "sort": |
| 97 | train_dataset = LMSortDataset(data_args.train_file) |
| 98 | data_collator = DataCollatorForLMDataset() |
| 99 | print("finish loading data") |
| 100 | return dict(train_dataset=train_dataset, data_collator=data_collator) |
| 101 | |
| 102 | def train(): |
| 103 | parser = transformers.HfArgumentParser((ModelArguments, DataArguments, TrainingArguments)) |
no test coverage detected