Provide train and validation datasets.
(args, tokenizer)
| 54 | |
| 55 | |
| 56 | def train_valid_datasets_provider(args, tokenizer): |
| 57 | """Provide train and validation datasets.""" |
| 58 | if args.task.lower() == 'blank': |
| 59 | train_dataset = BlankLMDataset(args, split='train', tokenizer=tokenizer) |
| 60 | valid_dataset = None |
| 61 | elif args.task.lower() == 'extraction': |
| 62 | train_dataset = ExtractionDataset(args, split='train', tokenizer=tokenizer) |
| 63 | valid_dataset = None |
| 64 | elif args.task.lower() == 'customization': |
| 65 | train_dataset = CustomizationDataset(args, split='train', tokenizer=tokenizer) |
| 66 | valid_dataset = None |
| 67 | else: |
| 68 | train_dataset = Seq2SeqDataset(args, split='train', tokenizer=tokenizer) |
| 69 | valid_dataset = None |
| 70 | global global_tokenizer |
| 71 | global_tokenizer = tokenizer |
| 72 | return train_dataset, valid_dataset |
| 73 | |
| 74 | |
| 75 | def metrics_func_provider(args, tokenizer, is_test): |
no test coverage detected