(dataset)
| 27 | # from .starcoder import StarcoderRawDataset |
| 28 | |
| 29 | def build_evaluation_dataset(dataset): |
| 30 | |
| 31 | args = get_args() |
| 32 | build_tokenizer(args) |
| 33 | if dataset == 'LLama-SFT' or dataset == 'LLama-Pretrain-Raw': |
| 34 | val_dataset = LLamaRawDataset(args.valid_data_path, args.max_padding_length) |
| 35 | return val_dataset |
| 36 | else: |
| 37 | raise NotImplementedError('dataset {} is not implemented.'.format(dataset)) |
| 38 | |
| 39 | def build_finetune_dataset(dataset): |
| 40 |
no test coverage detected