| 213 | return data_dict |
| 214 | |
| 215 | def load_and_preprocess_it(tokenizer, args): |
| 216 | |
| 217 | raw_train_datasets = load_dataset( |
| 218 | args.data_path, |
| 219 | split=args.dataset_split) |
| 220 | |
| 221 | train_dataset = raw_train_datasets.map( |
| 222 | train_tokenize_function, |
| 223 | batched=True, |
| 224 | batch_size=3000, |
| 225 | num_proc=32, |
| 226 | remove_columns=raw_train_datasets.column_names, |
| 227 | load_from_cache_file=True, |
| 228 | desc="Running tokenizer on train dataset", |
| 229 | fn_kwargs={ |
| 230 | "tokenizer": tokenizer, |
| 231 | "query": args.dataset_field[0], |
| 232 | "response": args.dataset_field[1]} |
| 233 | ) |
| 234 | |
| 235 | return train_dataset |
| 236 | |
| 237 | |
| 238 | def load_and_preprocess_cr(tokenizer, args): |