(dataset, tokenizer, cutoff_len, num_worker, cache_dataset_overwrite)
| 278 | return sp_dataset |
| 279 | |
| 280 | def packing_dataset(dataset, tokenizer, cutoff_len, num_worker, cache_dataset_overwrite): |
| 281 | preprocess_func = partial(preprocess_packed_supervised_dataset, tokenizer=tokenizer, cutoff_len=cutoff_len) |
| 282 | kwargs = dict( |
| 283 | num_proc=num_worker, |
| 284 | load_from_cache_file=not cache_dataset_overwrite, |
| 285 | desc="Running postprocess on dataset", |
| 286 | ) |
| 287 | import pdb; pdb.set_trace() |
| 288 | dataset = dataset.map( |
| 289 | preprocess_func, |
| 290 | batched=True, |
| 291 | batch_size=num_worker, |
| 292 | **kwargs, |
| 293 | ) |
| 294 | return dataset |
| 295 | |
| 296 | def data_post_process_sequence_parallel( |
| 297 | dataset, |
no outgoing calls
no test coverage detected