(
dataset,
training_args,
sequence_parallel_size,
sequence_parallel_mode,
cutoff_len,
num_worker=16,
packing=False,
tokenizer=None,
cache_dataset_overwrite=False,
)
| 294 | return dataset |
| 295 | |
| 296 | def data_post_process_sequence_parallel( |
| 297 | dataset, |
| 298 | training_args, |
| 299 | sequence_parallel_size, |
| 300 | sequence_parallel_mode, |
| 301 | cutoff_len, |
| 302 | num_worker=16, |
| 303 | packing=False, |
| 304 | tokenizer=None, |
| 305 | cache_dataset_overwrite=False, |
| 306 | ): |
| 307 | dataset = dataset.shuffle(seed=training_args.seed) |
| 308 | if packing: |
| 309 | dataset = packing_dataset(dataset, tokenizer, cutoff_len, num_worker, cache_dataset_overwrite) |
| 310 | |
| 311 | dataset = _get_sequence_parallel_dataset(dataset, |
| 312 | num_works=num_worker, |
| 313 | tokenizer=tokenizer, |
| 314 | cutoff_len=cutoff_len, |
| 315 | sequence_parallel_size=sequence_parallel_size, |
| 316 | sequence_parallel_mode=sequence_parallel_mode, |
| 317 | cache_dataset_overwrite=cache_dataset_overwrite) |
| 318 | return dataset |
nothing calls this directly
no test coverage detected