Function get_sequence_parallel_preprocess

src/utils/data_utils.py:237–246

(stage, tokenizer, cutoff_len=None, sequence_parallel_size=1, sequence_parallel_mode="ulysses")

    return examples

def get_sequence_parallel_preprocess(stage, tokenizer, cutoff_len=None, sequence_parallel_size=1, sequence_parallel_mode="ulysses"):
    if stage == "pad":
        assert cutoff_len is not None
        preprocess_func = partial(pad_sequence, cutoff_len=cutoff_len, tokenizer=tokenizer)
    elif stage == "split":
        preprocess_func = partial(sp_split, sequence_parallel_size=sequence_parallel_size, sequence_parallel_mode=sequence_parallel_mode)
    else:
        raise NotImplementedError(f"Unexpected stage in sequence_parallel_preprocess: {stage}")

    return preprocess_func

def _get_sequence_parallel_dataset(dataset, num_works, tokenizer=None, cutoff_len=10000, 
                                   sequence_parallel_size=1, sequence_parallel_mode="ulysses",