Function preprocess

data/generation/single_generate.py:84–91

(
    sources,
    targets,
    tokenizer: transformers.PreTrainedTokenizer,
)

    )

def preprocess(
    sources,
    targets,
    tokenizer: transformers.PreTrainedTokenizer,
):
    sources_tokenized = _tokenize_fn(sources, tokenizer)
    input_ids = sources_tokenized["input_ids"]
    return dict(input_ids=input_ids, labels=copy.deepcopy(input_ids))

class SupervisedDataset(Dataset):
    """Dataset for supervised fine-tuning."""