Function preprocess

test/gsm8k/test.py:96–103

(
    sources,
    targets,
    tokenizer: transformers.PreTrainedTokenizer,
)

def preprocess(
    sources,
    targets,
    tokenizer: transformers.PreTrainedTokenizer,
):
    sources_tokenized = _tokenize_fn(sources, tokenizer)
    input_ids = sources_tokenized["input_ids"]
    return dict(input_ids=input_ids, labels=copy.deepcopy(input_ids))


class SupervisedDataset(Dataset):