Function get_code_dataset

data/generation/data_utils.py:191–204

(max_sample, tokenizer)

    return extract_random_dataset(sources, targets, max_sample)

def get_code_dataset(max_sample, tokenizer):
    code_dataset = load_dataset("nickrosh/Evol-Instruct-Code-80k-v1", split='train')
    # code_dataset = load_dataset('json', data_files="/root/model/datasets/code/EvolInstruct-Code-80k.json", split='train')

    prompt_input, prompt_no_input = CODE_PROMPT_DICT["prompt_input"], CODE_PROMPT_DICT["prompt_no_input"]

    sources = [
                prompt_input.format_map(example) if example.get("input", "") != "" else prompt_no_input.format_map(example)
                for example in code_dataset
                ]

    targets = [f"{example['output']}{tokenizer.eos_token}" for example in code_dataset]

    return extract_random_dataset(sources, targets, max_sample)

def get_math_dataset(max_sample, tokenizer):
    math_dataset = load_dataset("meta-math/MetaMathQA-40K", split='train')