(max_sample, tokenizer)
| 204 | return extract_random_dataset(sources, targets, max_sample) |
| 205 | |
| 206 | def get_math_dataset(max_sample, tokenizer): |
| 207 | math_dataset = load_dataset("meta-math/MetaMathQA-40K", split='train') |
| 208 | # math_dataset = load_dataset('json', data_files="/root/model/acr_duda/gsm8k/data/MetaMath-40K.json", split='train') |
| 209 | prompt_no_input = MATH_PROMPT_DICT["prompt_no_input"] |
| 210 | |
| 211 | sources = [prompt_no_input.format_map(example) for example in math_dataset] |
| 212 | |
| 213 | targets = [f"{example['response']}{tokenizer.eos_token}" for example in math_dataset] |
| 214 | |
| 215 | return extract_random_dataset(sources, targets, max_sample) |
no test coverage detected