(max_sample, tokenizer)
| 173 | return extract_random_dataset(sources, targets, max_sample) |
| 174 | |
| 175 | def get_ultra_solar_dataset(max_sample, tokenizer): |
| 176 | # alpaca_dataset = load_dataset("yahma/alpaca-cleaned", split='train') |
| 177 | ultra_dataset = load_dataset("/root/model/datasets/ultrafeedback_binarized_cleaned", split='train_sft') |
| 178 | |
| 179 | prompt_no_input = ULTRA_PROMPT_DICT_SOLAR["prompt_no_input"] |
| 180 | |
| 181 | sources = [] |
| 182 | targets = [] |
| 183 | for example in ultra_dataset: |
| 184 | if len(example['prompt']) > 1024: |
| 185 | continue |
| 186 | sources.append(prompt_no_input.format_map(example)) |
| 187 | targets.append(f" ") |
| 188 | |
| 189 | return extract_random_dataset(sources, targets, max_sample) |
| 190 | |
| 191 | def get_code_dataset(max_sample, tokenizer): |
| 192 | code_dataset = load_dataset("nickrosh/Evol-Instruct-Code-80k-v1", split='train') |
no test coverage detected