(max_sample, tokenizer)
| 139 | return extract_random_dataset(sources, targets, max_sample) |
| 140 | |
| 141 | def get_alpaca_solar_dataset(max_sample, tokenizer): |
| 142 | alpaca_dataset = load_dataset("yahma/alpaca-cleaned", split='train') |
| 143 | # alpaca_dataset = load_dataset("/root/model/datasets/alpaca-clean", split='train') |
| 144 | |
| 145 | prompt_input, prompt_no_input = ALPACA_PROMPT_DICT_SOLAR["prompt_input"], ALPACA_PROMPT_DICT_SOLAR["prompt_no_input"] |
| 146 | |
| 147 | sources = [ |
| 148 | prompt_input.format_map(example) if example.get("input", "") != "" else prompt_no_input.format_map(example) |
| 149 | for example in alpaca_dataset |
| 150 | ] |
| 151 | targets = [f"{example['output']}{tokenizer.eos_token}" for example in alpaca_dataset] |
| 152 | |
| 153 | return extract_random_dataset(sources, targets, max_sample) |
| 154 | |
| 155 | def get_openorca_solar_dataset(max_sample, tokenizer): |
| 156 | openorca_dataset = load_dataset("Open-Orca/OpenOrca", split='train') |
no test coverage detected