Function get_alpaca_dataset

data/generation/data_utils.py:127–139

(max_sample, tokenizer)

def get_alpaca_dataset(max_sample, tokenizer):
    alpaca_dataset = load_dataset("yahma/alpaca-cleaned", split='train')
    # alpaca_dataset = load_dataset("/root/model/datasets/alpaca-clean", split='train')

    prompt_input, prompt_no_input = ALPACA_PROMPT_DICT["prompt_input"], ALPACA_PROMPT_DICT["prompt_no_input"]

    sources = [
                prompt_input.format_map(example) if example.get("input", "") != "" else prompt_no_input.format_map(example)
                for example in alpaca_dataset
                ]
    targets = [f"{example['output']}{tokenizer.eos_token}" for example in alpaca_dataset]

    return extract_random_dataset(sources, targets, max_sample)

def get_alpaca_solar_dataset(max_sample, tokenizer):
    alpaca_dataset = load_dataset("yahma/alpaca-cleaned", split='train')