(nsamples, seed, seqlen, tokenizer)
| 61 | |
| 62 | # Load and process wikitext2 dataset |
| 63 | def get_wikitext2(nsamples, seed, seqlen, tokenizer): |
| 64 | # Load train and test datasets |
| 65 | testdata = load_dataset("wikitext", "wikitext-2-raw-v1", split="test") |
| 66 | testenc = tokenizer("\n\n".join(testdata["text"]), return_tensors="pt") |
| 67 | |
| 68 | return None, testenc |
| 69 | |
| 70 | |
| 71 | def get_alpaca(nsamples, seed, seqlen, tokenizer, disentangle=False, dataset="alpaca"): |