(tokenizer, dataset_name, max_length, num_samples=200)
| 71 | |
| 72 | |
| 73 | def prepare_data(tokenizer, dataset_name, max_length, num_samples=200): |
| 74 | ds = load_dataset(dataset_name, split="train") |
| 75 | ds = ds.select(range(min(num_samples, len(ds)))) |
| 76 | |
| 77 | def tokenize(example): |
| 78 | text = format_alpaca(example) |
| 79 | enc = tokenizer(text, truncation=True, max_length=max_length, padding="max_length") |
| 80 | enc["labels"] = enc["input_ids"].copy() |
| 81 | return enc |
| 82 | |
| 83 | ds = ds.map(tokenize, remove_columns=ds.column_names) |
| 84 | return ds |
| 85 | |
| 86 | |
| 87 | def collate_fn(batch): |
no outgoing calls
no test coverage detected