(batch)
| 125 | ds = load_dataset(cfg["dataset_name"], split=cfg["dataset_split"], trust_remote_code=True) |
| 126 | |
| 127 | def tokenise(batch): |
| 128 | encoded = tokenizer( |
| 129 | batch["text"], |
| 130 | truncation=True, |
| 131 | max_length=ctx, |
| 132 | padding="max_length", |
| 133 | return_tensors=None, |
| 134 | ) |
| 135 | encoded["labels"] = encoded["input_ids"].copy() |
| 136 | return encoded |
| 137 | |
| 138 | ds = ds.map(tokenise, batched=True, remove_columns=ds.column_names, num_proc=4) |
| 139 | ds.set_format(type="torch", columns=["input_ids", "attention_mask", "labels"]) |
nothing calls this directly
no outgoing calls
no test coverage detected