(data_batch)
| 161 | ds = datasets.Dataset.from_dict({data_name: new_data_list}) |
| 162 | |
| 163 | def tokenize(data_batch): |
| 164 | # tokenize then pad each batch according to the longest sequence in the batch |
| 165 | batch = tokenizer( |
| 166 | data_batch[data_name], |
| 167 | padding="longest", |
| 168 | max_length=max_seqlen, |
| 169 | truncation=True, |
| 170 | return_tensors="pt", |
| 171 | ) |
| 172 | batch["labels"] = batch["input_ids"].clone() |
| 173 | return batch |
| 174 | |
| 175 | # tokenize lazily |
| 176 | ds.set_transform(tokenize) |
nothing calls this directly
no outgoing calls
no test coverage detected