(example, tokenizer)
| 9 | device = torch.device( 'cuda' ) if torch.cuda.is_available() else torch.device( 'cpu' ) |
| 10 | |
| 11 | def preprocess_data(example, tokenizer): |
| 12 | tokenizer.pad_token = tokenizer.eos_token |
| 13 | return tokenizer(example["text"], truncation=True, padding="max_length", max_length=512) |
| 14 | |
| 15 | def collate_fn(batch): |
| 16 | input_ids = torch.stack([torch.tensor(b["input_ids"]) for b in batch]) |
nothing calls this directly
no outgoing calls
no test coverage detected