(self, dataset_name: str, tokenizer: transformers.PreTrainedTokenizer, max_sample=None)
| 93 | class SupervisedDataset(Dataset): |
| 94 | """Dataset for supervised fine-tuning.""" |
| 95 | def __init__(self, dataset_name: str, tokenizer: transformers.PreTrainedTokenizer, max_sample=None): |
| 96 | super(SupervisedDataset, self).__init__() |
| 97 | |
| 98 | sources, targets = get_gen_dataset(dataset_name, max_sample, tokenizer) |
| 99 | |
| 100 | data_dict = preprocess(sources, targets, tokenizer) |
| 101 | |
| 102 | self.input_ids = data_dict["input_ids"] |
| 103 | self.labels = data_dict["labels"] |
| 104 | |
| 105 | def __len__(self): |
| 106 | return len(self.input_ids) |
nothing calls this directly
no test coverage detected