| 152 | return glucose_spec_data |
| 153 | |
| 154 | class GlucoseDataset(Dataset): |
| 155 | def __init__(self, data, tokenizer, source_col="input", target_col="target", max_source_len=512, max_target_len=512): |
| 156 | self.tokenizer = tokenizer |
| 157 | self.data = data |
| 158 | self.max_source_len = max_source_len |
| 159 | self.max_target_len = max_target_len |
| 160 | self.source_col = source_col |
| 161 | self.target_col = target_col |
| 162 | |
| 163 | def __len__(self): |
| 164 | return len(self.data) |
| 165 | |
| 166 | def __getitem__(self, index): |
| 167 | source = self.tokenizer( |
| 168 | [self.data.loc[index, self.source_col]], |
| 169 | padding="max_length", |
| 170 | max_length=self.max_source_len, |
| 171 | return_tensors="pt", |
| 172 | truncation=True, |
| 173 | ) |
| 174 | target = self.tokenizer( |
| 175 | [self.data.loc[index, self.target_col]], |
| 176 | padding="max_length", |
| 177 | max_length=self.max_target_len, |
| 178 | return_tensors="pt", |
| 179 | truncation=True, |
| 180 | ) |
| 181 | |
| 182 | source_ids = source["input_ids"].squeeze() |
| 183 | source_mask = source["attention_mask"].squeeze() |
| 184 | target_ids = target["input_ids"].squeeze() |
| 185 | |
| 186 | return { |
| 187 | "input_ids": source_ids.to(dtype=torch.long), |
| 188 | "attention_mask": source_mask.to(dtype=torch.long), |
| 189 | "labels": target_ids.to(dtype=torch.long), |
| 190 | } |
| 191 | |
| 192 | @dataclass(kw_only=True) |
| 193 | class PhraseConfig: |
nothing calls this directly
no outgoing calls
no test coverage detected