| 341 | |
| 342 | |
| 343 | class MyDataset(Dataset): |
| 344 | def __init__(self, input_path): |
| 345 | # data = [] |
| 346 | with open(input_path) as f: |
| 347 | data = json.load(f) |
| 348 | print(f"loading {len(data)} data from {input_path}") |
| 349 | self.data = data |
| 350 | |
| 351 | |
| 352 | def __getitem__(self, index): |
| 353 | item: dict = self.data[index] |
| 354 | return item |
| 355 | |
| 356 | def __len__(self): |
| 357 | return len(self.data) |
| 358 | |
| 359 | def collate_fn(self, batch): |
| 360 | # print(batch); exit() |
| 361 | ''' |
| 362 | [id: '', title: '', description: '', QA_pairs: [ |
| 363 | {question: '', answer: ''}, |
| 364 | {question: '', answer: ''}, |
| 365 | ]] |
| 366 | ''' |
| 367 | return batch |
| 368 | |
| 369 | |