Creating a custom dataset for reading the dataset and loading it into the dataloader to pass it to the neural network for finetuning the model
| 66 | return problems, qids, name_maps, image_features |
| 67 | |
| 68 | class ScienceQADatasetStd(Dataset): |
| 69 | """ |
| 70 | Creating a custom dataset for reading the dataset and |
| 71 | loading it into the dataloader to pass it to the |
| 72 | neural network for finetuning the model |
| 73 | |
| 74 | """ |
| 75 | |
| 76 | def __init__( |
| 77 | self, problems, qids, tokenizer, source_len, target_len, args, test_le=None |
| 78 | ): |
| 79 | self.tokenizer = tokenizer |
| 80 | self.data = {qid : problems[qid] for qid in qids} |
| 81 | self.source_len = source_len |
| 82 | self.summ_len = target_len |
| 83 | self.target_text = [] |
| 84 | self.source_text = [] |
| 85 | if test_le is not None: |
| 86 | test_le_data =json.load(open(test_le))["preds"] |
| 87 | else: |
| 88 | test_le_data = None |
| 89 | idx = 0 |
| 90 | for qid in self.data: |
| 91 | if test_le_data is not None: |
| 92 | curr_le_data = test_le_data[idx] |
| 93 | idx += 1 |
| 94 | else: |
| 95 | curr_le_data = None |
| 96 | prompt, target = build_train_pair(problems, qid, args, curr_le_data) |
| 97 | self.target_text.append(target) |
| 98 | self.source_text.append(prompt) |
| 99 | |
| 100 | def __len__(self): |
| 101 | return len(self.target_text) |
| 102 | |
| 103 | def __getitem__(self, index): |
| 104 | source_text = str(self.source_text[index]) |
| 105 | target_text = str(self.target_text[index]) |
| 106 | |
| 107 | # cleaning data so as to ensure data is in string type |
| 108 | source_text = " ".join(source_text.split()) |
| 109 | target_text = " ".join(target_text.split()) |
| 110 | |
| 111 | source = self.tokenizer.batch_encode_plus( |
| 112 | [source_text], |
| 113 | max_length=self.source_len, |
| 114 | pad_to_max_length=True, |
| 115 | truncation=True, |
| 116 | padding="max_length", |
| 117 | return_tensors="pt", |
| 118 | ) |
| 119 | target = self.tokenizer.batch_encode_plus( |
| 120 | [target_text], |
| 121 | max_length=self.summ_len, |
| 122 | pad_to_max_length=True, |
| 123 | truncation=True, |
| 124 | padding="max_length", |
| 125 | return_tensors="pt", |