| 145 | |
| 146 | # method parameters are unpacked from `gen_kwargs` as given in `_split_generators` |
| 147 | def _generate_examples(self, filepath, split): |
| 148 | with open(filepath, encoding="utf-8") as f: |
| 149 | data = json.load(f) |
| 150 | key = 0 |
| 151 | for section_id, example in data.items(): |
| 152 | # Each example (passage) has multiple sub-question-answer pairs. |
| 153 | for qa in example["qa_pairs"]: |
| 154 | # Build answer. |
| 155 | answer = qa["answer"] |
| 156 | answer = { |
| 157 | "number": answer["number"], |
| 158 | "date": { |
| 159 | "day": answer["date"].get("day", ""), |
| 160 | "month": answer["date"].get("month", ""), |
| 161 | "year": answer["date"].get("year", ""), |
| 162 | }, |
| 163 | "spans": answer["spans"], |
| 164 | "worker_id": answer.get("worker_id", ""), |
| 165 | "hit_id": answer.get("hit_id", ""), |
| 166 | } |
| 167 | validated_answers = [] |
| 168 | if "validated_answers" in qa: |
| 169 | for validated_answer in qa["validated_answers"]: |
| 170 | va = { |
| 171 | "number": validated_answer.get("number", ""), |
| 172 | "date": { |
| 173 | "day": validated_answer["date"].get("day", ""), |
| 174 | "month": validated_answer["date"].get("month", ""), |
| 175 | "year": validated_answer["date"].get("year", ""), |
| 176 | }, |
| 177 | "spans": validated_answer.get("spans", ""), |
| 178 | "worker_id": validated_answer.get("worker_id", ""), |
| 179 | "hit_id": validated_answer.get("hit_id", ""), |
| 180 | } |
| 181 | validated_answers.append(va) |
| 182 | else: |
| 183 | validated_answers = _EMPTY_VALIDATED_ANSWER |
| 184 | yield key, { |
| 185 | "section_id": section_id, |
| 186 | "passage": example["passage"], |
| 187 | "question": qa["question"], |
| 188 | "query_id": qa["query_id"], |
| 189 | "answer": answer, |
| 190 | "validated_answers": validated_answers, |
| 191 | } |
| 192 | key += 1 |