Read a SQuAD json file into a list of SquadExample.
(input_file, is_training,
version_2_with_negative,
translated_input_folder=None)
| 159 | |
| 160 | |
| 161 | def read_squad_examples(input_file, is_training, |
| 162 | version_2_with_negative, |
| 163 | translated_input_folder=None): |
| 164 | """Read a SQuAD json file into a list of SquadExample.""" |
| 165 | with tf.io.gfile.GFile(input_file, "r") as reader: |
| 166 | input_data = json.load(reader)["data"] |
| 167 | |
| 168 | if translated_input_folder is not None: |
| 169 | translated_files = tf.io.gfile.glob( |
| 170 | os.path.join(translated_input_folder, "*.json")) |
| 171 | for file in translated_files: |
| 172 | with tf.io.gfile.GFile(file, "r") as reader: |
| 173 | input_data.extend(json.load(reader)["data"]) |
| 174 | |
| 175 | def is_whitespace(c): |
| 176 | if c == " " or c == "\t" or c == "\r" or c == "\n" or ord(c) == 0x202F: |
| 177 | return True |
| 178 | return False |
| 179 | |
| 180 | examples = [] |
| 181 | for entry in input_data: |
| 182 | for paragraph in entry["paragraphs"]: |
| 183 | paragraph_text = paragraph["context"] |
| 184 | doc_tokens = [] |
| 185 | char_to_word_offset = [] |
| 186 | prev_is_whitespace = True |
| 187 | for c in paragraph_text: |
| 188 | if is_whitespace(c): |
| 189 | prev_is_whitespace = True |
| 190 | else: |
| 191 | if prev_is_whitespace: |
| 192 | doc_tokens.append(c) |
| 193 | else: |
| 194 | doc_tokens[-1] += c |
| 195 | prev_is_whitespace = False |
| 196 | char_to_word_offset.append(len(doc_tokens) - 1) |
| 197 | |
| 198 | for qa in paragraph["qas"]: |
| 199 | qas_id = qa["id"] |
| 200 | question_text = qa["question"] |
| 201 | start_position = None |
| 202 | end_position = None |
| 203 | orig_answer_text = None |
| 204 | is_impossible = False |
| 205 | if is_training: |
| 206 | |
| 207 | if version_2_with_negative: |
| 208 | is_impossible = qa["is_impossible"] |
| 209 | if (len(qa["answers"]) != 1) and (not is_impossible): |
| 210 | raise ValueError( |
| 211 | "For training, each question should have exactly 1 answer.") |
| 212 | if not is_impossible: |
| 213 | answer = qa["answers"][0] |
| 214 | orig_answer_text = answer["text"] |
| 215 | answer_offset = answer["answer_start"] |
| 216 | answer_length = len(orig_answer_text) |
| 217 | start_position = char_to_word_offset[answer_offset] |
| 218 | end_position = char_to_word_offset[answer_offset + answer_length - |
no test coverage detected