| 190 | |
| 191 | |
| 192 | def squad_fix_tokenization(text): |
| 193 | text = text.replace(" - ", "-") |
| 194 | text = text.replace(" \u2013 ", "\u2013") |
| 195 | text = re.sub(r"\ba \. m \.\b", "a.m.", text) |
| 196 | text = re.sub(r"\ba \. m\b", "a.m", text) |
| 197 | text = re.sub(r"\bp \. m \.\b", "p.m.", text) |
| 198 | text = re.sub(r"\bp \. m\b", "p.m", text) |
| 199 | text = re.sub(r"\b' s\b", "'s", text) |
| 200 | text = re.sub(r"\bu \. s \.\b", "u.s.", text) |
| 201 | text = re.sub(r"\bu \. s\b", "u.s", text) |
| 202 | tokens = text.split() |
| 203 | i = 0 |
| 204 | while i < len(tokens): |
| 205 | if tokens[i] in [",", ".", ":"] and i > 0 and i+1 < len(tokens): |
| 206 | if tokens[i - 1][-1].isdigit() and tokens[i + 1][0].isdigit(): |
| 207 | if tokens[i] == ',' and len(tokens[i + 1]) > 3: |
| 208 | pass |
| 209 | else: |
| 210 | tokens[i - 1] = tokens[i - 1] + tokens[i] + tokens[i + 1] |
| 211 | tokens = tokens[:i] + tokens[i+2:] |
| 212 | i -= 1 |
| 213 | i += 1 |
| 214 | text = ' '.join(tokens) |
| 215 | return text |
| 216 | |
| 217 | |
| 218 | def squad_decode(example, prediction, tokenizer): |