| 391 | |
| 392 | |
| 393 | def blanklm_fix_tokenization(text): |
| 394 | text = text.replace("` `", "``") |
| 395 | text = text.replace("\' \'", "\'\'") |
| 396 | text = text.replace("n \' t", "n\'t") |
| 397 | text = text.replace("\' s", "\'s") |
| 398 | text = text.replace("\' m", "\'m") |
| 399 | text = text.replace("\' re", "\'re") |
| 400 | text = text.replace(". . .", "...") |
| 401 | text = text.replace(" . .", " ..") |
| 402 | text = text.replace("- -", "--") |
| 403 | text = text.replace("u . s .", "u.s.") |
| 404 | text = text.replace("u . k .", "u.k.") |
| 405 | text = text.replace("e . g .", "e.g.") |
| 406 | return text |
| 407 | |
| 408 | |
| 409 | class BlankLMEvaluater(DecoderEvaluater): |