| 359 | |
| 360 | |
| 361 | def post_process(sentence: str, symbol: str): |
| 362 | if symbol == "sentencepiece": |
| 363 | sentence = sentence.replace(" ", "").replace("\u2581", " ").strip() |
| 364 | elif symbol == "wordpiece": |
| 365 | sentence = sentence.replace(" ", "").replace("_", " ").strip() |
| 366 | elif symbol == "letter": |
| 367 | sentence = sentence.replace(" ", "").replace("|", " ").strip() |
| 368 | elif symbol == "_EOW": |
| 369 | sentence = sentence.replace(" ", "").replace("_EOW", " ").strip() |
| 370 | elif symbol == "bert": |
| 371 | sentence = sentence.replace(" ##", "").rstrip() |
| 372 | elif symbol in {"subword_nmt", "@@ ", "@@"}: |
| 373 | if symbol == "subword_nmt": |
| 374 | symbol = "@@ " |
| 375 | sentence = (sentence + " ").replace(symbol, "").rstrip() |
| 376 | elif symbol == "none": |
| 377 | pass |
| 378 | elif symbol is not None: |
| 379 | raise NotImplementedError(f"Unknown post_process option: {symbol}") |
| 380 | return sentence |
| 381 | |
| 382 | |
| 383 | def compute_mask_indices( |