| 6 | |
| 7 | @TEXT_POSTPROCESSORS.register_module('general') |
| 8 | def general_postprocess(text: str) -> str: |
| 9 | # Cut off the first newline, period, or comma |
| 10 | truncated_text = re.split(r'[\n.,]', text, 1)[0] |
| 11 | |
| 12 | # Remove punctuation |
| 13 | no_punctuation = re.sub(r'[^\w\s]', '', truncated_text) |
| 14 | |
| 15 | # Remove article |
| 16 | no_articles = re.sub(r'\b(a|an|the)\b', |
| 17 | '', |
| 18 | no_punctuation, |
| 19 | flags=re.IGNORECASE) |
| 20 | |
| 21 | # Remove duplicated blank spaces |
| 22 | cleaned_text = re.sub(r'\s+', ' ', no_articles).strip() |
| 23 | |
| 24 | return cleaned_text |
| 25 | |
| 26 | |
| 27 | @TEXT_POSTPROCESSORS.register_module('general_cn') |