| 110 | |
| 111 | |
| 112 | class TextNormalizer: |
| 113 | def __init__(self): |
| 114 | self.language_detector = LanguageDetectorBuilder.from_languages( |
| 115 | Language.ENGLISH, Language.CHINESE |
| 116 | ).build() |
| 117 | self.zh_normalizer = ZhNormalizer() |
| 118 | self.en_normalizer = EnNormalizer() |
| 119 | self.inflect_parser = inflect.engine() |
| 120 | self.lang2token = {Language.ENGLISH: "en", Language.CHINESE: "zh"} |
| 121 | |
| 122 | def tn(self, text): |
| 123 | text = preprocess_text(text) |
| 124 | text = rettt(text) # regex replacements |
| 125 | # for non chinese languages |
| 126 | language = self.language_detector.detect_language_of(text) |
| 127 | # enforce chinese if text contains any chinese character |
| 128 | if contains_chinese(text): |
| 129 | language = Language.CHINESE |
| 130 | text_lang = self.lang2token.get(language, "zh") |
| 131 | |
| 132 | if is_upper_eng_and_digit(text): |
| 133 | language = Language.CHINESE |
| 134 | |
| 135 | if language == Language.CHINESE: |
| 136 | text = self.zh_normalizer.normalize(text) |
| 137 | text = text.replace("\n", "") |
| 138 | text = re.sub(r"[，,]+$", "。", text) |
| 139 | else: |
| 140 | text = re.sub(r"[^ 0-9A-Za-z\[\]'.,:?!_\-]", "", text) |
| 141 | text = self.en_normalizer.normalize(text) |
| 142 | # fallback number normalization |
| 143 | pieces = re.split(r"(\d+)", text) |
| 144 | text = "".join( |
| 145 | [ |
| 146 | self.inflect_parser.number_to_words(p) if p.isnumeric() else p |
| 147 | for p in pieces |
| 148 | if len(p) > 0 |
| 149 | ] |
| 150 | ) |
| 151 | |
| 152 | # cleanup |
| 153 | text = text.replace("_", " ") |
| 154 | text = re.sub(r"[ ]+", " ", text) |
| 155 | |
| 156 | # spell caplital words |
| 157 | pieces = re.split(r"([A-Z]{2,4}|[ ])", text) |
| 158 | for idx, p in enumerate(pieces): |
| 159 | if re.match("[A-Z]{2,4}", p): |
| 160 | pieces[idx] = " ".join(p) |
| 161 | text = " ".join([p for p in pieces if p != " "]) |
| 162 | |
| 163 | # post TN full to half |
| 164 | text = text.replace("。", ".") |
| 165 | text = text.replace("，", ",") |
| 166 | text = text.replace("：", ":") |
| 167 | |
| 168 | # model limitations |
| 169 | text = text.lower().strip() |