(sentence)
| 13 | |
| 14 | |
| 15 | def preprocess_text(sentence): |
| 16 | # preprocessing |
| 17 | sentence = bytes(sentence, "utf-8").decode("utf-8", "ignore") |
| 18 | sentence = regex.sub("[\p{Cf}--[\u200d]]", "", sentence, flags=regex.V1) |
| 19 | sentence = regex.sub("\p{Co}", "", sentence) |
| 20 | sentence = sentence.replace("\u00a0", " ") |
| 21 | sentence = sentence.replace("\ufffd", "") |
| 22 | sentence = regex.sub("\p{Zl}", "\n", sentence) |
| 23 | sentence = regex.sub("\p{Zp}", "\n", sentence) |
| 24 | |
| 25 | sentence = unicode(sentence) |
| 26 | sentence = "".join( |
| 27 | char |
| 28 | for char in unicodedata.normalize("NFD", sentence) |
| 29 | if unicodedata.category(char) != "Mn" |
| 30 | ) # Strip accents |
| 31 | |
| 32 | sentence = strip_kaomoji(sentence) |
| 33 | # full to half with exemption (to be converted after number TN): 。，： |
| 34 | sentence = f2b(sentence, exemption="。，：") |
| 35 | |
| 36 | # clean spaces |
| 37 | sentence = sentence.replace("\n", "，") |
| 38 | sentence = sentence.replace("\t", "，") |
| 39 | sentence = sentence.replace("\r", "，") |
| 40 | sentence = re.sub(r"[。.]{3,}", "…", sentence) |
| 41 | sentence = re.sub(r"[…⋯]{1,}", "…", sentence) |
| 42 | sentence = re.sub(r"[ ]+", " ", sentence) |
| 43 | sentence = sentence.strip() |
| 44 | |
| 45 | # punctuation reduction |
| 46 | result = "" |
| 47 | for idx, char in enumerate(sentence): |
| 48 | if char in symbol_reduction: |
| 49 | char = symbol_reduction[char] |
| 50 | |
| 51 | if char == " ": |
| 52 | if idx == 0: |
| 53 | continue |
| 54 | if is_chinese(sentence[idx + 1]) and ( |
| 55 | is_chinese(sentence[idx - 1]) or sentence[idx - 1] in '") ' |
| 56 | ): |
| 57 | result += "，" |
| 58 | else: |
| 59 | result += " " |
| 60 | continue |
| 61 | |
| 62 | if is_valid_char(char): |
| 63 | result += char |
| 64 | result = re.sub(r"[ ]+", " ", result) |
| 65 | return result |
| 66 | |
| 67 | |
| 68 | def rettt(sentence): |
no test coverage detected