Normalize the text by lowercasing and removing punctuation.
(
text: str,
remove_punct: bool = True,
lowercase: bool = True,
nfd_unicode: bool = True,
white_space: bool = True
)
| 6 | |
| 7 | |
| 8 | def normalize( |
| 9 | text: str, |
| 10 | remove_punct: bool = True, |
| 11 | lowercase: bool = True, |
| 12 | nfd_unicode: bool = True, |
| 13 | white_space: bool = True |
| 14 | ) -> str: |
| 15 | """ Normalize the text by lowercasing and removing punctuation. """ |
| 16 | # remove punctuation |
| 17 | if remove_punct: |
| 18 | text = text.translate(TRANSLATION_TABLE_PUNCTUATION) |
| 19 | |
| 20 | # lowercase |
| 21 | if lowercase: |
| 22 | text = text.lower() |
| 23 | |
| 24 | if white_space: |
| 25 | text = text.strip() |
| 26 | text = re.sub(r"\s+", " ", text) |
| 27 | |
| 28 | # NFD unicode normalization |
| 29 | if nfd_unicode: |
| 30 | text = unicodedata.normalize("NFD", text) |
| 31 | |
| 32 | return text |