Split long text into sentences with sentence-splitting punctuations. Args: text (str): The input text. Returns: List[str]: Sentences.
(self, text: str, lang="zh")
| 61 | self.SENTENCE_SPLITOR = re.compile(r'([：、，；。？！,;?!][”’]?)') |
| 62 | |
| 63 | def _split(self, text: str, lang="zh") -> List[str]: |
| 64 | """Split long text into sentences with sentence-splitting punctuations. |
| 65 | Args: |
| 66 | text (str): The input text. |
| 67 | Returns: |
| 68 | List[str]: Sentences. |
| 69 | """ |
| 70 | # Only for pure Chinese here |
| 71 | if lang == "zh": |
| 72 | text = text.replace(" ", "") |
| 73 | # 过滤掉特殊字符 |
| 74 | text = re.sub(r'[——《》【】<>{}()（）#&@“”^_|\\]', '', text) |
| 75 | text = self.SENTENCE_SPLITOR.sub(r'\1\n', text) |
| 76 | text = text.strip() |
| 77 | sentences = [sentence.strip() for sentence in re.split(r'\n+', text)] |
| 78 | return sentences |
| 79 | |
| 80 | def _post_replace(self, sentence: str) -> str: |
| 81 | sentence = sentence.replace('/', '每') |