Splits punctuation on a piece of text.
(self, text, never_split=None)
| 412 | return "".join(output) |
| 413 | |
| 414 | def _run_split_on_punc(self, text, never_split=None): |
| 415 | """Splits punctuation on a piece of text.""" |
| 416 | if never_split is not None and text in never_split: |
| 417 | return [text] |
| 418 | chars = list(text) |
| 419 | i = 0 |
| 420 | start_new_word = True |
| 421 | output = [] |
| 422 | while i < len(chars): |
| 423 | char = chars[i] |
| 424 | if _is_punctuation(char): |
| 425 | output.append([char]) |
| 426 | start_new_word = True |
| 427 | else: |
| 428 | if start_new_word: |
| 429 | output.append([]) |
| 430 | start_new_word = False |
| 431 | output[-1].append(char) |
| 432 | i += 1 |
| 433 | |
| 434 | return ["".join(x) for x in output] |
| 435 | |
| 436 | def _tokenize_chinese_chars(self, text): |
| 437 | """Adds whitespace around any CJK character.""" |
no test coverage detected