Returns a list of sentences. Each sentence is a space-separated string of tokens (words). Handles common cases of abbreviations (e.g., etc., ...). Punctuation marks are split from other words. Periods (or ?!) mark the end of a sentence. Headings without an ending period are
(string, punctuation=PUNCTUATION, abbreviations=ABBREVIATIONS, replace=replacements, linebreak=r"\n{2,}")
| 746 | EOS = "END-OF-SENTENCE" |
| 747 | |
| 748 | def find_tokens(string, punctuation=PUNCTUATION, abbreviations=ABBREVIATIONS, replace=replacements, linebreak=r"\n{2,}"): |
| 749 | """ Returns a list of sentences. Each sentence is a space-separated string of tokens (words). |
| 750 | Handles common cases of abbreviations (e.g., etc., ...). |
| 751 | Punctuation marks are split from other words. Periods (or ?!) mark the end of a sentence. |
| 752 | Headings without an ending period are inferred by line breaks. |
| 753 | """ |
| 754 | # Handle periods separately. |
| 755 | punctuation = tuple(punctuation.replace(".", "")) |
| 756 | # Handle replacements (contractions). |
| 757 | for a, b in replace.items(): |
| 758 | string = re.sub(a, b, string) |
| 759 | # Handle Unicode quotes. |
| 760 | if isinstance(string, unicode): |
| 761 | string = string.replace(u"“", u" “ ") |
| 762 | string = string.replace(u"”", u" ” ") |
| 763 | string = string.replace(u"‘", u" ‘ ") |
| 764 | string = string.replace(u"’", u" ’ ") |
| 765 | # Collapse whitespace. |
| 766 | string = re.sub("\r\n", "\n", string) |
| 767 | string = re.sub(linebreak, " %s " % EOS, string) |
| 768 | string = re.sub(r"\s+", " ", string) |
| 769 | tokens = [] |
| 770 | for t in TOKEN.findall(string+" "): |
| 771 | if len(t) > 0: |
| 772 | tail = [] |
| 773 | while t.startswith(punctuation) and \ |
| 774 | not t in replace: |
| 775 | # Split leading punctuation. |
| 776 | if t.startswith(punctuation): |
| 777 | tokens.append(t[0]); t=t[1:] |
| 778 | while t.endswith(punctuation+(".",)) and \ |
| 779 | not t in replace: |
| 780 | # Split trailing punctuation. |
| 781 | if t.endswith(punctuation): |
| 782 | tail.append(t[-1]); t=t[:-1] |
| 783 | # Split ellipsis (...) before splitting period. |
| 784 | if t.endswith("..."): |
| 785 | tail.append("..."); t=t[:-3].rstrip(".") |
| 786 | # Split period (if not an abbreviation). |
| 787 | if t.endswith("."): |
| 788 | if t in abbreviations or \ |
| 789 | RE_ABBR1.match(t) is not None or \ |
| 790 | RE_ABBR2.match(t) is not None or \ |
| 791 | RE_ABBR3.match(t) is not None: |
| 792 | break |
| 793 | else: |
| 794 | tail.append(t[-1]); t=t[:-1] |
| 795 | if t != "": |
| 796 | tokens.append(t) |
| 797 | tokens.extend(reversed(tail)) |
| 798 | sentences, i, j = [[]], 0, 0 |
| 799 | while j < len(tokens): |
| 800 | if tokens[j] in ("...", ".", "!", "?", EOS): |
| 801 | # There may be a trailing parenthesis. |
| 802 | while j < len(tokens) \ |
| 803 | and tokens[j] in ("...", ".", "!", "?", ")", "'", "\"", u"”", u"’", EOS): |
| 804 | j += 1 |
| 805 | sentences[-1].extend(t for t in tokens[i:j] if t != EOS) |