One left & one right token, both case-normalized. Skip over non-sentence-final punctuation. Used by the ``ContextIndex`` that is created for ``similar()`` and ``common_contexts()``.
(self, tokens, i)
| 665 | _CONTEXT_RE = re.compile(r"\w+|[\.\!\?]") |
| 666 | |
| 667 | def _context(self, tokens, i): |
| 668 | """ |
| 669 | One left & one right token, both case-normalized. Skip over |
| 670 | non-sentence-final punctuation. Used by the ``ContextIndex`` |
| 671 | that is created for ``similar()`` and ``common_contexts()``. |
| 672 | """ |
| 673 | # Left context |
| 674 | j = i - 1 |
| 675 | while j >= 0 and not self._CONTEXT_RE.match(tokens[j]): |
| 676 | j -= 1 |
| 677 | left = tokens[j] if j != 0 else "*START*" |
| 678 | |
| 679 | # Right context |
| 680 | j = i + 1 |
| 681 | while j < len(tokens) and not self._CONTEXT_RE.match(tokens[j]): |
| 682 | j += 1 |
| 683 | right = tokens[j] if j != len(tokens) else "*END*" |
| 684 | |
| 685 | return (left, right) |
| 686 | |
| 687 | # //////////////////////////////////////////////////////////// |
| 688 | # String Display |