Function index_tokenizer

src/licensedcode/tokenize.py:217–244

Return an iterable of tokens from a rule or query ``text`` using index tokenizing rules. Ignore words that exist as lowercase in the ``stopwords`` set. For example:: >>> list(index_tokenizer('')) [] >>> x = list(index_tokenizer('some Text with spAces! + _ -')) >>>

(text, stopwords=STOPWORDS, preserve_case=False)

def index_tokenizer(text, stopwords=STOPWORDS, preserve_case=False):
    """
    Return an iterable of tokens from a rule or query ``text`` using index
    tokenizing rules. Ignore words that exist as lowercase in the ``stopwords``
    set.

    For example::
    >>> list(index_tokenizer(''))
    []
    >>> x = list(index_tokenizer('some Text with   spAces! + _ -'))
    >>> assert x == ['some', 'text', 'with', 'spaces']

    >>> x = list(index_tokenizer('{{}some }}Text with   spAces! + _ -'))
    >>> assert x == ['some', 'text', 'with', 'spaces']

    >>> x = list(index_tokenizer('{{Hi}}some {{}}Text with{{noth+-_!@ing}}   {{junk}}spAces! + _ -{{}}'))
    >>> assert x == ['hi', 'some', 'text', 'with', 'noth+', 'ing', 'junk', 'spaces']

    >>> stops = set(['quot', 'lt', 'gt'])
    >>> x = list(index_tokenizer('some &quot&lt markup &gt&quot', stopwords=stops))
    >>> assert x == ['some', 'markup']
    """
    if not text:
        return []
    if not preserve_case:
        text = text.lower()
    words = word_splitter(text)
    return (token for token in words if token and token not in stopwords)


def index_tokenizer_with_stopwords(text, stopwords=STOPWORDS):