Function required_phrase_tokenizer

src/licensedcode/tokenize.py:90–119

Yield tokens from a rule ``text`` including required phrases {{brace}} markers. This tokenizer behaves the same as as the ``index_tokenizer`` returning also REQUIRED_PHRASE_OPEN and REQUIRED_PHRASE_CLOSE as separate tokens so that they can be used to parse required phrases. >>>

(text, stopwords=STOPWORDS, preserve_case=False)

def required_phrase_tokenizer(text, stopwords=STOPWORDS, preserve_case=False):
    """
    Yield tokens from a rule ``text`` including required phrases {{brace}} markers.
    This tokenizer behaves the same as as the ``index_tokenizer`` returning also
    REQUIRED_PHRASE_OPEN and REQUIRED_PHRASE_CLOSE as separate tokens so that they can be
    used to parse required phrases.

    >>> x = list(required_phrase_splitter('{{AGPL-3.0  GNU Affero License v3.0}}'))
    >>> assert x == ['{{', 'AGPL', '3', '0', 'GNU', 'Affero', 'License', 'v3', '0', '}}'], x

    >>> x = list(required_phrase_splitter('{{{AGPL{{{{Affero }}License}}0}}'))
    >>> assert x == ['{{', 'AGPL', '{{', '{{', 'Affero', '}}', 'License', '}}', '0', '}}'], x

    >>> list(index_tokenizer('')) == []
    True

    >>> x = list(index_tokenizer('{{AGPL-3.0  GNU Affero License v3.0}}'))
    >>> assert x == ['agpl', '3', '0', 'gnu', 'affero', 'license', 'v3', '0']

    >>> x = list(required_phrase_tokenizer('{{AGPL-3.0  GNU Affero License v3.0}}'))
    >>> assert x == ['{{', 'agpl', '3', '0', 'gnu', 'affero', 'license', 'v3', '0', '}}']
    """
    if not text:
        return
    if not preserve_case:
        text = text.lower()

    for token in required_phrase_splitter(text):
        if token and token not in stopwords:
            yield token


def get_existing_required_phrase_spans(text):