Function tokenize_matched_text

src/licensedcode/match.py:2918–2946

Return a list of Token objects with pos and line number collected from the file at `location` or the `query_string` string. `dictionary` is the index mapping a token string to a token id. NOTE: the _cache={} arg IS A GLOBAL mutable by design.

(
    location,
    query_string,
    dictionary,
    start_line=1,
    _cache={},
)

def tokenize_matched_text(
    location,
    query_string,
    dictionary,
    start_line=1,
    _cache={},
):
    """
    Return a list of Token objects with pos and line number collected from the
    file at `location` or the `query_string` string. `dictionary` is the index
    mapping a token string to a token id.

    NOTE: the _cache={} arg IS A GLOBAL mutable by design.
    """
    key = location, query_string, start_line
    cached = _cache.get(key)
    if cached:
        return cached
    # we only cache the last call
    _cache.clear()
    _cache[key] = result = list(
        _tokenize_matched_text(
            location=location,
            query_string=query_string,
            dictionary=dictionary,
            start_line=start_line,
        )
    )
    return result


def _tokenize_matched_text(