Class LicenseIndex

src/licensedcode/index.py:131–1202

A license detection index. An index is queried for license matches found in a query file. The index support multiple strategies for finding exact and approximate matches.

class LicenseIndex(object):
    """
    A license detection index. An index is queried for license matches found in
    a query file. The index support multiple strategies for finding exact and
    approximate matches.
    """
    # slots are not really needed but they help with sanity and avoid an
    # unchecked proliferation of new attributes
    __slots__ = (
        'len_legalese',
        'dictionary',
        'digit_only_tids',

        'rules_by_id',
        'rules_by_rid',
        'tids_by_rid',

        'high_postings_by_rid',

        'sets_by_rid',
        'msets_by_rid',

        'rid_by_hash',
        'rules_automaton',
        'fragments_automaton',
        'starts_automaton',
        'unknown_automaton',

        'regular_rids',
        'false_positive_rids',
        'approx_matchable_rids',

        'optimized',
        'all_languages',
    )

    def __init__(
        self,
        rules=None,
        _legalese=common_license_words,
        _spdx_tokens=frozenset(),
        _license_tokens=frozenset(),
        _all_languages=False,
    ):
        """
        Initialize the index with an iterable of Rule objects.
        ``_legalese`` is a sorted mapping of common license-specific words aka. legalese as {token: id}
        ``_spdx_tokens`` is a set of tokens used in SPDX license identifiers
        ``_license_tokens`` is a set of "license" tokens used as start or end of a rule
        If ``_all_languages`` is True, use all spoken languages license and rules.
        Otherwise, use only English rules and licenses.
        """
        # largest token ID for a "legalese" token. A token with a larger id than
        # len_legalese is considered a "junk" very common token
        self.len_legalese = 0

        # mapping of token string > integer token id
        self.dictionary = {}