Method __init__

src/licensedcode/index.py:167–268

Initialize the index with an iterable of Rule objects. ``_legalese`` is a sorted mapping of common license-specific words aka. legalese as {token: id} ``_spdx_tokens`` is a set of tokens used in SPDX license identifiers ``_license_tokens`` is a set of "license" token

(
        self,
        rules=None,
        _legalese=common_license_words,
        _spdx_tokens=frozenset(),
        _license_tokens=frozenset(),
        _all_languages=False,
    )

    )

    def __init__(
        self,
        rules=None,
        _legalese=common_license_words,
        _spdx_tokens=frozenset(),
        _license_tokens=frozenset(),
        _all_languages=False,
    ):
        """
        Initialize the index with an iterable of Rule objects.
        ``_legalese`` is a sorted mapping of common license-specific words aka. legalese as {token: id}
        ``_spdx_tokens`` is a set of tokens used in SPDX license identifiers
        ``_license_tokens`` is a set of "license" tokens used as start or end of a rule
        If ``_all_languages`` is True, use all spoken languages license and rules.
        Otherwise, use only English rules and licenses.
        """
        # largest token ID for a "legalese" token. A token with a larger id than
        # len_legalese is considered a "junk" very common token
        self.len_legalese = 0

        # mapping of token string > integer token id
        self.dictionary = {}

        # set of token ids made entirely of digits
        self.digit_only_tids = set()

        # Note: all the following are mappings-like (using lists) of
        # rid-> data are lists of data where the index integer is the rule id.

        # mapping of rule int id -> rule objects
        # TODO: rename to rules_by_identifier for consistency
        self.rules_by_id = {}

        # maping-like of rule int id -> rule objects proper
        self.rules_by_rid = []

        # maping-like of rule int id -> sequence of token int ids
        self.tids_by_rid = []

        # mapping-like of rule id->(mapping of (token_id->[positions, ...])
        # We track only high/good tokens there. This is a "traditional"
        # inverted index postings list
        self.high_postings_by_rid = []

        # mapping-like of rule int id -> tokens ids sets/multisets
        self.sets_by_rid = []
        self.msets_by_rid = []

        # mapping of hash -> single int rid for hash match: duplicated rules are not allowed
        self.rid_by_hash = {}

        # Aho-Corasick automatons for regular rules and experimental fragments
        self.rules_automaton = match_aho.get_automaton()
        self.fragments_automaton = USE_AHO_FRAGMENTS and match_aho.get_automaton()
        self.starts_automaton = USE_RULE_STARTS and match_aho.get_automaton()
        self.unknown_automaton = match_unknown.get_automaton()

        # disjunctive sets of rule ids: regular and false positive