Method _add_rules

src/licensedcode/index.py:270–577

Add a list of Rule objects to the index and constructs optimized and immutable index structures. ``_legalese`` is a sorted mapping of common license-specific words aka. legalese as {token: id} ``_spdx_tokens`` is a set of token strings used in SPDX license identifie

(
        self,
        rules,
        _legalese=common_license_words,
        _spdx_tokens=frozenset(),
        _license_tokens=frozenset(),
    )

                self._print_index_stats()

    def _add_rules(
        self,
        rules,
        _legalese=common_license_words,
        _spdx_tokens=frozenset(),
        _license_tokens=frozenset(),
    ):
        """
        Add a list of Rule objects to the index and constructs optimized and
        immutable index structures.

        ``_legalese`` is a sorted mapping of common license-specific words aka. legalese as {token: id}
        ``_spdx_tokens`` is a set of token strings used in SPDX license identifiers
        ``_license_tokens`` is a set of "license" tokens used as start or end of a rule
        """
        if self.optimized:
            raise Exception('Index has been optimized and cannot be updated.')

        # initial dictionary mapping for known legalese tokens
        ########################################################################

        # TODO: we should start enumerating at 1 below: token ids then become
        # valid "unichr" values, making it easier downstream when used in
        # automatons

        self.dictionary = dictionary = dict(_legalese)
        dictionary_get = dictionary.get

        self.len_legalese = len_legalese = len(set(dictionary.values()))
        highest_tid = len_legalese -1

        # Add SPDX key tokens to the dictionary: these are always treated as
        # non-legalese. This may seem weird but they are detected in expressions
        # alright and some of their tokens exist as rules too (e.g. GPL).
        # Treating their words as legalese by default creates problems as common
        # words such as mit may become legalese words even though we do not want
        # this to happen.
        ########################################################################
        for sts in sorted(_spdx_tokens):
            stid = dictionary_get(sts)
            if stid is None:
                # we have a never yet seen token, so we assign a new tokenid
                highest_tid += 1
                stid = highest_tid
                dictionary[sts] = stid

        self.rules_by_rid = rules_by_rid = list(rules)
        self.rules_by_id = {r.identifier: r for r in self.rules_by_rid}
        if TRACE_INDEXING:
            for _rid, _rule in enumerate(rules_by_rid):
                logger_debug('rules_by_rid:', _rid, _rule)

        # ensure that rules are sorted
        rules_by_rid.sort()
        len_rules = len(rules_by_rid)

        # create index data structures
        # OPTIMIZATION: bind frequently used methods to the local scope for