Method tokens

src/licensedcode/models.py:2313–2342

Return a sequence of token strings for this rule text. SIDE EFFECT: Computed attributes such as "length", "relevance", "is_continuous", "minimum_coverage" and "stopword_by_pos" are recomputed as a side effect.

(self)

        return self

    def tokens(self):
        """
        Return a sequence of token strings for this rule text.

        SIDE EFFECT: Computed attributes such as "length", "relevance",
        "is_continuous",  "minimum_coverage" and "stopword_by_pos" are
        recomputed as a side effect.
        """

        text = self.text
        # We tag this rule as being a bare URL if it starts with a scheme and is
        # on one line: this is used to determine a matching approach

        if (
            text.startswith(('http://', 'https://', 'ftp://'))
            and '\n' not in text[:1000]
        ):
            self.minimum_coverage = 100

        toks, stopwords_by_pos = index_tokenizer_with_stopwords(text)
        self.length = len(toks)
        self.stopwords_by_pos = stopwords_by_pos
        self.set_relevance()

        # set required phrase spans that must be present for the rule
        # to pass through refinement
        self.required_phrase_spans = self.build_required_phrase_spans()
        self._set_continuous()

        return toks

    def _set_continuous(self):
        """