Method tokenize_and_build_runs

src/licensedcode/query.py:527–545

Tokenize this query and populate tokens and query_runs at each break point. Only keep known token ids but consider unknown token ids to break a query in runs. `tokens_by_line` is the output of the self.tokens_by_line() method and is an iterator of lines (eg.

(self, tokens_by_line, line_threshold=4)

            logger_debug(f'  self.stopwords_by_pos: {self.stopwords_by_pos}')

    def tokenize_and_build_runs(self, tokens_by_line, line_threshold=4):
        """
        Tokenize this query and populate tokens and query_runs at each break
        point. Only keep known token ids but consider unknown token ids to break
        a query in runs.

        `tokens_by_line` is the output of the self.tokens_by_line() method and
        is an iterator of lines (eg. lists) of token ids.

        `line_threshold` is the number of empty or junk lines to break a new run.
        """
        self._tokenize_and_build_runs(tokens_by_line, line_threshold)

        if TRACE_QR:
            print()
            logger_debug('Initial Query runs for query:', self.location)
            for qr in self.query_runs:
                print(' ' , repr(qr))
            print()

    def refine_runs(self):
        # TODO: move me to the approximate matching loop so that this is done