Method tokens_by_line

src/licensedcode/query.py:352–525

Yield multiple sequences of tokens, one for each line in this query. Line numbers start at ``start_line`` which is 1-based by default. SIDE EFFECT: This populates the query `line_by_pos`, `unknowns_by_pos`, `unknowns_span`, `stopwords_by_pos`, `shorts_and_digits_pos

(
        self,
        location=None,
        query_string=None,
        start_line=1,
    )

        return all_pos

    def tokens_by_line(
        self,
        location=None,
        query_string=None,
        start_line=1,
    ):
        """
        Yield multiple sequences of tokens, one for each line in this query.
        Line numbers start at ``start_line`` which is 1-based by default.

        SIDE EFFECT: This populates the query `line_by_pos`, `unknowns_by_pos`,
        `unknowns_span`, `stopwords_by_pos`, `shorts_and_digits_pos` and `spdx_lines` .
        """
        from licensedcode.match_spdx_lid import split_spdx_lid
        from licensedcode.stopwords import STOPWORDS

        location = location or self.location
        query_string = query_string or self.query_string

        # bind frequently called functions to local scope
        line_by_pos_append = self.line_by_pos.append

        # we use a defaultdict as a convenience at construction time
        unknowns_by_pos = defaultdict(int)
        unknowns_pos = set()
        unknowns_pos_add = unknowns_pos.add

        # we use a defaultdict as a convenience at construction time
        stopwords_by_pos = defaultdict(int)
        stopwords_pos = set()
        stopwords_pos_add = stopwords_pos.add

        self_shorts_and_digits_pos_add = self.shorts_and_digits_pos.add
        dic_get = self.idx.dictionary.get

        # note: positions start at zero
        # absolute position in a query, including only known tokens
        known_pos = -1

        # flag ifset to True when we have found the first known token globally
        # across all query lines
        started = False

        spdx_lid_token_ids = self.spdx_lid_token_ids

        qlines = query_lines(
            location=location,
            query_string=query_string,
            start_line=start_line,
        )
        if TRACE or TRACE_STOP_AND_UNKNOWN:
            logger_debug('tokens_by_line: query lines:')
            qlines = list(qlines)
            for line_num, line in qlines:
                logger_debug(' ', line_num, ':', line)

        for line_num, line in qlines:
            if TRACE_STOP_AND_UNKNOWN: