Function reportable_tokens

src/licensedcode/match.py:3051–3167

Yield Tokens from a ``tokens`` iterable of Token objects (built from a query- side scanned file or string) that are inside a ``match_qspan`` matched Span starting at `start_line` and ending at ``end_line``. If whole_lines is True, also yield unmatched Tokens that are before and afte

(
    tokens,
    match_qspan,
    start_line,
    end_line,
    whole_lines=False,
    trace=TRACE_MATCHED_TEXT_DETAILS,
)

def reportable_tokens(
    tokens,
    match_qspan,
    start_line,
    end_line,
    whole_lines=False,
    trace=TRACE_MATCHED_TEXT_DETAILS,
):
    """
    Yield Tokens from a ``tokens`` iterable of Token objects (built from a query-
    side scanned file or string) that are inside a ``match_qspan`` matched Span
    starting at `start_line` and ending at ``end_line``. If whole_lines is True,
    also yield unmatched Tokens that are before and after the match and on the
    first and last line of a match (unless the lines are very long text lines or
    the match is from binary content.)

    As a side effect, known matched tokens are tagged as "is_matched=True" if
    they are matched.

    If ``whole_lines`` is True, any token within matched lines range is
    included. Otherwise, a token is included if its position is within the
    matched ``match_qspan`` or it is a punctuation token immediately after the
    matched ``match_qspan`` even though not matched.
    """
    start = match_qspan.start
    end = match_qspan.end

    started = False
    finished = False

    end_pos = 0
    last_pos = 0
    for real_pos, tok in enumerate(tokens):
        if trace:
            logger_debug('reportable_tokens: processing', real_pos, tok)

        # ignore tokens outside the matched lines range
        if tok.line_num < start_line:
            if trace:
                logger_debug('  tok.line_num < start_line:', tok.line_num, '<', start_line)

            continue

        if tok.line_num > end_line:
            if trace:
                logger_debug('  tok.line_num > end_line', tok.line_num, '>', end_line)

            break

        if trace:
            logger_debug('reportable_tokens:', real_pos, tok)

        is_included = False

        # tagged known matched tokens (useful for highlighting)
        if tok.pos != -1 and tok.is_known and tok.pos in match_qspan:
            tok = attr.evolve(tok, is_matched=True)
            is_included = True