Function break_long_lines

src/licensedcode/query.py:710–717

Given an iterable of lines (each being a list of token ids), break lines that contain more than threshold in chunks. Return an iterable of lines.

(lines, threshold=MAX_TOKEN_PER_LINE)

def break_long_lines(lines, threshold=MAX_TOKEN_PER_LINE):
    """
    Given an iterable of lines (each being a list of token ids), break lines
    that contain more than threshold in chunks. Return an iterable of lines.
    """
    for line in lines:
        for i in range(0, len(line), threshold):
            yield line[i:i + threshold]


class QueryRun(object):