Function compute_candidates

src/licensedcode/match_set.py:244–367

Return a ranked list of rule candidates for further matching give a `query_run`. Use approximate matching based on token sets ignoring positions. Only consider rules that have an rid in a `matchable_rids` rids set if provided. The ranking is based on a combo of resemblance, con

(query_run, idx, matchable_rids, top=50,
                       high_resemblance=False, high_resemblance_threshold=0.8,
                       _use_bigrams=False)

def compute_candidates(query_run, idx, matchable_rids, top=50,
                       high_resemblance=False, high_resemblance_threshold=0.8,
                       _use_bigrams=False):
    """
    Return a ranked list of rule candidates for further matching give a
    `query_run`. Use approximate matching based on token sets ignoring
    positions. Only consider rules that have an rid in a `matchable_rids` rids
    set if provided.

    The ranking is based on a combo of resemblance, containment, length and
    other measures.

    if `high_resemblance` is True, this return only candidates that have a a
    high resemblance above `high_resemblance_threshold`.
    """
    # collect query-side sets used for matching
    token_ids = query_run.matchable_tokens()
    qset, qmset = build_set_and_mset(token_ids, _use_bigrams=_use_bigrams)

    len_legalese = idx.len_legalese

    # perform two steps of ranking:
    # step one with tid sets and step two with tid multisets for refinement

    ############################################################################
    # step 1 is on token id sets:
    ############################################################################

    sortable_candidates = []
    sortable_candidates_append = sortable_candidates.append

    sets_by_rid = idx.sets_by_rid

    for rid, rule in enumerate(idx.rules_by_rid):
        if rid not in matchable_rids:
            continue

        scores_vectors, high_set_intersection = compare_token_sets(
            qset=qset,
            iset=sets_by_rid[rid],
            intersector=tids_sets_intersector,
            counter=tids_set_counter,
            high_intersection_filter=high_tids_set_subset,
            len_legalese=len_legalese,
            unique=True,
            rule=rule,
            filter_non_matching=True,
            high_resemblance_threshold=high_resemblance_threshold)

        if scores_vectors:
            svr, svf = scores_vectors
            if (not high_resemblance
            or (high_resemblance and svr.is_highly_resemblant and svf.is_highly_resemblant)):
                sortable_candidates_append((scores_vectors, rid, rule, high_set_intersection))

    if not sortable_candidates:
        return sortable_candidates