Compare a `qset` query set or multiset with a `iset` index rule set or multiset. Return a tuple of (ScoresVector tuple, intersection) from comparing the sets. The ScoresVector is designed to be used as a rank sorting key to rank multiple set intersections. Return (None, None) if the
(qset, iset,
intersector, counter, high_intersection_filter,
len_legalese, unique,
rule,
filter_non_matching=True,
high_resemblance_threshold=0.8)
| 368 | |
| 369 | |
| 370 | def compare_token_sets(qset, iset, |
| 371 | intersector, counter, high_intersection_filter, |
| 372 | len_legalese, unique, |
| 373 | rule, |
| 374 | filter_non_matching=True, |
| 375 | high_resemblance_threshold=0.8): |
| 376 | """ |
| 377 | Compare a `qset` query set or multiset with a `iset` index rule set or |
| 378 | multiset. Return a tuple of (ScoresVector tuple, intersection) from |
| 379 | comparing the sets. The ScoresVector is designed to be used as a rank |
| 380 | sorting key to rank multiple set intersections. Return (None, None) if there |
| 381 | is no relevant intersection between sets. |
| 382 | """ |
| 383 | intersection = intersector(qset, iset) |
| 384 | if not intersection: |
| 385 | return None, None |
| 386 | |
| 387 | high_intersection = high_intersection_filter(intersection, len_legalese) |
| 388 | |
| 389 | if filter_non_matching: |
| 390 | if not high_intersection: |
| 391 | return None, None |
| 392 | |
| 393 | high_matched_length = counter(high_intersection) |
| 394 | min_high_matched_length = rule.get_min_high_matched_length(unique) |
| 395 | |
| 396 | # need some high match above min high |
| 397 | if high_matched_length < min_high_matched_length: |
| 398 | return None, None |
| 399 | |
| 400 | matched_length = counter(intersection) |
| 401 | min_matched_length = rule.get_min_matched_length(unique) |
| 402 | |
| 403 | if filter_non_matching and matched_length < min_matched_length: |
| 404 | return None, None |
| 405 | |
| 406 | # Compute resemblance and containment: note we are interested in the index- |
| 407 | # side containment of a rule in the query and not how much of a query is |
| 408 | # contained in a rule. In practice we have three main cases: |
| 409 | # 1. A smaller notice contained in a larger code file or doc file |
| 410 | # 2. A single whole license text |
| 411 | # 3. A file that contains mostly licenses notices and texts and contain several of these |
| 412 | # Containment captures best case 1. |
| 413 | # Resemblance captures best case 2. |
| 414 | # Containment first and resemblance second also helps with case 3. which is |
| 415 | # mixed and gives the best rankings in practice, as we want to further |
| 416 | # process first rules that are highly contained in the query. |
| 417 | iset_len = rule.get_length(unique) |
| 418 | qset_len = counter(qset) |
| 419 | union_len = qset_len + iset_len - matched_length |
| 420 | resemblance = matched_length / union_len |
| 421 | containment = matched_length / iset_len |
| 422 | # by squaring the resemblance that is otherwise between 0 and 1, we make |
| 423 | # higher resemblance more important and lower ones less so. This is |
| 424 | # capturing that resemblance matters when high (e.g. the sets are highly |
| 425 | # similar) and that otherwise containment matters most. This is could be |
| 426 | # seen as a form of "smoothing" |
| 427 | amplified_resemblance = resemblance ** 2 |
no test coverage detected