Return a LicenseMatch (or None) by matching the ``query_run`` against the ``automaton`` and ``idx`` index.
(
idx,
query_run,
automaton,
unknown_ngram_length=UNKNOWN_NGRAM_LENGTH,
**kwargs,
)
| 130 | |
| 131 | |
| 132 | def match_unknowns( |
| 133 | idx, |
| 134 | query_run, |
| 135 | automaton, |
| 136 | unknown_ngram_length=UNKNOWN_NGRAM_LENGTH, |
| 137 | **kwargs, |
| 138 | ): |
| 139 | """ |
| 140 | Return a LicenseMatch (or None) by matching the ``query_run`` against the |
| 141 | ``automaton`` and ``idx`` index. |
| 142 | """ |
| 143 | matched_ngrams = get_matched_ngrams( |
| 144 | tokens=query_run.tokens, |
| 145 | qbegin=query_run.start, |
| 146 | automaton=automaton, |
| 147 | unknown_ngram_length=unknown_ngram_length, |
| 148 | ) |
| 149 | |
| 150 | # build match from merged matched ngrams |
| 151 | qspans = (Span(qstart, qend) for qstart, qend in matched_ngrams) |
| 152 | qspan = Span().union(*qspans) |
| 153 | |
| 154 | if TRACE: |
| 155 | query_tokens = query_run.query.tokens |
| 156 | tokens_by_tid = idx.tokens_by_tid |
| 157 | |
| 158 | def get_tokens(_toks): |
| 159 | return (' '.join(tokens_by_tid[t] for t in _toks)) |
| 160 | |
| 161 | print('match_unknowns: matched_ngrams') |
| 162 | |
| 163 | for qstart, qend in matched_ngrams: |
| 164 | _span = Span(qstart, qend) |
| 165 | _tokens = [query_tokens[qpos] for qpos in _span] |
| 166 | print( |
| 167 | ' ', 'qstart', qstart, |
| 168 | 'qend', qend, |
| 169 | 'matched_toks', get_tokens(_tokens)) |
| 170 | |
| 171 | if not qspan: |
| 172 | return |
| 173 | |
| 174 | query = query_run.query |
| 175 | query_tokens = query.tokens |
| 176 | |
| 177 | matched_tokens = [query_tokens[qpos] for qpos in qspan] |
| 178 | match_len = len(qspan) |
| 179 | |
| 180 | if TRACE: |
| 181 | # print('match_unknowns: matched_span:', get_tokens(matched_tokens)) |
| 182 | print('match_unknowns: qspan, match_len, matched_span:', qspan, match_len, matched_tokens) |
| 183 | |
| 184 | # we use the query side to build the ispans |
| 185 | ispan = Span(0, match_len) |
| 186 | |
| 187 | # build synthetic rule text for "only_matched" text |
| 188 | line_by_pos = query.line_by_pos |
| 189 |