Return a tuple of (tids set, multiset) given a `token_ids` tids sequence.
(token_ids, _use_bigrams=False)
| 226 | |
| 227 | |
| 228 | def build_set_and_mset(token_ids, _use_bigrams=False): |
| 229 | """ |
| 230 | Return a tuple of (tids set, multiset) given a `token_ids` tids |
| 231 | sequence. |
| 232 | """ |
| 233 | if _use_bigrams: |
| 234 | return build_set_and_bigrams_mset(token_ids) |
| 235 | else: |
| 236 | return build_set_and_tids_mset(token_ids) |
| 237 | |
| 238 | # FIXME: we should consider more aggressively the thresholds and what a match |
| 239 | # filters would discard when we compute candidates to eventually discard many or |
no test coverage detected