Return an iterable of tokens and non-tokens punctuation from a unicode query text keeping everything (including punctuations, line endings, etc.) The returned iterable contains 2-tuples of: - True if the string is a text token or False if this is not (such as punctuation, spac
(text)
| 347 | |
| 348 | |
| 349 | def matched_query_text_tokenizer(text): |
| 350 | """ |
| 351 | Return an iterable of tokens and non-tokens punctuation from a unicode query |
| 352 | text keeping everything (including punctuations, line endings, etc.) |
| 353 | The returned iterable contains 2-tuples of: |
| 354 | - True if the string is a text token or False if this is not |
| 355 | (such as punctuation, spaces, etc). |
| 356 | - the corresponding string. |
| 357 | This is used to reconstruct the matched query text for reporting. |
| 358 | """ |
| 359 | if not text: |
| 360 | return |
| 361 | for match in tokens_and_non_tokens(text): |
| 362 | if match: |
| 363 | mgd = match.groupdict() |
| 364 | token = mgd.get('token') |
| 365 | punct = mgd.get('punct') |
| 366 | if token: |
| 367 | yield True, token |
| 368 | elif punct: |
| 369 | yield False, punct |
| 370 | else: |
| 371 | # this should never happen |
| 372 | raise Exception('Internal error in matched_query_text_tokenizer') |
| 373 | |
| 374 | |
| 375 | def ngrams(iterable, ngram_length): |