Yield Token objects with pos and line number collected from the file at `location` or the `query_string` string. `dictionary` is the index mapping of tokens to token ids.
(
location,
query_string,
dictionary,
start_line=1,
trace=TRACE_MATCHED_TEXT_DETAILS,
)
| 2947 | |
| 2948 | |
| 2949 | def _tokenize_matched_text( |
| 2950 | location, |
| 2951 | query_string, |
| 2952 | dictionary, |
| 2953 | start_line=1, |
| 2954 | trace=TRACE_MATCHED_TEXT_DETAILS, |
| 2955 | ): |
| 2956 | """ |
| 2957 | Yield Token objects with pos and line number collected from the file at |
| 2958 | `location` or the `query_string` string. `dictionary` is the index mapping |
| 2959 | of tokens to token ids. |
| 2960 | """ |
| 2961 | pos = 0 |
| 2962 | qls = query.query_lines( |
| 2963 | location=location, |
| 2964 | query_string=query_string, |
| 2965 | strip=False, |
| 2966 | start_line=start_line, |
| 2967 | ) |
| 2968 | for line_num, line in qls: |
| 2969 | if trace: |
| 2970 | logger_debug(' _tokenize_matched_text:', |
| 2971 | 'line_num:', line_num, |
| 2972 | 'line:', line) |
| 2973 | |
| 2974 | for is_text, token_str in matched_query_text_tokenizer(line): |
| 2975 | if trace: |
| 2976 | logger_debug(' is_text:', is_text, 'token_str:', repr(token_str)) |
| 2977 | |
| 2978 | # Determine if a token is is_known in the license index or not. This |
| 2979 | # is essential as we need to realign the query-time tokenization |
| 2980 | # with the full text to report proper matches. |
| 2981 | if is_text and token_str and token_str.strip(): |
| 2982 | |
| 2983 | # we retokenize using the query tokenizer: |
| 2984 | # 1. to lookup for is_known tokens in the index dictionary |
| 2985 | |
| 2986 | # 2. to ensure the number of tokens is the same in both |
| 2987 | # tokenizers (though, of course, the case will differ as the |
| 2988 | # regular query tokenizer ignores case and punctuations). |
| 2989 | |
| 2990 | # NOTE: we have a rare Unicode bug/issue because of some Unicode |
| 2991 | # codepoint such as some Turkish characters that decompose to |
| 2992 | # char + punct when casefolded. This should be fixed in Unicode |
| 2993 | # release 14 and up and likely implemented in Python 3.10 and up |
| 2994 | # See https://github.com/nexB/scancode-toolkit/issues/1872 |
| 2995 | # See also: https://bugs.python.org/issue34723#msg359514 |
| 2996 | qtokenized = list(index_tokenizer(token_str)) |
| 2997 | if not qtokenized: |
| 2998 | |
| 2999 | yield Token( |
| 3000 | value=token_str, |
| 3001 | line_num=line_num, |
| 3002 | is_text=is_text, |
| 3003 | is_known=False, |
| 3004 | pos=-1, |
| 3005 | ) |
| 3006 |
no test coverage detected