自适应中英文词边界检测
(text, start, end)
| 57 | |
| 58 | |
| 59 | def is_word_boundary(text, start, end): |
| 60 | """自适应中英文词边界检测""" |
| 61 | # 判断文本是否包含中文（包括扩展CJK字符） |
| 62 | has_chinese = re.search(r"[\u4e00-\u9fff\u3400-\u4dbf\U00020000-\U0002a6df]", text) |
| 63 | |
| 64 | if has_chinese: |
| 65 | # 中文模式：使用jieba分词检测词边界 |
| 66 | words = list(jieba.cut(text)) |
| 67 | current_pos = 0 |
| 68 | boundaries = set() |
| 69 | |
| 70 | # 构建词边界集合 |
| 71 | for word in words: |
| 72 | boundaries.add(current_pos) # 词开始位置 |
| 73 | boundaries.add(current_pos + len(word)) # 词结束位置 |
| 74 | current_pos += len(word) |
| 75 | |
| 76 | # 检查输入位置是否在分词边界上 |
| 77 | return start in boundaries or end in boundaries |
| 78 | else: |
| 79 | # 英文模式：使用正则表达式检测单词边界 |
| 80 | word_chars = r"\w" # 仅字母、数字、下划线 |
| 81 | |
| 82 | # 前字符检查 |
| 83 | prev_is_word = False |
| 84 | if start > 0: |
| 85 | prev_char = text[start - 1] |
| 86 | prev_is_word = re.match(f"[{word_chars}]", prev_char, re.UNICODE) |
| 87 | |
| 88 | # 后字符检查 |
| 89 | next_is_word = False |
| 90 | if end < len(text): |
| 91 | next_char = text[end] |
| 92 | next_is_word = re.match(f"[{word_chars}]", next_char, re.UNICODE) |
| 93 | |
| 94 | return not prev_is_word and not next_is_word |
| 95 | |
| 96 | |
| 97 | def read_jsonl(file_path): |
nothing calls this directly
no outgoing calls
no test coverage detected