(text: string, locale: SupportedLocale, options: SegmentOptions = {})
| 146 | * 通用分词入口 |
| 147 | */ |
| 148 | export function segment(text: string, locale: SupportedLocale, options: SegmentOptions = {}): string[] { |
| 149 | const { |
| 150 | minLength, |
| 151 | posFilterMode = 'meaningful', |
| 152 | customPosTags, |
| 153 | enableStopwords = true, |
| 154 | dictType = 'default', |
| 155 | } = options |
| 156 | const isChinese = locale.startsWith('zh') |
| 157 | const isJapanese = locale === 'ja-JP' |
| 158 | const defaultMinLength = isChinese || isJapanese ? 2 : 3 |
| 159 | const effectiveMinLength = minLength ?? defaultMinLength |
| 160 | |
| 161 | let words: string[] |
| 162 | if (isChinese) { |
| 163 | words = segmentChinese(text, { posFilterMode, customPosTags, dictType }) |
| 164 | } else if (isJapanese) { |
| 165 | words = segmentJapanese(text) |
| 166 | } else { |
| 167 | words = segmentEnglish(text) |
| 168 | } |
| 169 | |
| 170 | return words.filter((word) => isValidWord(word, locale, effectiveMinLength, enableStopwords, isStopword)) |
| 171 | } |
| 172 | |
| 173 | /** |
| 174 | * 批量分词并统计词频 |
no test coverage detected