| 220 | * 用于消除对同一批文本重复分词的开销。 |
| 221 | */ |
| 222 | export function batchSegmentChineseWithStats( |
| 223 | texts: string[], |
| 224 | locale: SupportedLocale, |
| 225 | options: BatchSegmentOptions = {} |
| 226 | ): BatchSegmentResult & { posTagStats: Map<string, number> } { |
| 227 | const { |
| 228 | minLength, |
| 229 | minCount = 2, |
| 230 | topN = 100, |
| 231 | posFilterMode = 'meaningful', |
| 232 | customPosTags, |
| 233 | enableStopwords = true, |
| 234 | dictType = 'default', |
| 235 | excludeWords, |
| 236 | } = options |
| 237 | |
| 238 | const effectiveMinLength = minLength ?? 2 |
| 239 | const allowedTags = posFilterMode === 'custom' && customPosTags ? new Set(customPosTags) : MEANINGFUL_POS_TAGS |
| 240 | const excludeSet = excludeWords?.length ? new Set(excludeWords.map((w) => w.toLowerCase())) : null |
| 241 | |
| 242 | const wordFrequency = new Map<string, number>() |
| 243 | const posTagStats = new Map<string, number>() |
| 244 | |
| 245 | try { |
| 246 | const jieba = getJieba(dictType) |
| 247 | for (const text of texts) { |
| 248 | const cleaned = cleanText(text) |
| 249 | if (!cleaned) continue |
| 250 | for (const { tag, word } of jieba.tag(cleaned)) { |
| 251 | if (!isValidWord(word, locale, effectiveMinLength, enableStopwords, isStopword)) continue |
| 252 | // 词性统计覆盖全部有效词（与 collectPosTagStats 一致） |
| 253 | posTagStats.set(tag, (posTagStats.get(tag) || 0) + 1) |
| 254 | // 词频仅统计命中允许词性的词（与 batchSegmentWithFrequency 一致） |
| 255 | if (!allowedTags.has(tag)) continue |
| 256 | if (excludeSet && excludeSet.has(word.toLowerCase())) continue |
| 257 | wordFrequency.set(word, (wordFrequency.get(word) || 0) + 1) |
| 258 | } |
| 259 | } |
| 260 | } catch (error) { |
| 261 | console.error('[NLP] Chinese single-pass segmentation failed:', error) |
| 262 | } |
| 263 | |
| 264 | const filtered = new Map<string, number>() |
| 265 | let totalWords = 0 |
| 266 | for (const [word, count] of wordFrequency) { |
| 267 | if (count >= minCount) { |
| 268 | filtered.set(word, count) |
| 269 | totalWords += count |
| 270 | } |
| 271 | } |
| 272 | |
| 273 | const sorted = [...filtered.entries()].sort((a, b) => b[1] - a[1]).slice(0, topN) |
| 274 | return { words: new Map(sorted), uniqueWords: filtered.size, totalWords, posTagStats } |
| 275 | } |
| 276 | |
| 277 | /** |
| 278 | * 收集文本的词性统计 |