Function batchSegmentWithFrequency

packages/node-runtime/src/nlp/segmenter.ts:176–213

(
  texts: string[],
  locale: SupportedLocale,
  options: BatchSegmentOptions = {}
)

 * 批量分词并统计词频
 */
export function batchSegmentWithFrequency(
  texts: string[],
  locale: SupportedLocale,
  options: BatchSegmentOptions = {}
): BatchSegmentResult {
  const {
    minLength,
    minCount = 2,
    topN = 100,
    posFilterMode,
    customPosTags,
    enableStopwords,
    dictType,
    excludeWords,
  } = options
  const wordFrequency = new Map<string, number>()
  const excludeSet = excludeWords?.length ? new Set(excludeWords.map((w) => w.toLowerCase())) : null

  for (const text of texts) {
    const words = segment(text, locale, { minLength, posFilterMode, customPosTags, enableStopwords, dictType })
    for (const word of words) {
      if (excludeSet && excludeSet.has(word.toLowerCase())) continue
      wordFrequency.set(word, (wordFrequency.get(word) || 0) + 1)
    }
  }

  const filtered = new Map<string, number>()
  let totalWords = 0
  for (const [word, count] of wordFrequency) {
    if (count >= minCount) {
      filtered.set(word, count)
      totalWords += count
    }
  }

  const sorted = [...filtered.entries()].sort((a, b) => b[1] - a[1]).slice(0, topN)
  return { words: new Map(sorted), uniqueWords: filtered.size, totalWords }
}

/**
 * 中文单遍分词：一次 jieba.tag 同时产出词频与词性统计。