| 94 | } |
| 95 | |
| 96 | function segmentChinese( |
| 97 | text: string, |
| 98 | options: { posFilterMode?: PosFilterMode; customPosTags?: string[]; dictType?: DictType } = {} |
| 99 | ): string[] { |
| 100 | const { posFilterMode = 'meaningful', customPosTags, dictType = 'default' } = options |
| 101 | const cleaned = cleanText(text) |
| 102 | if (!cleaned) return [] |
| 103 | |
| 104 | try { |
| 105 | const jieba = getJieba(dictType) |
| 106 | if (posFilterMode === 'all') return jieba.cut(cleaned, false) |
| 107 | const tagged = jieba.tag(cleaned) |
| 108 | const allowedTags = posFilterMode === 'custom' && customPosTags ? new Set(customPosTags) : MEANINGFUL_POS_TAGS |
| 109 | return tagged.filter((item) => allowedTags.has(item.tag)).map((item) => item.word) |
| 110 | } catch (error) { |
| 111 | console.error('[NLP] Chinese segmentation failed:', error) |
| 112 | try { |
| 113 | return getJieba('default').cut(cleanText(text), false) |
| 114 | } catch { |
| 115 | return cleaned.split('') |
| 116 | } |
| 117 | } |
| 118 | } |
| 119 | |
| 120 | function segmentEnglish(text: string): string[] { |
| 121 | const cleaned = cleanText(text) |