| 118 | } |
| 119 | |
| 120 | function segmentEnglish(text: string): string[] { |
| 121 | const cleaned = cleanText(text) |
| 122 | if (!cleaned) return [] |
| 123 | try { |
| 124 | const segmenter = new Intl.Segmenter('en', { granularity: 'word' }) |
| 125 | return [...segmenter.segment(cleaned)].filter((s) => s.isWordLike).map((s) => s.segment.toLowerCase()) |
| 126 | } catch { |
| 127 | return cleaned |
| 128 | .toLowerCase() |
| 129 | .split(/\s+/) |
| 130 | .filter((w) => w.length > 0) |
| 131 | } |
| 132 | } |
| 133 | |
| 134 | function segmentJapanese(text: string): string[] { |
| 135 | const cleaned = cleanText(text) |