| 184 | } |
| 185 | |
| 186 | function pruneOverlappingWords(entries: Array<[string, number]>): Array<[string, number]> { |
| 187 | return entries.filter(([word, count]) => { |
| 188 | if (word.length <= 1) return false; |
| 189 | return !entries.some(([other, otherCount]) => { |
| 190 | if (other === word) return false; |
| 191 | if (other.length <= word.length) return false; |
| 192 | if (!other.includes(word)) return false; |
| 193 | // 如果短词只是更长词里的碎片，并且频次没有明显更强，就丢掉。 |
| 194 | return otherCount >= count * 0.9; |
| 195 | }); |
| 196 | }); |
| 197 | } |
| 198 | |
| 199 | function collectWords(text: string, counts: Map<string, number>): void { |
| 200 | const cleaned = text |