Function collectWords

cy/cy.ts:199–226

(text: string, counts: Map<string, number>)

}

function collectWords(text: string, counts: Map<string, number>): void {
  const cleaned = text
    .replace(/https?:\/\/\S+/gi, " ")
    .replace(/[@#][\w_一-龥-]+/g, " ")
    .replace(/[^\p{Script=Han}a-zA-Z0-9_+\-.]+/gu, " ");

  for (const match of cleaned.matchAll(/[a-zA-Z][a-zA-Z0-9_+\-.]{1,24}/g)) {
    addWord(counts, match[0], 2);
  }
  for (const match of cleaned.matchAll(/\d{2,}[a-zA-Z%]?/g)) {
    addWord(counts, match[0], 1);
  }

  const hanParts = cleaned.match(/[\p{Script=Han}]{2,}/gu) || [];
  for (const part of hanParts) {
    if (part.length <= 4) {
      addWord(counts, part, 3);
      continue;
    }
    for (let size = 2; size <= 5; size++) {
      for (let i = 0; i <= part.length - size; i++) {
        const word = part.slice(i, i + size);
        const edgeBonus = i === 0 || i === part.length - size ? 1 : 0;
        addWord(counts, word, size <= 3 ? 1 + edgeBonus : 2 + edgeBonus);
      }
    }
  }
}

function buildWordItems(counts: Map<string, number>): WordItem[] {
  const entries = pruneOverlappingWords([...counts.entries()])