Function tokenizeKnowledgeText

deployments/desktop/static/desktop.js:3332–3339

(text)

  }

  function tokenizeKnowledgeText(text) {
    return String(text || "")
      .toLowerCase()
      .replace(/[^a-z0-9\u4e00-\u9fff-]+/g, " ")
      .split(/\s+/)
      .map((token) => token.trim())
      .filter((token) => token.length > 2 && !keywordStopWords.has(token) && !/^\d+$/.test(token));
  }

  function aliasLabelsForText(text) {
    return phraseAliases