MCPcopy Index your code
hub / github.com/ChatLab/ChatLab / tokenizeQueryForFts

Function tokenizeQueryForFts

packages/node-runtime/src/nlp/fts-tokenizer.ts:46–75  ·  view source on GitHub ↗
(keywords: string[])

Source from the content-addressed store, hash-verified

44 * - Multiple keywords: groups joined with OR (any match)
45 */
46export function tokenizeQueryForFts(keywords: string[]): string {
47 if (keywords.length === 0) return ''
48
49 const groups = keywords
50 .map((kw) => {
51 const trimmed = kw.trim()
52 if (!trimmed) return ''
53
54 try {
55 const jieba = getJieba()
56 const tokens = jieba
57 .cut(trimmed, false)
58 .map((t) => t.trim().toLowerCase())
59 .filter((t) => t.length > 0)
60
61 if (tokens.length === 0) return ''
62 if (tokens.length === 1) return escapeToken(tokens[0])
63 return tokens.map(escapeToken).join(' ')
64 } catch {
65 const simple = trimmed.toLowerCase().trim()
66 return simple ? escapeToken(simple) : ''
67 }
68 })
69 .filter((g) => g.length > 0)
70
71 if (groups.length === 0) return ''
72 if (groups.length === 1) return groups[0]
73
74 return groups.map((g) => (g.includes(' ') ? `(${g})` : g)).join(' OR ')
75}

Callers 3

searchByFtsFunction · 0.90
searchMessagesFunction · 0.90

Calls 2

getJiebaFunction · 0.90
escapeTokenFunction · 0.85

Tested by

no test coverage detected