MCPcopy
hub / github.com/mksglu/context-mode / searchWithFallback

Method searchWithFallback

cli.bundle.mjs:440–440  ·  view source on GitHub ↗
(e,n=3,r,o,s="like",i)

Source from the content-addressed store, hash-verified

438 (SELECT COUNT(*) FROM chunks) AS chunks,
439 (SELECT COUNT(*) FROM chunks WHERE content_type = 'code') AS codeChunks
440 `),this.#N=this.#e.prepare("DELETE FROM chunks WHERE source_id IN (SELECT id FROM sources WHERE datetime(indexed_at) < datetime('now', '-' || ? || ' days'))"),this.#M=this.#e.prepare("DELETE FROM chunks_trigram WHERE source_id IN (SELECT id FROM sources WHERE datetime(indexed_at) < datetime('now', '-' || ? || ' days'))"),this.#D=this.#e.prepare("DELETE FROM sources WHERE datetime(indexed_at) < datetime('now', '-' || ? || ' days')")}setDenyChecker(e){this.#n=e}index(e){let{content:n,path:r,source:o,attribution:s}=e,i=typeof n=="string"&&n.length>0;if(!i&&!r)throw new Error("Either content or path must be provided");let a;if(i)a=n;else{let p=$v(r,"r");try{if(!Pv(p).isFile())throw new Error(`refusing to index ${r}: not a regular file`);a=Tv(p,"utf-8")}finally{Rv(p)}}let c=o??r??"untitled",u=this.#K(a),l=r??void 0,d=l?Cv("sha256").update(a).digest("hex"):void 0;return fr(()=>this.#d(u,c,a,l,d,s))}indexDirectory(e){let{path:n,source:r,attribution:o,perFileDeny:s,...i}=e,a=wv(n,i),c=0,u=0,l=0,d=0;for(let p of a.files){if(s&&s(p)){l++;continue}try{let f=r?`${r}:${p}`:p,m=this.index({path:p,source:f,attribution:o});c++,u+=m.totalChunks}catch{d++}}return{filesIndexed:c,totalChunks:u,capped:a.capped,totalSeen:a.totalSeen,denied:l,failed:d,label:r??n}}indexPlainText(e,n,r=20,o,s=Gc){if(!e||e.trim().length===0)return this.#d([],n,"",void 0,void 0,o);let i=this.#J(e,r,s);return fr(()=>this.#d(i.map(a=>({...a,hasCode:!1})),n,e,void 0,void 0,o))}indexJSON(e,n,r=Gc,o){if(!e||e.trim().length===0)return this.indexPlainText("",n,void 0,o,r);let s;try{s=JSON.parse(e)}catch{return this.indexPlainText(e,n,void 0,o,r)}let i=[];return this.#F(s,[],i,r),i.length===0?this.indexPlainText(e,n,void 0,o,r):fr(()=>this.#d(i,n,e,void 0,void 0,o))}#d(e,n,r,o,s,i){let a=e.filter(p=>p.hasCode).length,c=i?.sessionId??"",u=i?.eventId??"",d=this.#e.transaction(()=>{if(this.#u.run(n),this.#l.run(n),this.#f.run(n),e.length===0){let h=this.#o.run(n,o??null,s??null);return Number(h.lastInsertRowid)}let p=this.#s.run(n,e.length,a,o??null,s??null),f=Number(p.lastInsertRowid),m=new Date().toISOString();for(let h of e){let g=h.hasCode?"code":"prose";this.#a.run(h.title,h.content,f,g,null,c,u,m),this.#c.run(h.title,h.content,f,g,null,c,u,m)}return f})();return r&&this.#W(r),this.#j++,this.#j%t.OPTIMIZE_EVERY===0&&this.#U(),{sourceId:d,label:n,totalChunks:e.length,codeChunks:a}}#L(e){return e.map(n=>({title:n.title,content:n.content,source:n.label,rank:n.rank,contentType:n.content_type,highlighted:n.highlighted,timestamp:n.timestamp??void 0,sessionId:n.session_id??""}))}#p(e,n){return n==="exact"?e:`%${e.replace(/\\/g,"\\\\").replace(/%/g,"\\%").replace(/_/g,"\\_")}%`}search(e,n=3,r,o="AND",s,i="like"){let a=qN(e,o),c,u;return r&&s?(c=i==="exact"?this.#w:this.#k,u=[a,this.#p(r,i),s,n]):r?(c=i==="exact"?this.#y:this.#g,u=[a,this.#p(r,i),n]):s?(c=this.#v,u=[a,s,n]):(c=this.#h,u=[a,n]),fr(()=>this.#L(c.all(...u)))}searchTrigram(e,n=3,r,o="AND",s,i="like"){let a=VN(e,o);if(!a)return[];let c,u;return r&&s?(c=i==="exact"?this.#$:this.#T,u=[a,this.#p(r,i),s,n]):r?(c=i==="exact"?this.#x:this.#b,u=[a,this.#p(r,i),n]):s?(c=this.#E,u=[a,s,n]):(c=this.#_,u=[a,n]),fr(()=>this.#L(c.all(...u)))}fuzzyCorrect(e){let n=e.toLowerCase().trim();if(n.length<3)return null;if(this.#r.has(n)){let u=this.#r.get(n)??null;return this.#r.delete(n),this.#r.set(n,u),u}let r=KN(n.length),o=this.#S.all(n.length-r,n.length+r),s=null,i=r+1,a=!1;for(let{word:u}of o){if(u===n){a=!0;break}let l=WN(n,u);l<i&&(i=l,s=u)}let c=a?null:i<=r?s:null;if(this.#r.size>=t.FUZZY_CACHE_SIZE){let u=this.#r.keys().next().value;u!==void 0&&this.#r.delete(u)}return this.#r.set(n,c),c}#z(e,n,r,o,s="like"){let a=Math.max(n*2,10),c=this.search(e,a,r,"OR",o,s),u=this.searchTrigram(e,a,r,"OR",o,s),l=new Map,d=p=>`${p.source}::${p.title}`;for(let[p,f]of c.entries()){let m=d(f),h=l.get(m);h?h.score+=1/(60+p+1):l.set(m,{result:f,score:1/(60+p+1)})}for(let[p,f]of u.entries()){let m=d(f),h=l.get(m);h?h.score+=1/(60+p+1):l.set(m,{result:f,score:1/(60+p+1)})}return Array.from(l.values()).sort((p,f)=>f.score-p.score).slice(0,n).map(({result:p,score:f})=>({...p,rank:-f}))}#H(e,n){let r=n.toLowerCase().split(/\s+/).filter(i=>i.length>=2),o=r.filter(i=>!xs.has(i)),s=o.length>0?o:r;return e.map(i=>{let a=i.title.toLowerCase(),c=s.filter(f=>a.includes(f)).length,u=i.contentType==="code"?.6:.3,l=c>0?u*(c/s.length):0,d=0,p=0;if(s.length>=2){let f=i.content.toLowerCase(),m=s.map(h=>QN(f,h));if(!m.some(h=>h.length===0)){d=1/(1+tM(m)/Math.max(f.length,1));let g=eM(m,s);p=.5*Math.min(1,g/4)}}return{result:i,boost:l+d+p}}).sort((i,a)=>a.boost-i.boost||i.result.rank-a.result.rank).map(({result:i})=>i)}searchWithFallback(e,n=3,r,o,s="like",i){this.#V();let a=i?Math.max(n*8,40):n,c=this.#q(i),u=this.#z(e,a,r,o,s),l=c?u.filter(c):u;if(l.length>0)return this.#H(l.slice(0,n),e).map(g=>({...g,matchLayer:"rrf"}));let d=e.toLowerCase().trim().split(/\s+/).filter(h=>h.length>=3&&!xs.has(h)),p=d.join(" "),m=d.map(h=>this.fuzzyCorrect(h)??h).join(" ");if(m!==p){let h=this.#z(m,a,r,o,s),g=c?h.filter(c):h;if(g.length>0)return this.#H(g.slice(0,n),m).map(_=>({..._,matchLayer:"rrf-fuzzy"}))}return[]}#q(e){return e?n=>{let r=n.sessionId??"";return r===""||e.has(r)}:null}lastRefreshCount=0;#V(){this.lastRefreshCount=0;let e=this.#e.prepare("SELECT label, file_path, content_hash, indexed_at FROM sources WHERE file_path IS NOT NULL").all();for(let n of e)try{if(!fm(n.file_path)||this.#n&&this.#n(n.file_path))continue;let r=Jc(n.file_path).mtime,o=new Date(n.indexed_at+"Z");if(r<=o)continue;let s=$v(n.file_path,"r"),i;try{if(!Pv(s).isFile())continue;i=Tv(s,"utf-8")}finally{Rv(s)}if(Cv("sha256").update(i).digest("hex")===n.content_hash)continue;this.index({content:i,path:n.file_path,source:n.label}),this.lastRefreshCount++}catch{}}getSourceMeta(e){let n=this.#A.get(e);return n?{label:n.label,chunkCount:n.chunk_count,codeChunkCount:n.code_chunk_count,indexedAt:n.indexed_at,filePath:n.file_path??null,contentHash:n.content_hash??null}:null}listSources(){return this.#P.all()}getIndexState(){let e=this.#e.prepare("SELECT COALESCE(SUM(chunk_count), 0) AS total_chunks, COUNT(*) AS total_sources, MAX(indexed_at) AS last_indexed_at FROM sources").get();return{totalChunks:e.total_chunks??0,totalSources:e.total_sources??0,lastIndexedAt:e.last_indexed_at??void 0}}getChunksBySource(e){return this.#R.all(e).map(r=>({title:r.title,content:r.content,source:r.label,rank:0,contentType:r.content_type}))}getDistinctiveTerms(e,n=40){let r=this.#C.get(e);if(!r||r.chunk_count<3)return[];let o=r.chunk_count,s=2,i=Math.max(3,Math.ceil(o*.4)),a=new Map;for(let l of this.#O.iterate(e)){let d=new Set(l.content.toLowerCase().split(/[^\p{L}\p{N}_-]+/u).filter(p=>p.length>=3&&!xs.has(p)));for(let p of d)a.set(p,(a.get(p)??0)+1)}return Array.from(a.entries()).filter(([,l])=>l>=s&&l<=i).map(([l,d])=>{let p=Math.log(o/d),f=Math.min(l.length/20,.5),m=/[_]/.test(l),h=l.length>=12,g=m?1.5:h?.8:0;return{word:l,score:p+f+g}}).sort((l,d)=>d.score-l.score).slice(0,n).map(l=>l.word)}getStats(){let e=this.#I.get();return{sources:e?.sources??0,chunks:e?.chunks??0,codeChunks:e?.codeChunks??0}}cleanupStaleSources(e){return this.#e.transaction(o=>(this.#N.run(o),this.#M.run(o),this.#D.run(o)))(e).changes}getDBSizeBytes(){try{return Jc(this.#t).size}catch{return 0}}#U(){try{this.#e.exec("INSERT INTO chunks(chunks) VALUES('optimize')"),this.#e.exec("INSERT INTO chunks_trigram(chunks_trigram) VALUES('optimize')")}catch{}}close(){this.#U(),os(this.#e)}#W(e){let n=e.toLowerCase().split(/[^\p{L}\p{N}_-]+/u).filter(s=>s.length>=3&&!xs.has(s)),r=[...new Set(n)],o=0;this.#e.transaction(()=>{for(let s of r){let i=this.#i.run(s);o+=i.changes}})(),o>0&&this.#r.clear()}#K(e,n=Gc){let r=[],o=e.split(`
441`),s=[],i=[],a="",c=()=>{let l=i.join(`
442`).trim();if(l.length===0)return;let d=this.#ee(s,a),p=i.some(y=>/^`{3,}/.test(y));if(Buffer.byteLength(l)<=n){r.push({title:d,content:l,hasCode:p}),i=[];return}let f=l.split(/\n\n+/),m=[],h=1,g=()=>{if(m.length===0)return;let y=m.join(`
443

Callers 12

cRFunction · 0.45
nCFunction · 0.45
fiFunction · 0.45
cli.bundle.mjsFile · 0.45
d9Function · 0.45
store.test.tsFile · 0.45
search.test.tsFile · 0.45
topOneFunction · 0.45
rankingFunction · 0.45
server.test.tsFile · 0.45

Calls 9

#VMethod · 0.95
#qMethod · 0.95
#zMethod · 0.95
#HMethod · 0.95
fuzzyCorrectMethod · 0.95
maxMethod · 0.45
trimMethod · 0.45
toLowerCaseMethod · 0.45
hasMethod · 0.45

Tested by 2

topOneFunction · 0.36
rankingFunction · 0.36