Method Retrieve

internal/retrieval/filekb.go:72–106

Retrieve scores chunks by token overlap with query (v1 demo).

(_ context.Context, query string, _ map[string]any)

// Retrieve scores chunks by token overlap with query (v1 demo).
func (f *FileKB) Retrieve(_ context.Context, query string, _ map[string]any) (types.RetrievalResult, error) {
	qTokens := tokenize(query)
	if len(qTokens) == 0 {
		return types.RetrievalResult{}, nil
	}
	type scored struct {
		ch types.RetrievalChunk
		s  float64
	}
	var out []scored
	for _, d := range f.docs {
		dt := tokenize(d.text)
		if len(dt) == 0 {
			continue
		}
		s := overlapScore(qTokens, dt)
		if s <= 0 {
			continue
		}
		out = append(out, scored{
			ch: types.RetrievalChunk{Text: d.text, Source: d.source, Score: s},
			s:  s,
		})
	}
	sort.Slice(out, func(i, j int) bool { return out[i].s > out[j].s })
	top := 5
	if len(out) < top {
		top = len(out)
	}
	chunks := make([]types.RetrievalChunk, 0, top)
	for i := 0; i < top; i++ {
		chunks = append(chunks, out[i].ch)
	}
	return types.RetrievalResult{Chunks: chunks}, nil
}

func tokenize(s string) []string {
	s = strings.ToLower(s)