ingestKnowledgeJSONL parses a docs-flatten JSONL file into per-chunk FileInfo entries. Each chunk becomes one virtual file whose Path is the chunk id ("source#nnnn"), preserving the corpus structure for segmentation, retrieval and the digest. Returns the files plus provenance metadata to be merged i
(path string, logger *zap.Logger)
| 68 | // must not be lost to one truncated write. An error is returned only when the |
| 69 | // file cannot be read at all or yields no usable chunk. |
| 70 | func ingestKnowledgeJSONL(path string, logger *zap.Logger) ([]utils.FileInfo, map[string]string, error) { |
| 71 | if logger == nil { |
| 72 | logger = zap.NewNop() |
| 73 | } |
| 74 | f, err := os.Open(path) // #nosec G304 -- user-supplied corpus path, same trust as /context create |
| 75 | if err != nil { |
| 76 | return nil, nil, fmt.Errorf("knowledge: open corpus: %w", err) |
| 77 | } |
| 78 | defer func() { _ = f.Close() }() |
| 79 | |
| 80 | sc := bufio.NewScanner(f) |
| 81 | sc.Buffer(make([]byte, 64*1024), maxKnowledgeLineBytes) |
| 82 | |
| 83 | files := make([]utils.FileInfo, 0, 256) |
| 84 | meta := map[string]string{} |
| 85 | sources := map[string]struct{}{} |
| 86 | var malformed, lineNo int |
| 87 | |
| 88 | for sc.Scan() { |
| 89 | lineNo++ |
| 90 | line := strings.TrimSpace(sc.Text()) |
| 91 | if line == "" { |
| 92 | continue |
| 93 | } |
| 94 | var c docFlattenChunk |
| 95 | if err := json.Unmarshal([]byte(line), &c); err != nil || strings.TrimSpace(c.Content) == "" { |
| 96 | malformed++ |
| 97 | continue |
| 98 | } |
| 99 | if c.ID == "" { |
| 100 | // Tolerate generators that omit ids: synthesize a stable one from |
| 101 | // the source (or the line number as a last resort). |
| 102 | src := c.Source |
| 103 | if src == "" { |
| 104 | src = filepath.Base(path) |
| 105 | } |
| 106 | c.ID = fmt.Sprintf("%s#%04d", src, lineNo) |
| 107 | } |
| 108 | if c.Source == "" { |
| 109 | c.Source = strings.SplitN(c.ID, "#", 2)[0] |
| 110 | } |
| 111 | files = append(files, utils.FileInfo{ |
| 112 | Path: c.ID, |
| 113 | Content: c.Content, |
| 114 | Size: int64(len(c.Content)), |
| 115 | Type: knowledgeChunkType(c.Source), |
| 116 | }) |
| 117 | sources[c.Source] = struct{}{} |
| 118 | if c.RepoURL != "" { |
| 119 | meta[knowledgeMetaRepoURL] = c.RepoURL |
| 120 | } |
| 121 | if c.Commit != "" { |
| 122 | meta[knowledgeMetaCommit] = c.Commit |
| 123 | } |
| 124 | if len(files) >= maxKnowledgeChunks { |
| 125 | logger.Warn("knowledge: corpus truncated at chunk cap", |
| 126 | zap.String("path", path), zap.Int("cap", maxKnowledgeChunks)) |
| 127 | break |