crawlDocsFlatten performs a bounded BFS crawl from cfg.URL, turning each page into docsFlattenChunks. It returns the chunks, the number of pages actually fetched, and whether a cap (MaxPages/MaxDepth) cut the walk short while links still remained (so the caller can surface a non-silent note).
(ctx context.Context, cfg docsFlattenArgs, emit func(string))
| 108 | // actually fetched, and whether a cap (MaxPages/MaxDepth) cut the walk short |
| 109 | // while links still remained (so the caller can surface a non-silent note). |
| 110 | func crawlDocsFlatten(ctx context.Context, cfg docsFlattenArgs, emit func(string)) ([]docsFlattenChunk, int, bool, error) { |
| 111 | seedURL, err := validateWebTarget(cfg.URL) |
| 112 | if err != nil { |
| 113 | return nil, 0, false, fmt.Errorf("invalid seed url %q: %w", cfg.URL, err) |
| 114 | } |
| 115 | seedHost := hostOf(seedURL) |
| 116 | |
| 117 | visited := make(map[string]bool) |
| 118 | queue := []docsFlattenCrawlItem{{url: seedURL, depth: 0}} |
| 119 | visited[normalizeDocsFlattenURL(seedURL)] = true |
| 120 | |
| 121 | var chunks []docsFlattenChunk |
| 122 | pages := 0 |
| 123 | chunkIndex := 1 |
| 124 | capped := false |
| 125 | |
| 126 | for len(queue) > 0 { |
| 127 | if err := ctx.Err(); err != nil { |
| 128 | return nil, 0, false, err |
| 129 | } |
| 130 | if pages >= cfg.MaxPages { |
| 131 | // Anything still queued is uncrawled because of the page cap. |
| 132 | if len(queue) > 0 { |
| 133 | capped = true |
| 134 | } |
| 135 | break |
| 136 | } |
| 137 | |
| 138 | item := queue[0] |
| 139 | queue = queue[1:] |
| 140 | |
| 141 | pageText, pageTitle, links, ok := fetchDocsFlattenPage(ctx, item.url, emit) |
| 142 | if !ok { |
| 143 | continue |
| 144 | } |
| 145 | pages++ |
| 146 | emit(fmt.Sprintf("crawled %s (%d/%d)", item.url, pages, cfg.MaxPages)) |
| 147 | |
| 148 | for _, c := range chunkMarkdown(pageText, cfg.MaxChars) { |
| 149 | chunks = append(chunks, docsFlattenChunk{ |
| 150 | ID: fmt.Sprintf("%s#%04d", seedHost, chunkIndex), |
| 151 | Source: item.url, |
| 152 | Title: pageTitle, |
| 153 | Content: c, |
| 154 | ChunkSize: len(c), |
| 155 | RepoURL: cfg.URL, |
| 156 | }) |
| 157 | chunkIndex++ |
| 158 | } |
| 159 | |
| 160 | // Enqueue children unless we've reached the depth limit. |
| 161 | if item.depth >= cfg.MaxDepth { |
| 162 | if len(links) > 0 { |
| 163 | // Links exist below the depth horizon but won't be followed. |
| 164 | capped = true |
| 165 | } |
| 166 | continue |
| 167 | } |