fetchSearchHTML centralizes the request path shared by the keyless scraping providers: SSRF validation, browser-like headers, body cap.
(ctx context.Context, endpoint, query string)
| 48 | // fetchSearchHTML centralizes the request path shared by the keyless |
| 49 | // scraping providers: SSRF validation, browser-like headers, body cap. |
| 50 | func fetchSearchHTML(ctx context.Context, endpoint, query string) (string, error) { |
| 51 | reqCtx, cancel := context.WithTimeout(ctx, keylessSearchTimeout) |
| 52 | defer cancel() |
| 53 | |
| 54 | safeURL, err := validateWebTarget(endpoint + url.QueryEscape(query)) |
| 55 | if err != nil { |
| 56 | return "", fmt.Errorf("refusing search request: %w", err) |
| 57 | } |
| 58 | resp, err := webGet(reqCtx, safeURL, map[string]string{ |
| 59 | "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", |
| 60 | "Accept-Language": "en-US,en;q=0.9", |
| 61 | }) |
| 62 | if err != nil { |
| 63 | return "", fmt.Errorf("search request failed: %w", err) |
| 64 | } |
| 65 | defer resp.Body.Close() |
| 66 | |
| 67 | if resp.StatusCode != 200 { |
| 68 | return "", fmt.Errorf("search returned HTTP %d", resp.StatusCode) |
| 69 | } |
| 70 | body, err := io.ReadAll(io.LimitReader(resp.Body, keylessSearchBodyCap)) |
| 71 | if err != nil { |
| 72 | return "", fmt.Errorf("reading response: %w", err) |
| 73 | } |
| 74 | return string(body), nil |
| 75 | } |
| 76 | |
| 77 | // ── Brave Search ──────────────────────────────────────────────────── |
| 78 |