(ctx context.Context, endpoint string, query url.Values, httpClient *http.Client)
| 564 | } |
| 565 | |
| 566 | func fetchWebSearchHTML(ctx context.Context, endpoint string, query url.Values, httpClient *http.Client) (string, error) { |
| 567 | parsedURL, err := url.Parse(endpoint) |
| 568 | if err != nil { |
| 569 | return "", err |
| 570 | } |
| 571 | |
| 572 | searchQuery := parsedURL.Query() |
| 573 | for key, values := range query { |
| 574 | for _, value := range values { |
| 575 | searchQuery.Add(key, value) |
| 576 | } |
| 577 | } |
| 578 | parsedURL.RawQuery = searchQuery.Encode() |
| 579 | |
| 580 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, parsedURL.String(), nil) |
| 581 | if err != nil { |
| 582 | return "", err |
| 583 | } |
| 584 | req.Header.Set("User-Agent", webSearchUserAgent) |
| 585 | req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") |
| 586 | |
| 587 | resp, err := httpClient.Do(req) |
| 588 | if err != nil { |
| 589 | return "", err |
| 590 | } |
| 591 | defer resp.Body.Close() |
| 592 | |
| 593 | if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { |
| 594 | return "", fmt.Errorf("HTTP %d: %s", resp.StatusCode, resp.Status) |
| 595 | } |
| 596 | |
| 597 | bodyBytes, err := io.ReadAll(io.LimitReader(resp.Body, webSearchMaxResponseSize+1)) |
| 598 | if err != nil { |
| 599 | return "", err |
| 600 | } |
| 601 | if len(bodyBytes) > webSearchMaxResponseSize { |
| 602 | return "", fmt.Errorf("response body exceeds %d bytes", webSearchMaxResponseSize) |
| 603 | } |
| 604 | |
| 605 | return string(bodyBytes), nil |
| 606 | } |
| 607 | |
| 608 | func parseDuckDuckGoHTML(body string) ([]webSearchResult, error) { |
| 609 | root, err := html.Parse(strings.NewReader(body)) |
no test coverage detected