extractTextFromHTML strips , , and all other tags, then collapses whitespace into readable text. Not a full HTML parser — regex-based, good enough for prose extraction from typical pages.
(htmlContent string)
| 165 | } |
| 166 | |
| 167 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, parsed.String(), nil) |
| 168 | if err != nil { |
| 169 | return "", fmt.Errorf("building request: %w", err) |
| 170 | } |
| 171 | req.Header.Set("User-Agent", webFetchUserAgent) |
| 172 | req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") |
| 173 | |
| 174 | resp, err := webFetchClient().Do(req) |
| 175 | if err != nil { |
| 176 | return "", fmt.Errorf("request failed: %w", err) |
| 177 | } |
| 178 | defer resp.Body.Close() |
| 179 | |
| 180 | body, err := io.ReadAll(http.MaxBytesReader(nil, resp.Body, webFetchMaxBytes)) |
| 181 | if err != nil { |
| 182 | var maxBytesErr *http.MaxBytesError |
| 183 | if errors.As(err, &maxBytesErr) { |
| 184 | return "", fmt.Errorf("response exceeded %d byte limit", webFetchMaxBytes) |
| 185 | } |