fetch retrieves content from a URL with safety checks
(urlStr string)
| 85 | |
| 86 | // fetch retrieves content from a URL with safety checks |
| 87 | func (p *FetchPlugin) fetch(urlStr string) (string, error) { |
| 88 | debugf("Fetch: requesting URL %q", urlStr) |
| 89 | |
| 90 | client := &http.Client{} |
| 91 | req, err := http.NewRequest("GET", urlStr, nil) |
| 92 | if err != nil { |
| 93 | return "", fmt.Errorf(i18n.T("fetch_error_create_request"), err) |
| 94 | } |
| 95 | req.Header.Set("User-Agent", UserAgent) |
| 96 | |
| 97 | resp, err := client.Do(req) |
| 98 | if err != nil { |
| 99 | return "", fmt.Errorf(i18n.T("fetch_error_fetching_url"), err) |
| 100 | } |
| 101 | defer resp.Body.Close() |
| 102 | |
| 103 | debugf("Fetch: got response status=%q", resp.Status) |
| 104 | if resp.StatusCode != http.StatusOK { |
| 105 | return "", fmt.Errorf(i18n.T("fetch_http_error"), resp.StatusCode, resp.Status) |
| 106 | } |
| 107 | |
| 108 | if contentLength := resp.ContentLength; contentLength > MaxContentSize { |
| 109 | return "", fmt.Errorf(i18n.T("fetch_content_too_large"), contentLength, MaxContentSize) |
| 110 | } |
| 111 | |
| 112 | contentType := resp.Header.Get("Content-Type") |
| 113 | debugf("Fetch: content-type=%q", contentType) |
| 114 | if !p.isTextContent(contentType) { |
| 115 | return "", fmt.Errorf(i18n.T("fetch_unsupported_content_type"), contentType) |
| 116 | } |
| 117 | |
| 118 | debugf("Fetch: reading response body") |
| 119 | limitReader := io.LimitReader(resp.Body, MaxContentSize+1) |
| 120 | content, err := io.ReadAll(limitReader) |
| 121 | if err != nil { |
| 122 | return "", fmt.Errorf(i18n.T("fetch_error_reading_response"), err) |
| 123 | } |
| 124 | |
| 125 | if len(content) > MaxContentSize { |
| 126 | return "", fmt.Errorf(i18n.T("fetch_content_exceeds_limit"), MaxContentSize) |
| 127 | } |
| 128 | |
| 129 | if err := p.validateTextContent(content); err != nil { |
| 130 | return "", err |
| 131 | } |
| 132 | |
| 133 | debugf("Fetch: operation completed successfully, read %d bytes", len(content)) |
| 134 | return string(content), nil |
| 135 | } |