Function executeWebFetch

api/service/tools/web_fetch.go:43–125

(ctx context.Context, args map[string]interface{})

}

func executeWebFetch(ctx context.Context, args map[string]interface{}) (*ToolResult, error) {
	rawURL, ok := args["url"].(string)
	if !ok || strings.TrimSpace(rawURL) == "" {
		return nil, fmt.Errorf("missing url argument")
	}
	rawURL = strings.TrimSpace(rawURL)

	extractMode := defaultWebFetchExtractMode
	if mode, ok := args["extractMode"].(string); ok {
		mode = strings.TrimSpace(strings.ToLower(mode))
		if mode == "text" || mode == "markdown" {
			extractMode = mode
		}
	}

	maxChars := defaultWebFetchMaxChars
	if v, exists := args["maxChars"]; exists {
		maxChars = parseIntValue(v, defaultWebFetchMaxChars)
		if maxChars < 256 {
			maxChars = 256
		}
	}

	parsed, err := url.Parse(rawURL)
	if err != nil {
		return nil, fmt.Errorf("invalid url: %w", err)
	}
	if parsed.Scheme != "http" && parsed.Scheme != "https" {
		return nil, fmt.Errorf("only http and https URLs are supported")
	}
	if strings.TrimSpace(parsed.Hostname()) == "" {
		return nil, fmt.Errorf("missing hostname in url")
	}

	proxyURL := webFetchProxyURLForRequest(parsed)
	proxyAware := proxyURL != nil
	if !proxyAware {
		if err := checkWebFetchSSRF(ctx, parsed.Hostname(), defaultWebFetchResolver); err != nil {
			return nil, err
		}
	}

	result, err := fetchWebContent(ctx, rawURL, proxyURL)
	if err != nil {
		return nil, err
	}

	content := result.Content
	if strings.Contains(result.ContentType, "text/html") {
		content = extractTextFromHTML(result.Content)
	}

	if !utf8.ValidString(content) {
		content = webFetchBinaryPreviewPrefix
	}
	if extractMode == "markdown" {
		content = normalizeWebFetchText(content)
	}