Method ExecuteWithStream

cli/plugins/builtin_webfetch.go:111–212

(ctx context.Context, args []string, onOutput func(string))

}

func (p *BuiltinWebFetchPlugin) ExecuteWithStream(ctx context.Context, args []string, onOutput func(string)) (string, error) {
	if len(args) == 0 {
		return "", fmt.Errorf("url required. Usage: @webfetch fetch --url <URL>")
	}

	parsed, err := parseFetchArgs(args)
	if err != nil {
		return "", err
	}
	if parsed.URL == "" {
		return "", fmt.Errorf("url required")
	}
	if parsed.MaxLength <= 0 {
		parsed.MaxLength = defaultWebFetchMaxLength
	}

	if onOutput != nil {
		onOutput(fmt.Sprintf("Fetching %s...", parsed.URL))
	}

	// Create HTTP request with timeout
	reqCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
	defer cancel()

	// SSRF guard: validate (and canonicalize) the target before it reaches the
	// HTTP client. Blocks cloud-metadata/link-local always; private/loopback
	// only when CHATCLI_WEBFETCH_BLOCK_PRIVATE is set.
	safeURL, err := validateWebTarget(parsed.URL)
	if err != nil {
		return "", fmt.Errorf("refusing to fetch %q: %w", parsed.URL, err)
	}

	// webGet sends a browser UA by default (avoids CDN bot-blocks) and
	// auto-retries with a neutral tool UA on a 401/407 gateway challenge —
	// the behavior of TLS-intercepting corporate proxies toward "browsers".
	resp, err := webGet(reqCtx, safeURL, map[string]string{
		"Accept":          "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
		"Accept-Language": "en-US,en;q=0.9",
	})
	if err != nil {
		return "", fmt.Errorf("fetching URL: %w", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		return "", fmt.Errorf("HTTP %d: %s", resp.StatusCode, resp.Status)
	}

	// Read body with hard cap of 10MB to avoid memory blowup.
	body, err := io.ReadAll(io.LimitReader(resp.Body, 10*1024*1024))
	if err != nil {
		return "", fmt.Errorf("reading body: %w", err)
	}

	fullContent := string(body)
	if !parsed.Raw {
		fullContent = p.extractWithRenderEscalation(reqCtx, parsed, safeURL, fullContent, onOutput)
	}