Function extractText

cli/plugins/builtin_webfetch.go:523–567

extractText extracts readable text from HTML, removing scripts, styles, and tags.

(htmlContent string)

// extractText extracts readable text from HTML, removing scripts, styles, and tags.
func extractText(htmlContent string) string {
	doc, err := html.Parse(strings.NewReader(htmlContent))
	if err != nil {
		// Fallback: strip tags with regex
		return stripHTMLTags(htmlContent)
	}

	var sb strings.Builder
	var extractNode func(*html.Node)
	extractNode = func(n *html.Node) {
		if n.Type == html.ElementNode {
			// Skip script, style, head
			if n.Data == "script" || n.Data == "style" || n.Data == "head" || n.Data == "noscript" {
				return
			}
			// Add newlines for block elements
			if isBlockElement(n.Data) {
				sb.WriteString("\n")
			}
		}

		if n.Type == html.TextNode {
			text := strings.TrimSpace(n.Data)
			if text != "" {
				sb.WriteString(text + " ")
			}
		}

		for c := n.FirstChild; c != nil; c = c.NextSibling {
			extractNode(c)
		}

		if n.Type == html.ElementNode && isBlockElement(n.Data) {
			sb.WriteString("\n")
		}
	}

	extractNode(doc)

	// Clean up excessive whitespace
	result := sb.String()
	re := regexp.MustCompile(`\n{3,}`)
	result = re.ReplaceAllString(result, "\n\n")
	return strings.TrimSpace(result)
}

func isBlockElement(tag string) bool {
	switch tag {