Function visitHTMLLink

pkg/crawler/colly.go:433–440

visitHTMLLink checks if the collector should visit a link or not.

(link string, event *Event, e *colly.HTMLElement, c *colly.Collector)

// visitHTMLLink checks if the collector should visit a link or not.
func visitHTMLLink(link string, event *Event, e *colly.HTMLElement, c *colly.Collector) {
	if len(link) != 0 && !strings.HasPrefix(link, "data:image") {
		absoluteURL := urlUtils.AbsoluteURL(event.ProtocolTemp, event.TargetTemp, e.Request.AbsoluteURL(link))
		// Visit link found on page
		// Only those links are visited which are in AllowedDomains
		visitLink(event, c, absoluteURL)
	}
}

// visitXMLLink checks if the collector should visit a link or not.
func visitXMLLink(link string, event *Event, e *colly.XMLElement, c *colly.Collector) {