Function CrawlUrls

pkg/recon.go:1356–1416

(u string, added map[string]bool, excluded map[string]bool)

}

func CrawlUrls(u string, added map[string]bool, excluded map[string]bool) []string {
	webStruct, err := GetWebsite(u, false, false) // get body without cachebuster. TODO use response w/o cachebuster from recon, so it doesn't have to be fetched again
	if err != nil {
		msg := fmt.Sprintf("Error while crawling %s: %s\n", u, err.Error())
		Print(msg, Red)
		return []string{}
	}
	bodyReader := strings.NewReader(webStruct.Body)
	tokenizer := html.NewTokenizer(bodyReader)

	var urls []string

	eof := false
	for !eof {
		tokentype := tokenizer.Next()

		switch tokentype {
		case html.StartTagToken, html.SelfClosingTagToken:

			token := tokenizer.Token()

			if token.Data == "a" || token.Data == "link" {
				for _, a := range token.Attr {
					if a.Key == "href" {
						//if strings.HasSuffix(a.Val, ".css") && strings.Contains(a.Val, ".css?") { // TODO: Flag to exclude css files from thorough scanning
						//	break
						//}
						urls = addUrl(urls, a.Val, added, excluded)
						break
					}
				}
			} else if token.Data == "script" {
				for _, a := range token.Attr {
					if a.Key == "src" {
						urls = addUrl(urls, a.Val, added, excluded)
						break
					}
				}
			}

		// When EOF is reached a html.ErrorToken appears
		case html.ErrorToken:
			err := tokenizer.Err()
			if err == io.EOF {
				eof = true
				break
			}
			msg := fmt.Sprintf("error tokenizing HTML: %+v", tokenizer.Err())
			Print(msg, Yellow)
		}
	}

	// redirect
	if webStruct.Headers["Location"] != nil {
		if h := webStruct.Headers["Location"][0]; h != "" {
			urls = addUrl(urls, h, added, excluded)
		}
	}