(urls []string, url string, added map[string]bool, excluded map[string]bool)
| 1331 | } |
| 1332 | |
| 1333 | func addUrl(urls []string, url string, added map[string]bool, excluded map[string]bool) []string { |
| 1334 | url = addDomain(url, Config.Website.Url.Hostname()) |
| 1335 | |
| 1336 | if url != "" { |
| 1337 | // Check if url isnt added yet and if it satisfies RecInclude (=contains it) |
| 1338 | if excluded[url] { |
| 1339 | msg := fmt.Sprintf("Skipped to add %s to the queue, because it is on the exclude list\n", url) |
| 1340 | PrintVerbose(msg, NoColor, 2) |
| 1341 | } else if added[url] { |
| 1342 | msg := fmt.Sprintf("Skipped to add %s to the queue, because it was already added\n", url) |
| 1343 | PrintVerbose(msg, NoColor, 2) |
| 1344 | } else if Config.RecInclude == "" || checkRecInclude(url, Config.RecInclude) { |
| 1345 | urls = append(urls, url) |
| 1346 | added[url] = true |
| 1347 | } else { |
| 1348 | msg := fmt.Sprintf("Skipped to add %s to the queue, because it doesn't satisfy RecInclude\n", url) |
| 1349 | PrintVerbose(msg, NoColor, 2) |
| 1350 | } |
| 1351 | } |
| 1352 | |
| 1353 | return urls |
| 1354 | } |
| 1355 | |
| 1356 | func CrawlUrls(u string, added map[string]bool, excluded map[string]bool) []string { |
| 1357 | webStruct, err := GetWebsite(u, false, false) // get body without cachebuster. TODO use response w/o cachebuster from recon, so it doesn't have to be fetched again |
no test coverage detected