Function ExtractSources

extractor/extractor.go:33–68

ExtractSources extracts all JavaScript sources found in the provided HTTP response reader. The optional extractionPoints can be used to overwrite the default extraction points map with a set of HTML tag names, together with a list of what attributes to extract from.

(input io.Reader, extractionPoints ...map[string][]string)

// The optional extractionPoints can be used to overwrite the default extraction points map
// with a set of HTML tag names, together with a list of what attributes to extract from.
func ExtractSources(input io.Reader, extractionPoints ...map[string][]string) (<-chan url.URL, error) {
	doc, err := goquery.NewDocumentFromReader(input)
	if err != nil {
		return nil, err
	}

	var (
		urls   = make(chan url.URL)
		points = ExtractionPoints
	)

	if len(extractionPoints) > 0 {
		points = extractionPoints[0]
	}

	go func() {
		defer close(urls)
		for tag, attributes := range points {
			doc.Find(tag).Each(func(i int, s *goquery.Selection) {
				for _, a := range attributes {
					if value, exists := s.Attr(a); exists {
						u, err := url.Parse(value)
						if err != nil {
							log.Println(fmt.Errorf("invalid attribute value %s cannot be parsed to a URL: %w", value, err))
							continue
						}

						urls <- *u
					}
				}
			})
		}
	}()

	return urls, nil
}

// Filter applies options to filter URLs from the input channel.
func Filter(input <-chan url.URL, options ...func([]url.URL) []url.URL) (<-chan url.URL, error) {