Method processURL

filter/external_match.go:175–225

(u string)

}

func (f *ExternalMatch) processURL(u string) (map[string]struct{}, error) {
	parsed, err := url.Parse(u)
	if err != nil {
		return nil, fmt.Errorf("error parsing URL: %v", err)
	}

	var r io.Reader

	log.WithField("url", u).Info("begin parsing file")
	switch parsed.Scheme {
	case "s3":
		sess, err := session.NewSession(&aws.Config{Region: aws.String(f.cfg.Region)})
		if err != nil {
			return nil, fmt.Errorf("error creating aws session: %v", err)
		}
		resp, err := s3.New(sess).GetObject(&s3.GetObjectInput{
			Bucket: aws.String(parsed.Host),
			Key:    aws.String(parsed.Path),
		})
		if err != nil {
			return nil, fmt.Errorf("error downloading file: %v", err)
		}

		defer resp.Body.Close()
		r = resp.Body

	case "file":
		path := filepath.Join(parsed.Host, parsed.Path) // On Windows Host contains the drive letter.
		f, err := os.Open(path)
		if err != nil {
			return nil, fmt.Errorf("error opening file: %v", err)
		}

		defer f.Close()
		r = f

	default:
		// scheme error should be caught during the validation of configuration.
		panic("unexpected scheme")
	}

	// Wrap the reader into a zt reader to transparently read gzip,
	// zstd or non compressed CSV data.
	zar, err := zt.NewReader(r)
	if err != nil {
		return nil, fmt.Errorf("can't read from %s: %s", u, err)
	}
	defer zar.Close()

	return valuesFromCSV(zar, f.cfg.CSVColumn)
}

func (f *ExternalMatch) updateValues() error {
	values := make(map[string]struct{})