Method ProcessDirectory

input/inpututils/s3.go:51–78

ProcessDirectory enqueues all files matching a specific prefix for processing by s3Input. If prefix is actually a s3 url use the bucket there instead of the one provided at creation time. This function makes (multiple) remotes call to acquire the listing of all files matching the specified prefix i

(prefix string)

// all files matching the specified prefix in the bucket, and enqueue
// them for processing,
func (s *S3Input) ProcessDirectory(prefix string) error {
	s3Scheme, s3Bucket, s3Key, err := s.choosePathComponents(prefix)
	if err != nil {
		return err
	}
	isFullPath, err := isValidScheme(s3Scheme)
	if err != nil {
		return err
	}
	return s.svc.ListObjectsPages(&s3.ListObjectsInput{
		Bucket: aws.String(s3Bucket),
		Prefix: aws.String(s3Key),
	}, func(page *s3.ListObjectsOutput, lastPage bool) bool {
		for _, o := range page.Contents {
			var key string
			// If prefix is a full s3 url it means we need to provide
			// openS3File with the full path in order to be able to correctly
			// fetch the size and the contents of the file.
			if isFullPath {
				key = fmt.Sprintf("%s://%s/%s", s3Scheme, s3Bucket, *o.Key)
			} else {
				key = *o.Key
			}
			s.ProcessFile(key)
		}
		return true
	})
}

func (s *S3Input) openS3File(fn string) (io.ReadCloser, int64, time.Time, *url.URL, error) {
	_, s3Bucket, s3Key, err := s.choosePathComponents(fn)