Method S3FileIterator

pkg/lib/aws/s3.go:655–693

(bucket string, s3Obj *s3.Object, partSize int, fn func(buffer io.ReadCloser, isLastPart bool) (bool, error))

}

func (c *Client) S3FileIterator(bucket string, s3Obj *s3.Object, partSize int, fn func(buffer io.ReadCloser, isLastPart bool) (bool, error)) error {
	size := int(*s3Obj.Size)

	iters := size / partSize
	if size%partSize != 0 {
		iters++
	}

	for i := 0; i < iters; i++ {
		min := i * (partSize)
		max := (i + 1) * (partSize)
		if max > size {
			max = size
		}
		max--

		byteRange := fmt.Sprintf("bytes=%d-%d", min, max)
		obj, err := c.S3().GetObject(&s3.GetObjectInput{
			Bucket: aws.String(bucket),
			Key:    s3Obj.Key,
			Range:  aws.String(byteRange), // use range instead of part numbers because only files uploaded using multipart have parts
		})
		if err != nil {
			return errors.Wrap(err, S3Path(bucket, *s3Obj.Key), "range "+byteRange)
		}

		isLastChunk := i+1 == iters
		shouldContinue, err := fn(obj.Body, isLastChunk)
		if err != nil {
			return errors.Wrap(err, S3Path(bucket, *s3Obj.Key))
		}

		if !shouldContinue {
			break
		}
	}

	return nil
}

func (c *Client) ListS3Dir(bucket string, s3Dir string, includeDirObjects bool, maxResults *int64, startAfter *string) ([]*s3.Object, error) {
	prefix := s.EnsureSuffix(s3Dir, "/")