Method GetAllObjects

pkg/agent/storage/s3.go:97–156

(s3Svc s3iface.S3API)

}

func (s *S3ObjectDownloader) GetAllObjects(s3Svc s3iface.S3API) ([]s3manager.BatchDownloadObject, error) {
	resp, err := s3Svc.ListObjects(&s3.ListObjectsInput{
		Bucket: aws.String(s.Bucket),
		Prefix: aws.String(s.Prefix),
	})
	if err != nil {
		return nil, err
	}
	results := make([]s3manager.BatchDownloadObject, 0)

	if len(resp.Contents) == 0 {
		return nil, fmt.Errorf("%s has no objects or does not exist", s.StorageUri)
	}

	foundObject := false

	for _, object := range resp.Contents {
		if strings.HasSuffix(*object.Key, "/") {
			continue
		}
		subObjectKey := strings.TrimPrefix(*object.Key, s.Prefix)
		fileName := filepath.Join(s.ModelDir, s.ModelName, subObjectKey)

		if FileExists(fileName) {
			// File got corrupted or is mid-download :(
			// TODO: Figure out if we can maybe continue?
			if err := os.Remove(fileName); err != nil {
				return nil, fmt.Errorf("file is unable to be deleted: %w", err)
			}
		}
		file, err := Create(fileName)
		if err != nil {
			return nil, fmt.Errorf("file is already created: %w", err)
		}
		object := s3manager.BatchDownloadObject{
			Object: &s3.GetObjectInput{
				Key:    aws.String(*object.Key),
				Bucket: aws.String(s.Bucket),
			},
			Writer: file,
			After: func() error {
				defer func(file *os.File) {
					closeErr := file.Close()
					if closeErr != nil {
						log.Error(closeErr, "failed to close file")
					}
				}(file)
				return nil
			},
		}
		foundObject = true
		results = append(results, object)
	}

	if !foundObject {
		return nil, fmt.Errorf("%s has no objects or does not exist", s.StorageUri)
	}