DownloadFileExtended retrieves a file with retry logic, optional progress bar, and SHA256 verification. It allows specifying the number of retries and timeout duration.
(destPath string, fileURL string, progressBar bool, shaSumURL string, retries int, timeout time.Duration)
| 28 | // DownloadFileExtended retrieves a file with retry logic, optional progress bar, and SHA256 verification. |
| 29 | // It allows specifying the number of retries and timeout duration. |
| 30 | func DownloadFileExtended(destPath string, fileURL string, progressBar bool, shaSumURL string, retries int, timeout time.Duration) (err error) { |
| 31 | const timeoutMax = 1 * time.Hour |
| 32 | |
| 33 | if output.JSONOutput || !term.IsTerminal(int(os.Stdin.Fd())) { |
| 34 | progressBar = false |
| 35 | } |
| 36 | |
| 37 | // Configure retryablehttp client with backoff, retry policy, and global timeout. |
| 38 | createClient := func(clientTimeout time.Duration, attempt int) (*retryablehttp.Client, time.Duration) { |
| 39 | client := retryablehttp.NewClient() |
| 40 | client.RetryMax = retries |
| 41 | client.RetryWaitMin = 500 * time.Millisecond |
| 42 | client.RetryWaitMax = 5 * time.Second |
| 43 | // "context deadline exceeded" error during file copying cannot be retried with retryablehttp |
| 44 | // See https://github.com/hashicorp/go-retryablehttp/issues/167 |
| 45 | // We use manual retry logic (for loop) around the entire Get+Copy operation instead |
| 46 | client.CheckRetry = retryablehttp.DefaultRetryPolicy |
| 47 | client.Backoff = retryablehttp.DefaultBackoff |
| 48 | client.Logger = nil |
| 49 | // Double the timeout for each retry attempt, up to timeoutMax |
| 50 | // 1st attempt = clientTimeout * 2^0 |
| 51 | // 2nd attempt = clientTimeout * 2^1 |
| 52 | // 3rd attempt = clientTimeout * 2^2 |
| 53 | clientTimeout = min(clientTimeout*time.Duration(1<<attempt), timeoutMax) |
| 54 | // Timeout for the entire request |
| 55 | client.HTTPClient.Timeout = clientTimeout |
| 56 | client.RequestLogHook = func(_ retryablehttp.Logger, req *http.Request, attempt int) { |
| 57 | if attempt > 0 { |
| 58 | // attempt==1 is the first retry, 2 the second, etc |
| 59 | Debug("Retrying download of %s try #%d", req.URL.String(), attempt) |
| 60 | } |
| 61 | } |
| 62 | return client, clientTimeout |
| 63 | } |
| 64 | |
| 65 | // Ensure partial files are removed on any error. |
| 66 | defer func() { |
| 67 | if err != nil { |
| 68 | _ = os.Remove(destPath) |
| 69 | } |
| 70 | }() |
| 71 | |
| 72 | // Download expected SHA sum if provided with retry logic. |
| 73 | var expectedSHA string |
| 74 | if shaSumURL != "" { |
| 75 | // SHA is a smaller file, use a shorter timeout |
| 76 | shaSumTimeout := 20 * time.Second |
| 77 | for attempt := 0; attempt <= retries; attempt++ { |
| 78 | client, currentTimeout := createClient(shaSumTimeout, attempt) |
| 79 | Debug("Attempting to download SHASUM URL=%s (attempt %d/%d) with timeout %v", shaSumURL, attempt+1, retries+1, currentTimeout) |
| 80 | req, reqErr := retryablehttp.NewRequest("GET", shaSumURL, nil) |
| 81 | if reqErr != nil { |
| 82 | err = fmt.Errorf("creating request for shaSum URL %s: %w", shaSumURL, reqErr) |
| 83 | return |
| 84 | } |
| 85 | gitHubHeaders := github.GetGitHubHeaders(shaSumURL) |
| 86 | for key, value := range gitHubHeaders { |
| 87 | req.Header.Set(key, value) |