Function fetchWithRateLimitHandling

main.go:265–315

fetchWithRateLimitHandling fetches the document from the given URL and handles 403 rate limiting or connection errors.

(url string)

// fetchWithRateLimitHandling fetches the document from the given URL and handles 403 rate limiting or connection errors.
func fetchWithRateLimitHandling(url string) (*http.Response, error) {
	maxRetries := 5
	for retries := 0; retries < maxRetries; retries++ {
		// Randomly select a user agent from the list
		userAgent := userAgents[rand.Intn(len(userAgents))]

		// Create a new HTTP request
		req, err := http.NewRequest("GET", url, nil)
		if err != nil {
			return nil, err
		}

		// Set the user agent header
		req.Header.Set("User-Agent", userAgent)

		// Send the request
		client := &http.Client{}
		resp, err := client.Do(req)
		if err != nil {
			// Check if it's a temporary network error
			if nerr, ok := err.(net.Error); ok && nerr.Temporary() {
				log.Printf("Temporary error fetching URL: %v, retrying in %s", err, sleepDuration)
				time.Sleep(sleepDuration)
				continue
			} else {
				// Non-recoverable error, log and exit
				log.Printf("Error fetching URL: %v", err)
				return nil, err
			}
		}

		// If successful response, return it
		if resp.StatusCode == http.StatusOK {
			return resp, nil
		}

		// Handle rate limiting (403 Forbidden)
		if resp.StatusCode == http.StatusForbidden {
			log.Printf("Received 403 Forbidden (rate limit), pausing for %s before retrying...", sleepDuration)
			time.Sleep(sleepDuration)
			resp.Body.Close()
			continue
		}

		// Handle other unexpected status codes
		log.Printf("Unexpected status code: %d for URL %s", resp.StatusCode, url)
		resp.Body.Close()
		return nil, fmt.Errorf("unexpected status code: %d for URL %s", resp.StatusCode, url)
	}
	return nil, fmt.Errorf("max retries exceeded for URL %s", url)
}

func downloadAndSaveAsWARC(url string) {
	// Get the current date