Function retryWithExponentialBackoff

apps/sim/lib/knowledge/documents/utils.ts:98–167

(
  operation: () => Promise<T>,
  options: RetryOptions = {}
)

 * Executes a function with exponential backoff retry logic
 */
export async function retryWithExponentialBackoff<T>(
  operation: () => Promise<T>,
  options: RetryOptions = {}
): Promise<T> {
  const {
    maxRetries = 5,
    initialDelayMs = 1000,
    maxDelayMs = 30000,
    backoffMultiplier = 2,
    retryCondition = isRetryableError,
  } = options

  let lastError: Error | undefined
  let delay = initialDelayMs

  for (let attempt = 0; attempt <= maxRetries; attempt++) {
    try {
      logger.debug(`Executing operation attempt ${attempt + 1}/${maxRetries + 1}`)
      const result = await operation()

      if (attempt > 0) {
        logger.info(`Operation succeeded after ${attempt + 1} attempts`)
      }

      return result
    } catch (error) {
      lastError = toError(error)
      logger.warn(`Operation failed on attempt ${attempt + 1}`, { error })

      // If this is the last attempt, throw the error
      if (attempt === maxRetries) {
        logger.error(`Operation failed after ${maxRetries + 1} attempts`, { error })
        throw lastError
      }

      // Check if error is retryable
      if (!retryCondition(error as RetryableError)) {
        logger.warn('Error is not retryable, throwing immediately', { error })
        throw lastError
      }

      // Use Retry-After if the server told us how long to wait, otherwise exponential backoff.
      // Cap Retry-After at maxDelayMs to bound total retry duration (matches Google Cloud SDK behavior).
      const retryAfterMs = (lastError as HTTPError)?.retryAfterMs
      const cappedRetryAfter = retryAfterMs ? Math.min(retryAfterMs, maxDelayMs) : undefined

      if (retryAfterMs && retryAfterMs > maxDelayMs) {
        logger.warn(
          `Retry-After ${retryAfterMs}ms exceeds maxDelayMs ${maxDelayMs}ms — capping to ${maxDelayMs}ms`
        )
      }

      const jitter = randomFloat() * 0.1 * delay
      const actualDelay = cappedRetryAfter ?? Math.min(delay + jitter, maxDelayMs)

      logger.info(
        `Retrying in ${Math.round(actualDelay)}ms (attempt ${attempt + 1}/${maxRetries + 1})${cappedRetryAfter ? ' (Retry-After)' : ''}`
      )