Function executeWithRetry

apps/sim/tools/index.ts:425–500

* Execute a function with exponential backoff retry for rate limiting errors. * Only used for hosted key requests. Tracks rate limit events via telemetry. * * On terminal upstream 429, optionally re-enters the hosted-key queue (which waits for * the per-workspace bucket to refill) and retries on

(
  fn: () => Promise<T>,
  context: RetryContext,
  maxRetries = 3,
  baseDelayMs = 1000
)

 * re-queue the call instead of surfacing the error.
 */
async function executeWithRetry<T>(
  fn: () => Promise<T>,
  context: RetryContext,
  maxRetries = 3,
  baseDelayMs = 1000
): Promise<T> {
  const {
    requestId,
    toolId,
    provider,
    envVarName,
    executionContext,
    reacquireAfterRetriesExhausted,
  } = context
  let lastError: unknown

  for (let attempt = 0; attempt <= maxRetries; attempt++) {
    try {
      return await fn()
    } catch (error) {
      lastError = error

      if (!isRateLimitError(error) || attempt === maxRetries) {
        if (isRateLimitError(error) && attempt === maxRetries) {
          if (reacquireAfterRetriesExhausted) {
            try {
              const requeued = await reacquireAfterRetriesExhausted()
              if (requeued) {
                logger.warn(
                  `[${requestId}] Upstream retries exhausted for ${toolId} (${envVarName}); re-queued and retrying once with fresh key`
                )
                return (await requeued()) as T
              }
            } catch (requeueError) {
              logger.error(
                `[${requestId}] Re-queue after exhausted upstream retries failed for ${toolId}`,
                { error: toError(requeueError).message }
              )
            }
          }

          PlatformEvents.hostedKeyUserThrottled({
            toolId,
            reason: 'upstream_retries_exhausted',
            provider,
            userId: executionContext?.userId,
            workspaceId: executionContext?.workspaceId,
            workflowId: executionContext?.workflowId,
          })
        }
        throw error
      }

      const delayMs = baseDelayMs * 2 ** attempt

      // Track throttling event via telemetry
      PlatformEvents.hostedKeyRateLimited({
        toolId,