Function cleanupLargeExecutionValues

apps/sim/background/cleanup-logs.ts:132–196

(
  workspaceIds: string[],
  retentionDate: Date,
  label: string
)

}

async function cleanupLargeExecutionValues(
  workspaceIds: string[],
  retentionDate: Date,
  label: string
): Promise<LargeValueCleanupStats> {
  const stats: LargeValueCleanupStats = {
    largeValuesTotal: 0,
    largeValuesDeleted: 0,
    largeValuesDeleteFailed: 0,
  }
  if (workspaceIds.length === 0) return stats

  const largeValueRetentionDate = new Date(
    retentionDate.getTime() - LARGE_VALUE_CLEANUP_GRACE_HOURS * 60 * 60 * 1000
  )
  const workspaceChunks = chunkArray(workspaceIds, 50)
  let attempted = 0

  for (const chunkIds of workspaceChunks) {
    while (attempted < LARGE_VALUE_CLEANUP_TOTAL_KEY_LIMIT) {
      const limit = Math.min(
        LARGE_VALUE_CLEANUP_BATCH_SIZE,
        LARGE_VALUE_CLEANUP_TOTAL_KEY_LIMIT - attempted
      )
      const rows = await db
        .select({ key: executionLargeValues.key })
        .from(executionLargeValues)
        .where(
          and(
            inArray(executionLargeValues.workspaceId, chunkIds),
            isNull(executionLargeValues.deletedAt),
            lt(executionLargeValues.createdAt, largeValueRetentionDate),
            unreferencedLargeValuePredicate()
          )
        )
        .orderBy(
          asc(executionLargeValues.workspaceId),
          asc(executionLargeValues.createdAt),
          asc(executionLargeValues.key)
        )
        .limit(limit)

      if (rows.length === 0) break

      const keys = rows.map((row) => row.key)
      stats.largeValuesTotal += keys.length
      attempted += keys.length
      const result = await deleteLargeValueKeys(keys)
      stats.largeValuesDeleted += result.deleted
      stats.largeValuesDeleteFailed += result.failed

      if (result.deleted === 0) {
        break
      }
    }

    if (attempted >= LARGE_VALUE_CLEANUP_TOTAL_KEY_LIMIT) break
  }