Function handleExternalUrl

apps/sim/app/api/files/parse/route.ts:480–601

* Handle external URL. * * Always fetches the URL fresh — there is no filename-based dedup. Distinct URLs * commonly share a path tail (e.g. every Slack clipboard paste is `image.png`), * so keying a cache by filename returns stale bytes. `fetchExternalUrlToWorkspace` * delegates to `uploadWork

(
  url: string,
  fileType: string,
  workspaceId: string,
  userId: string,
  executionContext?: ExecutionContext,
  headers?: Record<string, string>,
  signal?: AbortSignal,
  maxDownloadBytes = MAX_DOWNLOAD_SIZE_BYTES,
  maxParsedOutputBytes?: number
)

 * style aliases for files we already own).
 */
async function handleExternalUrl(
  url: string,
  fileType: string,
  workspaceId: string,
  userId: string,
  executionContext?: ExecutionContext,
  headers?: Record<string, string>,
  signal?: AbortSignal,
  maxDownloadBytes = MAX_DOWNLOAD_SIZE_BYTES,
  maxParsedOutputBytes?: number
): Promise<ParseResult> {
  try {
    logger.info('Fetching external URL:', url)

    const {
      S3_EXECUTION_FILES_CONFIG,
      BLOB_EXECUTION_FILES_CONFIG,
      USE_S3_STORAGE,
      USE_BLOB_STORAGE,
    } = await import('@/lib/uploads/config')

    let isExecutionFile = false
    try {
      const parsedUrl = new URL(url)

      if (USE_S3_STORAGE && S3_EXECUTION_FILES_CONFIG.bucket) {
        const bucketInHost = parsedUrl.hostname.startsWith(S3_EXECUTION_FILES_CONFIG.bucket)
        const bucketInPath = parsedUrl.pathname.startsWith(`/${S3_EXECUTION_FILES_CONFIG.bucket}/`)
        isExecutionFile = bucketInHost || bucketInPath
      } else if (USE_BLOB_STORAGE && BLOB_EXECUTION_FILES_CONFIG.containerName) {
        isExecutionFile = url.includes(`/${BLOB_EXECUTION_FILES_CONFIG.containerName}/`)
      }
    } catch (error) {
      logger.warn('Failed to parse URL for execution file check:', error)
      isExecutionFile = false
    }

    const { filename, buffer, mimeType } = await fetchExternalUrlToWorkspace({
      url,
      userId,
      workspaceId: workspaceId || undefined,
      saveToWorkspace: Boolean(workspaceId) && !isExecutionFile,
      headers,
      signal,
      maxDownloadBytes,
      timeoutMs: DOWNLOAD_TIMEOUT_MS,
    })
    const extension = path.extname(filename).toLowerCase().substring(1)

    logger.info(`Downloaded file from URL: ${url}, size: ${buffer.length} bytes`)

    let userFile: UserFile | undefined
    if (executionContext) {
      try {
        userFile = await uploadExecutionFile(executionContext, buffer, filename, mimeType, userId)
        logger.info(`Stored file in execution storage: ${filename}`, { key: userFile.key })
      } catch (uploadError) {
        logger.warn('Failed to store file in execution storage:', uploadError)