Function processFileInBackground

server/api/import/index.post.ts:416–526

(fileId: string, userId: string)

}

async function processFileInBackground(fileId: string, userId: string) {
  assertSafeFileId(fileId);
  const job = activeJobs.get(fileId);
  if (!job || job.fileId !== fileId) {
    throw handleApiError(
      404,
      `Job not found for file ID ${fileId}`,
      "Import job not found.",
    );
  }

  updateJob(job, {
    status: ImportStatus.Processing,
  });

  const userTempDir = path.join(tmpdir(), "ziit-chunks", userId);
  const chunksDir = path.join(userTempDir, fileId);
  const combinedFilePath = path.join(userTempDir, `${fileId}-combined.json`);
  assertWithinUserDir(userTempDir, chunksDir);
  assertWithinUserDir(userTempDir, combinedFilePath);

  try {
    handleLog(
      `Starting background processing for user ${userId}, fileId ${fileId}`,
    );

    if (!existsSync(chunksDir)) {
      throw new Error(`Chunks directory does not exist: ${chunksDir}`);
    }

    const chunkFiles = fs
      .readdirSync(chunksDir)
      .filter((file) => file.startsWith("chunk-"))
      .sort((a, b) => {
        const indexA = parseInt(a.split("-")[1] ?? "0");
        const indexB = parseInt(b.split("-")[1] ?? "0");
        return indexA - indexB;
      });

    let combinedContent = Buffer.alloc(0);
    for (const chunkFile of chunkFiles) {
      const chunkPath = path.join(chunksDir, chunkFile);
      if (!existsSync(chunkPath)) {
        throw new Error(`Chunk file not found: ${chunkPath}`);
      }
      const chunkData = fs.readFileSync(chunkPath);
      combinedContent = Buffer.concat([combinedContent, chunkData]);
    }
    fs.writeFileSync(combinedFilePath, combinedContent);
    handleLog(
      `Combined ${chunkFiles.length} chunks for file ${fileId} (${(
        combinedContent.length /
        (1024 * 1024)
      ).toFixed(2)} MB).`,
    );

    const fileContent = new TextDecoder().decode(combinedContent);
    const parsedData = JSON.parse(fileContent);