Function parseCSVStream

frontend/src/hooks/stream/useStreamingCSVParser.ts:80–276

(
    fileStream: ReadableStream<Uint8Array>,
    options: {
      chunkSize?: number;
      batchSize?: number; // Number of rows to process before yielding
      delimiter?: string;
      onChunkParsed?: (chunk: string[][], headers: string[], isFirstChunk: boolean) => Promise<void>;
      onProgress?: (progress: number, stats: StreamingCSVParseStats) => void;
      onComplete?: (result: CSVParseResult) => void;
      estimatedTotalBytes?: number;
      sampleRowsLimit?: number;
    } = {}
  )

   * Parse large CSV files in chunks with progress tracking and browser yielding
   */
  const parseCSVStream = async (
    fileStream: ReadableStream<Uint8Array>,
    options: {
      chunkSize?: number;
      batchSize?: number; // Number of rows to process before yielding
      delimiter?: string;
      onChunkParsed?: (chunk: string[][], headers: string[], isFirstChunk: boolean) => Promise<void>;
      onProgress?: (progress: number, stats: StreamingCSVParseStats) => void;
      onComplete?: (result: CSVParseResult) => void;
      estimatedTotalBytes?: number;
      sampleRowsLimit?: number;
    } = {}
  ): Promise<CSVParseResult> => {
    const {
      chunkSize = 50000,
      batchSize = 50000, // Process 5000 rows at a time
      delimiter = ',',
      onChunkParsed,
      onProgress,
      onComplete,
      estimatedTotalBytes,
      sampleRowsLimit = 1000000
    } = options;
    
    try {
      setIsLoading(true);
      setError(null);
      setProgress(0);
      
      // Initialize parser stats
      const stats: StreamingCSVParseStats = {
        bytesProcessed: 0,
        rowsProcessed: 0,
        chunksProcessed: 0,
        totalBytes: estimatedTotalBytes,
        startTime: Date.now()
      };
      
      // Use StreamReader to efficiently process the file in chunks
      const streamReader = new StreamReader(fileStream);
      
      // Variables to accumulate data
      let headers: string[] = [];
      let headersParsed = false;
      let sampleData: string[][] = [];
      let buffer = '';
      let lineCount = 0;
      const textDecoder = new TextDecoder();
      
      // Batch processing variables
      let currentBatch: string[][] = [];
      let lastYieldTime = Date.now();
      
      // Process the stream in chunks
      await streamReader.readByChunks(async (chunk) => {
        // Decode the chunk and add to buffer
        const text = textDecoder.decode(chunk, { stream: true });
        buffer += text;