Method readLines

frontend/src/lib/streamReader.ts:103–181

* Process a text stream line by line (for CSV, etc.) * This handles cases where lines span multiple chunks.

(
      callback: (line: string, lineNumber: number) => Promise<void> | void,
      options: {
        skipFirst?: boolean; // Skip first line (e.g., for headers)
        maxLines?: number; // Maximum number of lines to process (for sampling)
      } = {}
    )

     * This handles cases where lines span multiple chunks.
     */
    async readLines(
      callback: (line: string, lineNumber: number) => Promise<void> | void,
      options: {
        skipFirst?: boolean; // Skip first line (e.g., for headers)
        maxLines?: number; // Maximum number of lines to process (for sampling)
      } = {}
    ): Promise<{ lineCount: number; bytesProcessed: number }> {
      const { skipFirst = false, maxLines = Infinity } = options;
      const textDecoder = new TextDecoder();
      let buffer = '';
      let lineCount = 0;
      let processedLines = 0;
      
      try {
        while (!this.aborted) {
          const { done, value } = await this.reader.read();
          
          if (done) {
            // Process any remaining data in buffer
            if (buffer.length > 0) {
              lineCount++;
              if (!skipFirst || lineCount > 1) {
                processedLines++;
                if (processedLines <= maxLines) {
                  await callback(buffer, lineCount);
                }
              }
            }
            break;
          }
          
          this.bytesProcessed += value.length;
          
          // Decode chunk and add to buffer
          const text = textDecoder.decode(value, { stream: true });
          buffer += text;
          
          // Process complete lines
          let lineEndIndex;
          while ((lineEndIndex = buffer.indexOf('\n')) !== -1 && processedLines < maxLines) {
            const line = buffer.substring(0, lineEndIndex).trim();
            buffer = buffer.substring(lineEndIndex + 1);
            
            if (line.length > 0) {
              lineCount++;
              if (!skipFirst || lineCount > 1) {
                processedLines++;
                if (processedLines <= maxLines) {
                  await callback(line, lineCount);
                }
              }
            }
            
            if (processedLines >= maxLines) {
              break;
            }
          }