Class CsvParser

apps/sim/lib/file-parsers/csv-parser.ts:17–154

}

export class CsvParser implements FileParser {
  async parseFile(filePath: string): Promise<FileParseResult> {
    if (!filePath) {
      throw new Error('No file path provided')
    }

    if (!existsSync(filePath)) {
      throw new Error(`File not found: ${filePath}`)
    }

    const stream = createReadStream(filePath, {
      highWaterMark: CONFIG.STREAM_CHUNK_SIZE,
    })

    return this.parseStream(stream)
  }

  async parseBuffer(buffer: Buffer): Promise<FileParseResult> {
    const bufferSize = buffer.length
    logger.info(
      `Parsing CSV buffer, size: ${bufferSize} bytes (${(bufferSize / 1024 / 1024).toFixed(2)} MB)`
    )

    const stream = new Readable({ read() {} })
    stream.push(buffer)
    stream.push(null)

    return this.parseStream(stream)
  }

  private parseStream(inputStream: NodeJS.ReadableStream): Promise<FileParseResult> {
    return new Promise((resolve, reject) => {
      let rowCount = 0
      let errorCount = 0
      let headers: string[] = []
      let processedContent = ''
      const sampledRows: any[] = []
      const errors: string[] = []
      let firstRowProcessed = false
      let aborted = false

      const parserOptions: Options = {
        columns: true, // Use first row as headers
        skip_empty_lines: true, // Skip empty lines
        trim: true, // Trim whitespace
        relax_column_count: true, // Allow variable column counts
        relax_quotes: true, // Be lenient with quotes
        skip_records_with_error: true, // Skip bad records
        raw: false,
        cast: false,
      }
      const parser = parse(parserOptions)

      parser.on('readable', () => {
        let record
        while ((record = parser.read()) !== null && !aborted) {
          rowCount++