Function parseChatLabJsonl

packages/parser/src/formats/chatlab-jsonl.ts:124–278

(options: ParseOptions)

// ==================== 解析器实现 ====================

async function* parseChatLabJsonl(options: ParseOptions): AsyncGenerator<ParseEvent, void, unknown> {
  const { filePath, batchSize = 5000, onProgress, onLog } = options

  const totalBytes = getFileSize(filePath)
  let bytesRead = 0
  let messagesProcessed = 0

  // 发送初始进度
  const initialProgress = createProgress('parsing', 0, totalBytes, 0, '')
  yield { type: 'progress', data: initialProgress }
  onProgress?.(initialProgress)

  // 记录解析开始
  onLog?.('info', `开始解析 ChatLab JSONL 文件，大小: ${(totalBytes / 1024 / 1024).toFixed(2)} MB`)

  // 用于收集成员和消息
  const members: ParsedMember[] = []
  const memberMap = new Map<string, ParsedMember>()
  const messageBatch: ParsedMessage[] = []
  let meta: ParsedMeta | null = null
  let headerParsed = false

  // 创建逐行读取流
  const fileStream = fs.createReadStream(filePath, { encoding: 'utf-8' })
  const rl = readline.createInterface({
    input: fileStream,
    crlfDelay: Infinity,
  })

  // 跟踪已读取字节数
  fileStream.on('data', (chunk: string | Buffer) => {
    bytesRead += typeof chunk === 'string' ? Buffer.byteLength(chunk) : chunk.length
  })

  // 逐行解析
  for await (const line of rl) {
    const parsed = parseLine(line)
    if (!parsed) continue

    switch (parsed._type) {
      case 'header':
        if (!headerParsed) {
          meta = {
            name: parsed.meta.name || extractNameFromFilePath(filePath),
            platform: parsed.meta.platform || KNOWN_PLATFORMS.UNKNOWN,
            type: (parsed.meta.type as ChatType) || ChatType.GROUP,
            groupId: parsed.meta.groupId,
            groupAvatar: parsed.meta.groupAvatar,
          }
          headerParsed = true
          yield { type: 'meta', data: meta }
        }
        break

      case 'member':
        {
          const member: ParsedMember = {
            platformId: parsed.platformId,