Function parseGoogleChat

packages/parser/src/formats/google-chat-takeout.ts:276–361

(options: ParseOptions)

}

async function* parseGoogleChat(options: ParseOptions): AsyncGenerator<ParseEvent, void, unknown> {
  const { filePath, batchSize = 5000, onProgress, onLog } = options
  const { manifest, userInfoPath, groupInfoPath, messagesPath } = readManifest(filePath)
  const userInfo = readJson<GoogleChatUserInfo>(userInfoPath)
  const groupInfo = readJson<GoogleChatGroupInfo>(groupInfoPath)
  const owner = userInfo.user ? normalizeIdentity(userInfo.user) : null
  const totalBytes = getFileSize(messagesPath)

  const initialProgress = createProgress('parsing', 0, totalBytes, 0, '')
  yield { type: 'progress', data: initialProgress }
  onProgress?.(initialProgress)

  yield {
    type: 'meta',
    data: {
      name: deriveChatName(manifest, groupInfo, owner?.platformId),
      platform: KNOWN_PLATFORMS.GOOGLE_CHAT,
      type: manifest.chatType === 'group' ? ChatType.GROUP : ChatType.PRIVATE,
      ...(manifest.chatType === 'group' ? { groupId: manifest.chatId } : {}),
      ownerId: owner?.platformId,
    },
  }

  const memberMap = new Map<string, ParsedMember>()
  for (const member of groupInfo.members ?? []) {
    const identity = normalizeIdentity(member)
    memberMap.set(identity.platformId, {
      platformId: identity.platformId,
      accountName: identity.name,
    })
  }
  if (owner && !memberMap.has(owner.platformId)) {
    memberMap.set(owner.platformId, {
      platformId: owner.platformId,
      accountName: owner.name,
    })
  }
  yield { type: 'members', data: Array.from(memberMap.values()) }

  const readStream = fs.createReadStream(messagesPath, { encoding: 'utf8' })
  let bytesRead = 0
  let messagesProcessed = 0
  readStream.on('data', (chunk: string | Buffer) => {
    bytesRead += typeof chunk === 'string' ? Buffer.byteLength(chunk) : chunk.length
  })

  const pipeline = chain([readStream, parser(), pick({ filter: /^messages\.\d+$/ }), streamValues()])
  const messageBatch: ParsedMessage[] = []

  try {
    for await (const item of pipeline as AsyncIterable<{ value: GoogleChatMessage }>) {
      const message = item.value
      const sender = normalizeIdentity(message.creator)
      messageBatch.push({
        platformMessageId: message.message_id ? String(message.message_id) : undefined,
        senderPlatformId: sender.platformId,
        senderAccountName: sender.name,
        timestamp: parseGoogleChatDate(message.created_date ?? message.updated_date ?? '') ?? Number.NaN,