Function crawlExistingThread

apps/web/services/discord/threads.ts:168–247

({
  thread,
  onboardingTimestamp,
  token,
  lastExternalMessageId,
  logger,
}: {
  thread: threads;
  lastExternalMessageId?: string;
  onboardingTimestamp: Date;
  token: string;
  logger: Logger;
})

}

async function crawlExistingThread({
  thread,
  onboardingTimestamp,
  token,
  lastExternalMessageId,
  logger,
}: {
  thread: threads;
  lastExternalMessageId?: string;
  onboardingTimestamp: Date;
  token: string;
  logger: Logger;
}) {
  const messagesInThread: DiscordMessage[] = [];

  const threadId = thread.externalThreadId;
  if (!threadId) {
    logger.error({
      'crawlExistingThread finished': 'missing externalThreadId',
    });
    return [];
  }

  let hasMore = true;
  // before will have the last messageId from request to be used on next pagination request
  let before;
  // cursor/after should be the first messageId receive from the last run
  let after = lastExternalMessageId;
  while (hasMore) {
    const query = {
      // before should have priority because the API always return messages sort by timestamp desc
      // so in the second run we will have a cursor assign, we should use it to get messages from the cursorId forward
      // but if there is more than the limit, we will need to paginate backwards, so in case of hasMore
      // we will need to use the "before" parameter, that has the oldest message from the latest batch
      ...(before ? { before } : { after }),
    };
    // if query has after, it means we should clean up the after variable to receive a new cursor
    if ('after' in query) {
      after = undefined;
    }

    // messages are return in desc timestamp order
    const [err, response] = await to(
      DiscordApi.getMessages({
        limit: LIMIT,
        externalId: threadId,
        query,
        token,
      })
    );
    if (err) {
      logger.error({ crawlExistingThread: err });
      return [];
    }
    const messages = response as DiscordMessage[];
    // if there is less than the limit, means that there is no more messages
    if (messages.length < LIMIT) {
      hasMore = false;