Function tokenizeWithMockModel

src/node/services/mock/mockAiStreamPlayer.ts:44–109

(text: string, context: string)

}

async function tokenizeWithMockModel(text: string, context: string): Promise<number> {
  assert(typeof text === "string", `Mock stream ${context} expects string input`);

  // Prefer fast approximate token counting in mock mode.
  // We only use the real tokenizer if it's available and responds quickly.
  const approximateTokens = approximateTokenCount(text);

  let fallbackUsed = false;
  let timeoutId: ReturnType<typeof setTimeout> | undefined;
  let tokenizerErrorMessage: string | undefined;

  const fallbackPromise = new Promise<number>((resolve) => {
    timeoutId = setTimeout(() => {
      fallbackUsed = true;
      resolve(approximateTokens);
    }, TOKENIZE_TIMEOUT_MS);
  });

  const actualPromise = (async () => {
    try {
      const tokenizer = await getTokenizerForModel(MOCK_TOKENIZER_MODEL);
      assert(
        typeof tokenizer.encoding === "string" && tokenizer.encoding.length > 0,
        `Tokenizer for ${MOCK_TOKENIZER_MODEL} must expose a non-empty encoding`
      );
      const tokens = await tokenizer.countTokens(text);
      assert(
        Number.isFinite(tokens) && tokens >= 0,
        `Tokenizer for ${MOCK_TOKENIZER_MODEL} returned invalid token count`
      );
      return tokens;
    } catch (error) {
      tokenizerErrorMessage = getErrorMessage(error);
      return approximateTokens;
    }
  })();

  const tokens = await Promise.race([actualPromise, fallbackPromise]);

  if (timeoutId !== undefined) {
    clearTimeout(timeoutId);
  }

  if (fallbackUsed && !tokenizerFallbackLogged) {
    tokenizerFallbackLogged = true;
    void actualPromise.then((resolvedTokens) => {
      log.debug(
        `[MockAiStreamPlayer] Tokenizer fallback used for ${context}; emitted ${approximateTokens}, background tokenizer returned ${resolvedTokens}`
      );
    });
  }

  if (tokenizerErrorMessage && !tokenizerUnavailableLogged) {
    tokenizerUnavailableLogged = true;
    log.debug(
      `[MockAiStreamPlayer] Tokenizer unavailable for ${context}; using approximate (${tokenizerErrorMessage})`
    );
  }