Function countTokens

src/utils/tokenizer.ts:92–106

(text: string | null | undefined)

 * heuristic. Never throws.
 */
export function countTokens(text: string | null | undefined): number {
  if (!text) return 0;
  if (encoder) {
    try {
      return encoder(text).length;
    } catch (err) {
      if (!warnedEncoderFailed) {
        warnedEncoderFailed = true;
        logger.debug('tokenizer encoder failed; falling back to heuristic', { err });
      }
      /* fall through */
    }
  }
  return heuristicTokens(text);
}

/** Count tokens for an arbitrary JSON-serializable value (tool args, etc.). */
export function countTokensJson(value: unknown): number {