Function estimateGenericTokens

apps/sim/lib/tokenization/estimators.ts:303–306

* Generic token estimation fallback

(text: string, avgCharsPerToken: number)

 * Generic token estimation fallback
 */
function estimateGenericTokens(text: string, avgCharsPerToken: number): number {
  const charCount = text.trim().length
  return Math.ceil(charCount / avgCharsPerToken)
}

/**
 * Estimates tokens for input content including context