Function getTokenStrings

apps/sim/lib/tokenization/estimators.ts:64–92

(text: string, modelName = 'text-embedding-3-small')

 * Returns an array of token strings that can be displayed with colors
 */
export function getTokenStrings(text: string, modelName = 'text-embedding-3-small'): string[] {
  if (!text || text.length === 0) {
    return []
  }

  try {
    const encoding = getEncoding(modelName)
    const tokenIds = encoding.encode(text)

    const textChars = [...text]
    const result: string[] = []
    let prevCharCount = 0

    for (let i = 0; i < tokenIds.length; i++) {
      const decoded = encoding.decode(tokenIds.slice(0, i + 1))
      const currentCharCount = [...decoded].length
      const tokenCharCount = currentCharCount - prevCharCount

      const tokenStr = textChars.slice(prevCharCount, prevCharCount + tokenCharCount).join('')
      result.push(tokenStr)
      prevCharCount = currentCharCount
    }

    return result
  } catch (error) {
    logger.error('Error getting token strings:', error)
    return text.split(/(\s+)/).filter((s) => s.length > 0)
  }
}

/**
 * Truncate text to a maximum token count