Function processStreamingBlockLog

apps/sim/lib/tokenization/streaming.ts:23–99

(log: BlockLog, streamedContent: string)

 * Processes a block log and adds tokenization data if needed
 */
export function processStreamingBlockLog(log: BlockLog, streamedContent: string): boolean {
  // Check if this block should be tokenized
  if (!isTokenizableBlockType(log.blockType)) {
    return false
  }

  // Check if we already have meaningful token/cost data
  if (hasRealTokenData(log.output?.tokens) && hasRealCostData(log.output?.cost)) {
    return false
  }

  // Skip recalculation if cost was explicitly set by the billing layer (e.g. BYOK zero cost)
  if (log.output?.cost?.pricing) {
    return false
  }

  // Check if we have content to tokenize
  if (!streamedContent?.trim()) {
    return false
  }

  try {
    // Determine model to use
    const model = getModelForBlock(log)

    // Prepare input text from log
    const inputText = extractTextContent(log.input)

    // Calculate streaming cost
    const systemPrompt =
      typeof log.input?.systemPrompt === 'string' ? log.input.systemPrompt : undefined
    const context = typeof log.input?.context === 'string' ? log.input.context : undefined
    const messages = Array.isArray(log.input?.messages)
      ? (log.input.messages as Array<{ role: string; content: string }>)
      : undefined
    const result = calculateStreamingCost(
      model,
      inputText,
      streamedContent,
      systemPrompt,
      context,
      messages
    )

    // Update the log output with tokenization data
    if (!log.output) {
      log.output = {}
    }

    log.output.tokens = result.tokens
    log.output.cost = result.cost
    log.output.model = result.model

    logTokenizationDetails(`Streaming tokenization completed for ${log.blockType}`, {
      blockId: log.blockId,
      blockType: log.blockType,
      model: result.model,
      provider: result.provider,