Function handleOpenAIStream

web/src/llm-api/openai.ts:376–703

({
  body,
  userId,
  stripeCustomerId,
  agentId,
  fetch,
  logger,
  insertMessageBigquery,
}: {
  body: ChatCompletionRequestBody
  userId: string
  stripeCustomerId?: string | null
  agentId: string
  fetch: typeof globalThis.fetch
  logger: Logger
  insertMessageBigquery: InsertMessageBigqueryFn
})

}

export async function handleOpenAIStream({
  body,
  userId,
  stripeCustomerId,
  agentId,
  fetch,
  logger,
  insertMessageBigquery,
}: {
  body: ChatCompletionRequestBody
  userId: string
  stripeCustomerId?: string | null
  agentId: string
  fetch: typeof globalThis.fetch
  logger: Logger
  insertMessageBigquery: InsertMessageBigqueryFn
}) {
  const startTime = new Date()
  const { clientId, clientRequestId, costMode } = extractRequestMetadata({
    body,
    logger,
  })
  const auditRequest = createRequestAuditRecord(body)

  const originalModel = body.model
  const modelShortName = extractShortModelName(originalModel)
  const openaiBody = buildOpenAIBody(body, modelShortName)
  openaiBody.stream = true
  openaiBody.stream_options = { include_usage: true }

  const response = await fetch('https://api.openai.com/v1/chat/completions', {
    method: 'POST',
    headers: {
      Authorization: `Bearer ${env.OPENAI_API_KEY}`,
      'Content-Type': 'application/json',
    },
    body: JSON.stringify(openaiBody),
    // @ts-expect-error - dispatcher is a valid undici option not in fetch types
    dispatcher: openaiAgent,
  })

  if (!response.ok) {
    throw new OpenAIError(
      response.status,
      response.statusText,
      await response.text(),
    )
  }

  const reader = response.body?.getReader()
  if (!reader) {
    throw new Error('Failed to get response reader')
  }

  let heartbeatInterval: NodeJS.Timeout
  let responseText = ''
  let reasoningText = ''
  let ttftMs: number | null = null