Function formatTranscript

server/src/run_analysis.ts:141–201

(traceEntries: TraceEntry[], score: number)

}

export function formatTranscript(traceEntries: TraceEntry[], score: number): [string, number[]] {
  let actionsAdded = 0
  const includedActionIndices: number[] = []
  let formattedTranscript = ''

  function addAgentAction(agentEntries: Array<TraceEntry & { content: LogEC }>): void {
    if (!agentEntries.length) {
      return
    }
    const combined = agentEntries
      .map(entry => entry.content.content)
      .join('\n')
      .trim()
    includedActionIndices.push(agentEntries[0].index)
    actionsAdded++
    const s = `==AGENT ACTION ${actionsAdded}==\n${combined}\n`
    formattedTranscript += s
  }

  let agentEntries: Array<TraceEntry & { content: LogEC }> = []
  let generationSinceLog = false
  for (const entry of sortBy(traceEntries, 'calledAt')) {
    if (entry.content.type === 'generation') {
      generationSinceLog = true
    } else if (entry.content.type === 'log') {
      let role = generationSinceLog ? 'AGENT ACTION' : 'TOOL OUTPUT'
      generationSinceLog = false

      const content = entry.content.content.join('\n').trim()
      if (!content) {
        continue
      }
      if (content.toLowerCase().startsWith('ui message')) {
        continue
      }
      if (content.startsWith('Task:')) {
        role = 'TASK'
      }

      if (role === 'AGENT ACTION') {
        agentEntries.push(entry as TraceEntry & { content: LogEC })
      } else {
        addAgentAction(agentEntries)

        formattedTranscript += `==${role}==\n`
        formattedTranscript += truncateStep(content)
        formattedTranscript += '\n'

        agentEntries = []
      }
    }
  }

  addAgentAction(agentEntries)

  formattedTranscript += '==RESULTS==\n'
  formattedTranscript += `score: ${score}\n`
  formattedTranscript += '(1 indicates success; 0 indicates failure; intermediate values indicate partial success)\n'