| 299 | } |
| 300 | |
| 301 | function getQueryPrompt( |
| 302 | query: string, |
| 303 | traceEntrySummaries: JoinedTraceEntrySummary[], |
| 304 | ): [string, Record<string, StepLookupEntry>] { |
| 305 | let userPrompt = `Query: ${query}\n\nHere are the steps taken by the agents:` |
| 306 | |
| 307 | const groupedSummaries: Record<string, JoinedTraceEntrySummary[]> = {} |
| 308 | for (const summary of traceEntrySummaries) { |
| 309 | ;(groupedSummaries[summary.runId] ??= []).push(summary) |
| 310 | } |
| 311 | |
| 312 | const stepLookup: Record<string, StepLookupEntry> = {} |
| 313 | |
| 314 | for (const [runId, summaries] of Object.entries(groupedSummaries)) { |
| 315 | userPrompt += `\n\nRUN ID: r${runId}` |
| 316 | |
| 317 | summaries.forEach((step, stepIdx) => { |
| 318 | const stepId = `r${runId}s${stepIdx}` |
| 319 | userPrompt += `\n\nSTEP ID: ${stepId}` |
| 320 | userPrompt += `\n${step.summary}` |
| 321 | stepLookup[stepId] = { |
| 322 | runId: Number(runId) as RunId, |
| 323 | taskId: TaskId.parse(step.taskId), |
| 324 | index: step.index, |
| 325 | content: step.content?.content?.join('\n') ?? '', |
| 326 | } |
| 327 | }) |
| 328 | } |
| 329 | userPrompt += '\n\nPlease list the IDs of the best matches.' |
| 330 | |
| 331 | return [userPrompt, stepLookup] |
| 332 | } |
| 333 | |
| 334 | export async function analyzeRuns( |
| 335 | runIds: RunId[], |