| 54 | } |
| 55 | |
| 56 | function formatBuffBenchEmailContent( |
| 57 | results: Record<string, AgentEvalResults>, |
| 58 | metadata: any, |
| 59 | metaAnalysis?: MetaAnalysisResult, |
| 60 | ) { |
| 61 | const agents = Object.keys(results) |
| 62 | const date = new Date().toLocaleDateString() |
| 63 | |
| 64 | const agentScores = agents |
| 65 | .map((agentId) => `${agentId}: ${results[agentId].averageScore.toFixed(1)}`) |
| 66 | .join(' | ') |
| 67 | |
| 68 | const subject = `Nightly BuffBench Results - ${date} - ${agentScores}` |
| 69 | |
| 70 | const agentComparison = agents |
| 71 | .map( |
| 72 | (agentId) => |
| 73 | `${agentId}: |
| 74 | - Average Score: ${results[agentId].averageScore.toFixed(2)}/10 |
| 75 | - Average Cost: ${results[agentId].averageCost.toFixed(4)} |
| 76 | - Average Duration: ${(results[agentId].averageDuration / 1000).toFixed(1)}s |
| 77 | - Valid Runs: ${results[agentId].runs.length}`, |
| 78 | ) |
| 79 | .join('\n\n') |
| 80 | |
| 81 | let message = `📊 NIGHTLY BUFFBENCH RESULTS |
| 82 | |
| 83 | 📈 AGENT RESULTS: |
| 84 | ${agentComparison} |
| 85 | |
| 86 | 📁 Results Location: ${metadata.logsDirectory} |
| 87 | ⏱️ Total Evaluation Time: ${(metadata.totalDuration / 1000 / 60).toFixed(1)} minutes |
| 88 | • Total Tasks: ${metadata.commitsEvaluated} |
| 89 | • Agents Tested: ${agents.join(', ')} |
| 90 | |
| 91 | Generated on: ${metadata.timestamp} |
| 92 | Repositories: ${metadata.evalFiles.map((f: { repoUrl: string }) => f.repoUrl).join(', ')}` |
| 93 | |
| 94 | if (metaAnalysis) { |
| 95 | message += ` |
| 96 | |
| 97 | 🔍 META-ANALYSIS |
| 98 | |
| 99 | Overall Comparison: |
| 100 | ${metaAnalysis.overallComparison}` |
| 101 | |
| 102 | if (metaAnalysis.agentInsights.length > 0) { |
| 103 | message += `\n\nAgent-Specific Insights:` |
| 104 | for (const insight of metaAnalysis.agentInsights) { |
| 105 | message += `\n\n[${insight.agentId}]` |
| 106 | if (insight.consistentStrengths.length > 0) { |
| 107 | message += `\n Strengths: ${insight.consistentStrengths.join(', ')}` |
| 108 | } |
| 109 | if (insight.consistentWeaknesses.length > 0) { |
| 110 | message += `\n Weaknesses: ${insight.consistentWeaknesses.join(', ')}` |
| 111 | } |
| 112 | if (insight.recommendations.length > 0) { |
| 113 | message += `\n Recommendations:` |