(metrics: Metric[], content: string)
| 40 | } |
| 41 | |
| 42 | export const generateEvaluatorPrompt = (metrics: Metric[], content: string): string => { |
| 43 | // Filter out invalid/incomplete metrics first |
| 44 | const validMetrics = metrics.filter((m) => m?.name && m.range) |
| 45 | |
| 46 | // Create a clear metrics description with name, range, and description |
| 47 | const metricsDescription = validMetrics |
| 48 | .map( |
| 49 | (metric) => |
| 50 | `"${metric.name}" (${metric.range.min}-${metric.range.max}): ${metric.description || ''}` // Handle potentially missing description |
| 51 | ) |
| 52 | .join('\n') |
| 53 | |
| 54 | // Format the content properly - try to detect and format JSON |
| 55 | let formattedContent = content |
| 56 | try { |
| 57 | // If content looks like JSON (starts with { or [) |
| 58 | if ( |
| 59 | typeof content === 'string' && |
| 60 | (content.trim().startsWith('{') || content.trim().startsWith('[')) |
| 61 | ) { |
| 62 | // Try to parse and pretty-print |
| 63 | const parsedContent = JSON.parse(content) |
| 64 | formattedContent = JSON.stringify(parsedContent, null, 2) |
| 65 | } |
| 66 | // If it's already an object (shouldn't happen here but just in case) |
| 67 | else if (typeof content === 'object') { |
| 68 | formattedContent = JSON.stringify(content, null, 2) |
| 69 | } |
| 70 | } catch (e) { |
| 71 | logger.warn('Warning: Content may not be valid JSON, using as-is', { e }) |
| 72 | formattedContent = content |
| 73 | } |
| 74 | |
| 75 | // Generate an example of the expected output format using only valid metrics |
| 76 | const exampleOutput = validMetrics.reduce( |
| 77 | (acc, metric) => { |
| 78 | // Ensure metric and name are valid before using them |
| 79 | if (metric?.name) { |
| 80 | acc[metric.name.toLowerCase()] = Math.floor((metric.range.min + metric.range.max) / 2) // Use middle of range as example |
| 81 | } else { |
| 82 | logger.warn('Skipping invalid metric during example generation:', metric) |
| 83 | } |
| 84 | return acc |
| 85 | }, |
| 86 | {} as Record<string, number> |
| 87 | ) |
| 88 | |
| 89 | return `You are an objective evaluation agent. Analyze the content against the provided metrics and provide detailed scoring. |
| 90 | |
| 91 | Evaluation Instructions: |
| 92 | - You MUST evaluate the content against each metric |
| 93 | - For each metric, provide a numeric score within the specified range |
| 94 | - Your response MUST be a valid JSON object with each metric name as a key and a numeric score as the value |
| 95 | - IMPORTANT: Use lowercase versions of the metric names as keys in your JSON response |
| 96 | - Follow the exact schema of the response format provided to you |
| 97 | - Do not include explanations in the JSON - only numeric scores |
| 98 | - Do not add any additional fields not specified in the schema |
| 99 | - Do not include ANY text before or after the JSON object |
no test coverage detected