MCPcopy Create free account
hub / github.com/colbymchenry/codegraph / evaluate

Function evaluate

scripts/agent-eval/probe-allocation.mjs:129–223  ·  view source on GitHub ↗

Evaluate one fixture's assertions against its report. Returns check rows.

(fixture, report, text)

Source from the content-addressed store, hash-verified

127
128/** Evaluate one fixture's assertions against its report. Returns check rows. */
129function evaluate(fixture, report, text) {
130 const { groups, assert: want } = fixture;
131 const delivered = new Map();
132 const allocated = new Map();
133 for (const f of report.files) {
134 const g = groupOf(f.path, groups);
135 delivered.set(g, (delivered.get(g) ?? 0) + f.share);
136 allocated.set(g, (allocated.get(g) ?? 0) + f.allocatedShare);
137 }
138 const share = (g) => delivered.get(g) ?? 0;
139 const top = report.files
140 .filter((f) => f.finalChars > 0)
141 .sort((a, b) => b.finalChars - a.finalChars)[0];
142
143 const checks = [];
144 const add = (name, pass, detail) => checks.push({ name, pass, detail });
145
146 if (want.answerShareAtLeast !== undefined) {
147 add(
148 `answer group takes >= ${pct(want.answerShareAtLeast)} of the envelope`,
149 share('answer') >= want.answerShareAtLeast,
150 `answer ${pct(share('answer'))} delivered (${pct(allocated.get('answer') ?? 0)} allocated)`,
151 );
152 }
153 // Same question against the SOURCE the response delivered rather than the
154 // whole envelope (CG-26). The envelope-denominated gate above moves whenever
155 // the response's prose does — the epilogue surviving instead of being
156 // discarded costs it a point, and every additional admitted file that gets
157 // paid dilutes it further — so it cannot tell "the answer was starved" from
158 // "everything else was also delivered". Allocation is about source bytes;
159 // measure it in source bytes.
160 if (want.answerShareOfSourceAtLeast !== undefined) {
161 const sourceBy = new Map();
162 let totalSource = 0;
163 for (const f of report.files) {
164 const g = groupOf(f.path, groups);
165 sourceBy.set(g, (sourceBy.get(g) ?? 0) + f.finalChars);
166 totalSource += f.finalChars;
167 }
168 const answerSource = totalSource > 0 ? (sourceBy.get('answer') ?? 0) / totalSource : 0;
169 add(
170 `answer group takes >= ${pct(want.answerShareOfSourceAtLeast)} of DELIVERED SOURCE`,
171 answerSource >= want.answerShareOfSourceAtLeast,
172 `answer ${num(sourceBy.get('answer') ?? 0)} of ${num(totalSource)} source chars (${pct(answerSource)})`,
173 );
174 }
175 if (want.incidentalShareAtMost !== undefined) {
176 add(
177 `incidental group takes <= ${pct(want.incidentalShareAtMost)} of the envelope`,
178 share('incidental') <= want.incidentalShareAtMost,
179 `incidental ${pct(share('incidental'))} delivered (${pct(allocated.get('incidental') ?? 0)} allocated)`,
180 );
181 }
182 if (want.topFileGroup) {
183 const actual = top ? groupOf(top.path, groups) : '(nothing delivered)';
184 add(
185 `largest delivered file is in "${want.topFileGroup}"`,
186 actual === want.topFileGroup,

Callers 1

mainFunction · 0.85

Calls 7

groupOfFunction · 0.85
addFunction · 0.70
pctFunction · 0.70
shareFunction · 0.70
numFunction · 0.70
getMethod · 0.65
setMethod · 0.45

Tested by

no test coverage detected