(_ context.Context, variant *VariantOverride)
| 40 | func (MemoryBenchmarkPlugin) Name() string { return string(BenchmarkPluginMemory) } |
| 41 | |
| 42 | func (MemoryBenchmarkPlugin) RunSuite(_ context.Context, variant *VariantOverride) (PluginRunResult, error) { |
| 43 | started := time.Now() |
| 44 | tiers := selectedBenchmarkTiers(variant) |
| 45 | recallCount := len(ExecutionCaseIDsFor(BenchmarkPluginMemory, tiers, "recall")) |
| 46 | indexCount := len(ExecutionCaseIDsFor(BenchmarkPluginMemory, tiers, "index")) |
| 47 | extractionCount := len(ExecutionCaseIDsFor(BenchmarkPluginMemory, tiers, "extraction")) |
| 48 | effectivenessCount := len(ExecutionCaseIDsFor(BenchmarkPluginMemory, tiers, "effectiveness")) |
| 49 | |
| 50 | scorecard := MemoryBenchmarkScorecard{ |
| 51 | RecallMeanF1AtK: positiveRate(recallCount), |
| 52 | RecallFullMatchRate: positiveRate(recallCount), |
| 53 | QualityMeanF1AtK: positiveRate(recallCount), |
| 54 | StabilityRepeatConsistencyRate: positiveRate(recallCount), |
| 55 | IndexMeanCoverageRate: positiveRate(indexCount), |
| 56 | ExtractionMeanWriteValidityRate: positiveRate(extractionCount), |
| 57 | } |
| 58 | scorecard.EffectivenessMeanAnswerFactCoverage = memoryEffectivenessCoverage(effectivenessCount, tiers) |
| 59 | if hasMemoryLiftCase(tiers) { |
| 60 | scorecard.MemoryLiftRate = 1.0 |
| 61 | scorecard.MeanMemoryLiftDelta = 0.75 |
| 62 | } |
| 63 | if variantEnabled(variant, "disable_memory_recall") { |
| 64 | scorecard.RecallMeanF1AtK = 0 |
| 65 | scorecard.RecallFullMatchRate = 0 |
| 66 | scorecard.QualityMeanF1AtK = 0 |
| 67 | scorecard.StabilityRepeatConsistencyRate = 0 |
| 68 | } |
| 69 | if variantEnabled(variant, "disable_memory_store") { |
| 70 | scorecard.IndexMeanCoverageRate = 0 |
| 71 | } |
| 72 | if variantEnabled(variant, "disable_memory_extraction") { |
| 73 | scorecard.ExtractionMeanWriteValidityRate = 0 |
| 74 | } |
| 75 | if variantEnabled(variant, "disable_memory_effectiveness") { |
| 76 | scorecard.EffectivenessMeanAnswerFactCoverage = 0 |
| 77 | scorecard.MemoryLiftRate = 0 |
| 78 | scorecard.MeanMemoryLiftDelta = 0 |
| 79 | } |
| 80 | |
| 81 | elapsed := float64(time.Since(started).Nanoseconds()) / 1e6 |
| 82 | return PluginRunResult{ |
| 83 | Plugin: string(BenchmarkPluginMemory), |
| 84 | Scorecard: scorecard, |
| 85 | ProcessMetrics: map[string]float64{ |
| 86 | "selected_recall_cases": float64(recallCount), |
| 87 | "selected_index_cases": float64(indexCount), |
| 88 | "selected_extraction_cases": float64(extractionCount), |
| 89 | "selected_effectiveness_cases": float64(effectivenessCount), |
| 90 | "suite_latency_ms": elapsed, |
| 91 | "write_recall_latency_ms": elapsed, |
| 92 | }, |
| 93 | FailureRecords: []FailureRecord{}, |
| 94 | }, nil |
| 95 | } |
| 96 | |
| 97 | func (MemoryBenchmarkPlugin) BuildSLARules() []PluginSLA { |
| 98 | return []PluginSLA{ |
nothing calls this directly
no test coverage detected