Function demo_5_benchmark

demo.py:316–415

()

# =============================================================================

def demo_5_benchmark():
    print("\n" + "=" * 60)
    print("Demo 5: Benchmark - Naive vs Control Layer")
    print("=" * 60)

    schema = ResponseSchema(
        must_be_json=True,
        required_keys=["summary", "confidence"],
        max_length=400,
        forbidden_phrases=["I cannot", "As an AI"],
    )
    config = ControlLayerConfig(
        total_tokens=800,
        max_attempts=3,
        base_delay_ms=50,
        jitter_ms=10,
    )

    queries = [
        "Explain the token budget mechanism.",
        "What is prompt mutation?",
        "How does the retry engine work?",
        "Describe the fallback router.",
        "What does the response validator check?",
        "How is injection detected?",
        "What is exponential decay in memory?",
        "How does re-ranking work in RAG?",
        "What is extractive compression?",
        "How is quality score computed?",
    ]

    # Naive baseline
    naive_llm = MockLLM(failure_rate=0.55, failure_mode=FailureMode.SCHEMA_VIOLATION)
    naive_pass = 0
    naive_latencies = []

    print("  Running naive baseline (no control layer)...")
    for q in queries:
        t0 = time.perf_counter()
        response = naive_llm(q)
        latency = (time.perf_counter() - t0) * 1000
        naive_latencies.append(latency)
        try:
            cleaned = response.replace("```json", "").replace("```", "").strip()
            data = json.loads(cleaned)
            if "summary" in data and "confidence" in data and len(response) <= 400:
                naive_pass += 1
        except Exception:
            pass

    # Control Layer
    cl_llm = MockLLM(failure_rate=0.55, failure_mode=FailureMode.SCHEMA_VIOLATION)
    cl_layer = ControlLayer(
        llm_fn=cl_llm,
        system_prompt="You are a structured assistant. Always return JSON.",
        schema=schema,
        config=config,
    )