Simulates an LLM with configurable failure behavior. Swap this for your real LLM in production: layer = ControlLayer(llm_fn=your_openai_call, ...)
| 67 | |
| 68 | |
| 69 | class MockLLM: |
| 70 | """ |
| 71 | Simulates an LLM with configurable failure behavior. |
| 72 | Swap this for your real LLM in production: |
| 73 | layer = ControlLayer(llm_fn=your_openai_call, ...) |
| 74 | """ |
| 75 | |
| 76 | def __init__( |
| 77 | self, |
| 78 | failure_rate: float = 0.6, |
| 79 | failure_mode: FailureMode = FailureMode.SCHEMA_VIOLATION, |
| 80 | simulate_latency: bool = True, |
| 81 | ): |
| 82 | self.failure_rate = failure_rate |
| 83 | self.failure_mode = failure_mode |
| 84 | self.simulate_latency = simulate_latency |
| 85 | self._call_count = 0 |
| 86 | |
| 87 | def __call__(self, prompt: str) -> str: |
| 88 | self._call_count += 1 |
| 89 | |
| 90 | if self.simulate_latency: |
| 91 | base = 0.04 if self._call_count == 1 else 0.03 |
| 92 | time.sleep(base + random.uniform(0.005, 0.015)) |
| 93 | |
| 94 | correction_in_prompt = "Correction note:" in prompt |
| 95 | effective_rate = ( |
| 96 | self.failure_rate * 0.35 |
| 97 | if correction_in_prompt |
| 98 | else self.failure_rate |
| 99 | ) |
| 100 | |
| 101 | if random.random() < effective_rate: |
| 102 | return self._bad_response() |
| 103 | return self._good_response(prompt) |
| 104 | |
| 105 | def _good_response(self, prompt: str) -> str: |
| 106 | if "json" in prompt.lower() or "JSON" in prompt: |
| 107 | return random.choice(GOOD_JSON_RESPONSES) |
| 108 | return random.choice(GOOD_TEXT_RESPONSES) |
| 109 | |
| 110 | def _bad_response(self) -> str: |
| 111 | if self.failure_mode == FailureMode.SCHEMA_VIOLATION: |
| 112 | return random.choice(BAD_JSON_RESPONSES) |
| 113 | if self.failure_mode == FailureMode.CONSTRAINT_VIOLATION: |
| 114 | return random.choice(LONG_RESPONSES + FORBIDDEN_RESPONSES) |
| 115 | return "" |
| 116 | |
| 117 | |
| 118 | def make_deterministic_llm(responses: list): |
no outgoing calls
no test coverage detected