Stops hammering a failing LLM backend. States: CLOSED (normal) → OPEN (failing) → HALF_OPEN (testing) → CLOSED Production fix over v1: without this, a down LLM causes every request to retry max_attempts times, saturating thread pools and exploding latency across all concurrent
| 523 | |
| 524 | |
| 525 | class CircuitBreaker: |
| 526 | """ |
| 527 | Stops hammering a failing LLM backend. |
| 528 | |
| 529 | States: CLOSED (normal) → OPEN (failing) → HALF_OPEN (testing) → CLOSED |
| 530 | |
| 531 | Production fix over v1: without this, a down LLM causes every |
| 532 | request to retry max_attempts times, saturating thread pools |
| 533 | and exploding latency across all concurrent users. |
| 534 | """ |
| 535 | |
| 536 | def __init__(self, failure_threshold: int = 5, recovery_seconds: float = 30.0): |
| 537 | self.failure_threshold = failure_threshold |
| 538 | self.recovery_seconds = recovery_seconds |
| 539 | self._failures = 0 |
| 540 | self._last_failure_time: Optional[float] = None |
| 541 | self._state = CircuitState.CLOSED |
| 542 | self._lock = threading.Lock() |
| 543 | |
| 544 | @property |
| 545 | def state(self) -> CircuitState: |
| 546 | with self._lock: |
| 547 | if self._state == CircuitState.OPEN: |
| 548 | elapsed = time.monotonic() - (self._last_failure_time or 0) |
| 549 | if elapsed >= self.recovery_seconds: |
| 550 | self._state = CircuitState.HALF_OPEN |
| 551 | log.info("circuit_breaker.half_open", recovery_seconds=self.recovery_seconds) |
| 552 | return self._state |
| 553 | |
| 554 | def is_open(self) -> bool: |
| 555 | return self.state == CircuitState.OPEN |
| 556 | |
| 557 | def record_success(self) -> None: |
| 558 | with self._lock: |
| 559 | self._failures = 0 |
| 560 | self._state = CircuitState.CLOSED |
| 561 | |
| 562 | def record_failure(self) -> None: |
| 563 | with self._lock: |
| 564 | self._failures += 1 |
| 565 | self._last_failure_time = time.monotonic() |
| 566 | if self._failures >= self.failure_threshold: |
| 567 | if self._state != CircuitState.OPEN: |
| 568 | log.warning( |
| 569 | "circuit_breaker.open", |
| 570 | failures=self._failures, |
| 571 | threshold=self.failure_threshold, |
| 572 | ) |
| 573 | self._state = CircuitState.OPEN |
| 574 | |
| 575 | def reset(self) -> None: |
| 576 | with self._lock: |
| 577 | self._failures = 0 |
| 578 | self._state = CircuitState.CLOSED |
| 579 | self._last_failure_time = None |
| 580 | |
| 581 | |
| 582 | # ============================================================================= |
no outgoing calls