Track Signal B's shadow performance and decide when to promote it.
| 55 | |
| 56 | |
| 57 | class ShadowTracker: |
| 58 | """Track Signal B's shadow performance and decide when to promote it.""" |
| 59 | |
| 60 | def __init__( |
| 61 | self, |
| 62 | eval_window: int = 200, |
| 63 | promote_after: int = 3, |
| 64 | min_improvement: float = 0.005, |
| 65 | ): |
| 66 | self._eval_window = eval_window |
| 67 | self._promote_after = promote_after |
| 68 | self._min_improvement = min_improvement |
| 69 | self._records: list[ShadowRecord] = [] |
| 70 | self._pending: list[ShadowRecord] = [] # accumulates until eval_window |
| 71 | self._consecutive_wins = 0 |
| 72 | self._promoted = False |
| 73 | |
| 74 | @property |
| 75 | def promoted(self) -> bool: |
| 76 | return self._promoted |
| 77 | |
| 78 | @property |
| 79 | def record_count(self) -> int: |
| 80 | return len(self._pending) |
| 81 | |
| 82 | @property |
| 83 | def consecutive_wins(self) -> int: |
| 84 | return self._consecutive_wins |
| 85 | |
| 86 | def record( |
| 87 | self, |
| 88 | signal_a_pred: int | None, signal_a_conf: float, |
| 89 | signal_b_pred: int | None, signal_b_conf: float, |
| 90 | signal_c_pred: int | None, signal_c_conf: float, |
| 91 | ensemble_2sig_tier: int, |
| 92 | gold_tier: int | None = None, |
| 93 | ) -> None: |
| 94 | self._pending.append(ShadowRecord( |
| 95 | signal_a_pred=signal_a_pred, signal_a_conf=signal_a_conf, |
| 96 | signal_b_pred=signal_b_pred, signal_b_conf=signal_b_conf, |
| 97 | signal_c_pred=signal_c_pred, signal_c_conf=signal_c_conf, |
| 98 | ensemble_2sig_tier=ensemble_2sig_tier, |
| 99 | gold_tier=gold_tier, |
| 100 | )) |
| 101 | # Non-overlapping batches: evaluate only when we hit exactly eval_window |
| 102 | if len(self._pending) >= self._eval_window: |
| 103 | self._evaluate_window() |
| 104 | |
| 105 | def _evaluate_window(self) -> None: |
| 106 | """Replay 3-signal ensemble on the batch and compare to 2-signal outcome.""" |
| 107 | batch = self._pending[:self._eval_window] |
| 108 | self._pending = self._pending[self._eval_window:] # drain the batch |
| 109 | |
| 110 | labeled = [r for r in batch if r.gold_tier is not None] |
| 111 | if len(labeled) < 10: |
| 112 | return |
| 113 | |
| 114 | # Compare 2-signal vs replayed 3-signal |
no outgoing calls