(
state: Arc<SharedState>,
registry: Arc<AlertRegistry>,
mut shutdown: watch::Receiver<bool>,
)
| 41 | } |
| 42 | |
| 43 | async fn alert_eval_loop( |
| 44 | state: Arc<SharedState>, |
| 45 | registry: Arc<AlertRegistry>, |
| 46 | mut shutdown: watch::Receiver<bool>, |
| 47 | ) { |
| 48 | // Initial delay to let the system warm up. |
| 49 | tokio::time::sleep(Duration::from_secs(5)).await; |
| 50 | |
| 51 | // Use the shared hysteresis manager from SharedState so DROP/ALTER handlers |
| 52 | // and the eval loop operate on the same state. |
| 53 | let hysteresis = &state.alert_hysteresis; |
| 54 | |
| 55 | loop { |
| 56 | // Sleep for the shortest alert window (minimum 1 second). |
| 57 | let sleep_ms = next_eval_interval_ms(®istry); |
| 58 | |
| 59 | tokio::select! { |
| 60 | _ = tokio::time::sleep(Duration::from_millis(sleep_ms)) => {} |
| 61 | _ = shutdown.changed() => { |
| 62 | if *shutdown.borrow() { |
| 63 | info!("alert eval loop shutting down"); |
| 64 | return; |
| 65 | } |
| 66 | } |
| 67 | } |
| 68 | |
| 69 | let alerts = registry.list_all_enabled(); |
| 70 | if alerts.is_empty() { |
| 71 | continue; |
| 72 | } |
| 73 | |
| 74 | let now_ms = std::time::SystemTime::now() |
| 75 | .duration_since(std::time::UNIX_EPOCH) |
| 76 | .unwrap_or_default() |
| 77 | .as_millis() as u64; |
| 78 | |
| 79 | for alert in &alerts { |
| 80 | if let Err(e) = evaluate_alert(&state, alert, hysteresis, now_ms).await { |
| 81 | warn!( |
| 82 | alert = alert.name, |
| 83 | collection = alert.collection, |
| 84 | error = %e, |
| 85 | "alert evaluation failed" |
| 86 | ); |
| 87 | } |
| 88 | } |
| 89 | } |
| 90 | } |
| 91 | |
| 92 | /// Evaluate a single alert rule: execute aggregate query, check condition per group, |
| 93 | /// feed through hysteresis, dispatch notifications. |
no test coverage detected