(
self: &Arc<Self>,
lease: &lease::ReconcilerLease,
mut guard: lease::LeaseGuard,
shutdown_rx: &mut watch::Receiver<bool>,
)
| 913 | } |
| 914 | |
| 915 | async fn run_as_holder( |
| 916 | self: &Arc<Self>, |
| 917 | lease: &lease::ReconcilerLease, |
| 918 | mut guard: lease::LeaseGuard, |
| 919 | shutdown_rx: &mut watch::Receiver<bool>, |
| 920 | ) { |
| 921 | use lease::LEASE_RENEWAL_INTERVAL; |
| 922 | |
| 923 | let (cancel_tx, cancel_rx) = watch::channel(false); |
| 924 | |
| 925 | let runtime = self.clone(); |
| 926 | let watch_cancel = cancel_rx.clone(); |
| 927 | let watch_handle = tokio::spawn(async move { |
| 928 | runtime.watch_loop(watch_cancel).await; |
| 929 | }); |
| 930 | |
| 931 | let runtime = self.clone(); |
| 932 | let reconcile_handle = tokio::spawn(async move { |
| 933 | runtime.reconcile_loop(cancel_rx).await; |
| 934 | }); |
| 935 | |
| 936 | loop { |
| 937 | tokio::select! { |
| 938 | () = tokio::time::sleep(LEASE_RENEWAL_INTERVAL) => { |
| 939 | match lease.renew(&mut guard).await { |
| 940 | Ok(()) => { |
| 941 | debug!(replica = %lease.replica_id(), "renewed reconciler lease"); |
| 942 | } |
| 943 | Err(e) => { |
| 944 | warn!( |
| 945 | replica = %lease.replica_id(), |
| 946 | error = %e, |
| 947 | "reconciler lease renewal failed — releasing holder role" |
| 948 | ); |
| 949 | break; |
| 950 | } |
| 951 | } |
| 952 | } |
| 953 | _ = shutdown_rx.changed() => { |
| 954 | if *shutdown_rx.borrow() { |
| 955 | info!(replica = %lease.replica_id(), "shutdown — releasing reconciler lease"); |
| 956 | if let Err(e) = lease.release(guard).await { |
| 957 | warn!(error = %e, "failed to release reconciler lease on shutdown"); |
| 958 | } |
| 959 | let _ = cancel_tx.send(true); |
| 960 | let _ = watch_handle.await; |
| 961 | let _ = reconcile_handle.await; |
| 962 | return; |
| 963 | } |
| 964 | } |
| 965 | } |
| 966 | } |
| 967 | |
| 968 | let _ = cancel_tx.send(true); |
| 969 | let _ = watch_handle.await; |
| 970 | let _ = reconcile_handle.await; |
| 971 | info!(replica = %lease.replica_id(), "reconciler lease lost — returning to standby"); |
| 972 | } |
no test coverage detected