MCPcopy Create free account
hub / github.com/NVIDIA/OpenShell / full_failover_sequence

Function full_failover_sequence

crates/openshell-server/src/compute/lease.rs:448–478  ·  view source on GitHub ↗
()

Source from the content-addressed store, hash-verified

446
447 #[tokio::test]
448 async fn full_failover_sequence() {
449 let store = test_store().await;
450 // Both use TTL=0 so steals succeed immediately. The TTL on the
451 // stealer determines whether *it* can be stolen, so TTL=0 means
452 // the "can't steal back" check needs a separate lease with a
453 // long TTL to observe AlreadyHeld from the stealer's perspective.
454 let l1 = lease(store.clone(), "replica-1", Duration::ZERO);
455 let l2 = lease(store.clone(), "replica-2", Duration::ZERO);
456
457 // Replica-1 acquires and renews once
458 let mut guard1 = l1.try_acquire().await.unwrap();
459 l1.renew(&mut guard1).await.unwrap();
460
461 // Replica-2 steals the expired lease
462 let mut guard2 = l2.acquire_or_steal().await.unwrap();
463
464 // Replica-1 detects the loss on its next renew
465 let err = l1.renew(&mut guard1).await.unwrap_err();
466 assert!(matches!(err, LeaseError::Conflict));
467
468 // Replica-2 can renew successfully
469 l2.renew(&mut guard2).await.unwrap();
470
471 // Replica-1 cannot re-acquire (lease exists)
472 let l1_retry = lease(store.clone(), "replica-1", Duration::from_secs(60));
473 let err = l1_retry.try_acquire().await.unwrap_err();
474 assert!(matches!(err, LeaseError::AlreadyHeld));
475
476 let record = l2.read().await.unwrap().unwrap();
477 assert_eq!(record.holder, "replica-2");
478 }
479
480 #[tokio::test]
481 async fn concurrent_steal_exactly_one_wins() {

Callers

nothing calls this directly

Calls 6

leaseFunction · 0.85
try_acquireMethod · 0.80
renewMethod · 0.80
acquire_or_stealMethod · 0.80
test_storeFunction · 0.70
readMethod · 0.45

Tested by

no test coverage detected