primaryAndProxyCrash kills primary proxy and one another proxy(not the next in line primary) and restore them afterwards
(t *testing.T)
| 411 | // primaryAndProxyCrash kills primary proxy and one another proxy(not the next in line primary) |
| 412 | // and restore them afterwards |
| 413 | func primaryAndProxyCrash(t *testing.T) { |
| 414 | var ( |
| 415 | proxyURL = tutils.RandomProxyURL(t) |
| 416 | smap = tutils.GetClusterMap(t, proxyURL) |
| 417 | origProxyCount = smap.CountActiveProxies() |
| 418 | oldPrimaryURL, oldPrimaryID = smap.Primary.URL(cmn.NetPublic), smap.Primary.ID() |
| 419 | secondNode *cluster.Snode |
| 420 | secondID string |
| 421 | ) |
| 422 | tlog.Logf("targets: %d, proxies: %d\n", smap.CountActiveTargets(), smap.CountActiveProxies()) |
| 423 | |
| 424 | newPrimaryID, newPrimaryURL, err := chooseNextProxy(smap) |
| 425 | tassert.CheckFatal(t, err) |
| 426 | |
| 427 | tlog.Logf("Killing primary: %s - %s\n", oldPrimaryURL, oldPrimaryID) |
| 428 | cmd, err := tutils.KillNode(smap.Primary) |
| 429 | tassert.CheckFatal(t, err) |
| 430 | |
| 431 | // Do not choose the next primary in line, or the current primary proxy |
| 432 | // This is because the system currently cannot recover if the next proxy in line is |
| 433 | // also killed (TODO) |
| 434 | for k, v := range smap.Pmap { |
| 435 | if k != newPrimaryID && k != oldPrimaryID { |
| 436 | secondNode = v |
| 437 | secondID = secondNode.ID() |
| 438 | break |
| 439 | } |
| 440 | } |
| 441 | tassert.Errorf(t, secondID != "", "not enough proxies (%d)", origProxyCount) |
| 442 | n := cos.NowRand().Intn(20) |
| 443 | time.Sleep(time.Duration(n+1) * time.Second) |
| 444 | |
| 445 | tlog.Logf("Killing non-primary: %s\n", secondNode.StringEx()) |
| 446 | secondCmd, err := tutils.KillNode(secondNode) |
| 447 | tassert.CheckFatal(t, err) |
| 448 | |
| 449 | smap, err = tutils.WaitForClusterState(newPrimaryURL, "elect new primary", |
| 450 | smap.Version, origProxyCount-2, 0) |
| 451 | tassert.CheckFatal(t, err) |
| 452 | |
| 453 | err = tutils.RestoreNode(cmd, true, "previous primary "+oldPrimaryID) |
| 454 | tassert.CheckFatal(t, err) |
| 455 | |
| 456 | smap, err = tutils.WaitForClusterState(newPrimaryURL, "join back previous primary "+oldPrimaryID, |
| 457 | smap.Version, origProxyCount-1, 0) |
| 458 | tassert.CheckFatal(t, err) |
| 459 | |
| 460 | err = tutils.RestoreNode(secondCmd, false, "proxy") |
| 461 | tassert.CheckFatal(t, err) |
| 462 | |
| 463 | smap, err = tutils.WaitForClusterState(newPrimaryURL, "join back non-primary "+secondID, |
| 464 | smap.Version, origProxyCount, 0) |
| 465 | tassert.CheckFatal(t, err) |
| 466 | |
| 467 | if smap.Primary.ID() != newPrimaryID { |
| 468 | t.Fatalf("Wrong primary proxy: %s, expecting: %s", smap.Primary.ID(), newPrimaryID) |
| 469 | } |
| 470 |
nothing calls this directly
no test coverage detected