(t *testing.T, proxyURL string, restoreAsPrimary bool, postKill func(smap *cluster.Smap, newPrimary, oldPrimary *cluster.Snode))
| 98 | } |
| 99 | |
| 100 | func killRestorePrimary(t *testing.T, proxyURL string, restoreAsPrimary bool, |
| 101 | postKill func(smap *cluster.Smap, newPrimary, oldPrimary *cluster.Snode)) *cluster.Smap { |
| 102 | var ( |
| 103 | smap = tutils.GetClusterMap(t, proxyURL) |
| 104 | proxyCount = smap.CountActiveProxies() |
| 105 | oldPrimary = smap.Primary |
| 106 | oldPrimaryURL = smap.Primary.URL(cmn.NetPublic) |
| 107 | oldPrimaryID = smap.Primary.ID() |
| 108 | ) |
| 109 | |
| 110 | tlog.Logf("targets: %d, proxies: %d\n", smap.CountActiveTargets(), proxyCount) |
| 111 | newPrimaryID, newPrimaryURL, err := chooseNextProxy(smap) |
| 112 | tassert.CheckFatal(t, err) |
| 113 | newPrimary := smap.GetProxy(newPrimaryID) |
| 114 | |
| 115 | tlog.Logf("New primary: %s --> %s\n", newPrimaryID, newPrimaryURL) |
| 116 | tlog.Logf("Killing primary: %s --> %s\n", oldPrimaryURL, oldPrimaryID) |
| 117 | |
| 118 | // cmd and args are the original command line of how the proxy is started |
| 119 | cmd, err := tutils.KillNode(smap.Primary) |
| 120 | tassert.CheckFatal(t, err) |
| 121 | |
| 122 | smap, err = tutils.WaitForClusterState(newPrimaryURL, "designate new primary", smap.Version, |
| 123 | smap.CountActiveProxies()-1, smap.CountActiveTargets()) |
| 124 | tassert.CheckFatal(t, err) |
| 125 | tlog.Logf("New primary elected: %s\n", newPrimaryID) |
| 126 | |
| 127 | tassert.Errorf(t, smap.Primary.ID() == newPrimaryID, "Wrong primary proxy: %s, expecting: %s", smap.Primary.ID(), newPrimaryID) |
| 128 | |
| 129 | if postKill != nil { |
| 130 | postKill(smap, newPrimary, oldPrimary) |
| 131 | } |
| 132 | |
| 133 | // re-construct the command line to start the original proxy but add the current primary proxy to the args |
| 134 | err = tutils.RestoreNode(cmd, false, "proxy (prev primary)") |
| 135 | tassert.CheckFatal(t, err) |
| 136 | |
| 137 | smap, err = tutils.WaitForClusterState(newPrimaryURL, "restore", smap.Version, proxyCount, 0) |
| 138 | tassert.CheckFatal(t, err) |
| 139 | if _, ok := smap.Pmap[oldPrimaryID]; !ok { |
| 140 | t.Fatalf("Previous primary proxy did not rejoin the cluster") |
| 141 | } |
| 142 | checkSmaps(t, newPrimaryURL) |
| 143 | |
| 144 | if restoreAsPrimary { |
| 145 | return setPrimaryTo(t, oldPrimaryURL, smap, "", oldPrimaryID) |
| 146 | } |
| 147 | return smap |
| 148 | } |
| 149 | |
| 150 | func nodeCrashRestoreDifferentIP(t *testing.T) { |
| 151 | for _, ty := range []string{apc.Proxy, apc.Target} { |
no test coverage detected