Function WaitForClusterState

devtools/tutils/node.go:171–261

WaitForClusterState waits until a cluster reaches specified state, meaning: - smap has version larger than origVersion - number of active proxies is equal proxyCnt, unless proxyCnt == 0 - number of active targets is equal targetCnt, unless targetCnt == 0. It returns the smap which satisfies those r

(proxyURL, reason string, origVer int64, pcnt, tcnt int, ignoreIDs ...string)

// It returns the smap which satisfies those requirements.
// NOTE: Upon successful return from this function cluster state might have already changed.
func WaitForClusterState(proxyURL, reason string, origVer int64, pcnt, tcnt int, ignoreIDs ...string) (*cluster.Smap, error) {
	var (
		lastVersion                               int64
		smapChangeDeadline, timeStart, opDeadline time.Time

		expPrx = nodesCnt(pcnt)
		expTgt = nodesCnt(tcnt)

		baseParams = BaseAPIParams(proxyURL)
		loopCnt    int
		satisfied  bool
	)

	timeStart = time.Now()
	smapChangeDeadline = timeStart.Add(2 * proxyChangeLatency)
	opDeadline = timeStart.Add(3 * proxyChangeLatency)

	tlog.Logf("Waiting for: %q(p%d, t%d, Smap > v%d)\n", reason, expPrx, expTgt, origVer)

	// Repeat until success or timeout.
	for {
		smap, err := api.GetClusterMap(baseParams)
		if err != nil {
			if !cos.IsRetriableConnErr(err) {
				return nil, err
			}
			tlog.Logf("%v\n", err)
			goto next
		}
		satisfied = expTgt.satisfied(smap.CountActiveTargets()) &&
			expPrx.satisfied(smap.CountActiveProxies()) &&
			smap.Version > origVer
		if !satisfied {
			if d := time.Since(timeStart); d > 7*time.Second {
				p := "primary"
				if smap.Primary.PubNet.URL != proxyURL {
					p = proxyURL
				}
				tlog.Logf("Polling %s[%s] for (t=%d, p=%d, Smap > v%d)\n", p, smap, expTgt, expPrx, origVer)
			}
		}
		if smap.Version != lastVersion {
			smapChangeDeadline = cos.MinTime(time.Now().Add(proxyChangeLatency), opDeadline)
		}
		// if the primary's map changed to the state we want, wait for the map get populated
		if satisfied {
			syncedSmap := &cluster.Smap{}
			cos.CopyStruct(syncedSmap, smap)

			// skip primary proxy and mock targets
			var proxyID string
			for _, p := range smap.Pmap {
				if p.PubNet.URL == proxyURL {
					proxyID = p.ID()
				}
			}

			idsToIgnore := cos.NewStringSet(MockDaemonID, proxyID)