Function checkOnlyEndpointFailed

fdbclient/NativeAPI.actor.cpp:2912–2937

Checks if `endpoint` is failed on a healthy server or not. Returns true if we need to refresh the location cache for the endpoint.

// Checks if `endpoint` is failed on a healthy server or not. Returns true if we need to refresh the location cache for
// the endpoint.
bool checkOnlyEndpointFailed(const Database& cx, const Endpoint& endpoint) {
	if (IFailureMonitor::failureMonitor().onlyEndpointFailed(endpoint)) {
		// This endpoint is failed, but the server is still healthy. There are two cases this can happen:
		//    - There is a recent bounce in the cluster where the endpoints in SSes get updated.
		//    - The SS is failed and terminated on a server, but the server is kept running.
		// To account for the first case, we invalidate the cache and issue GetKeyLocation requests to the proxy to
		// update the cache with the new SS points. However, if the failure is caused by the second case, the
		// requested key location will continue to be the failed endpoint until the data movement is finished. But
		// every read will generate a GetKeyLocation request to the proxies (and still getting the failed endpoint
		// back), which may overload the proxy and affect data movement speed. Therefore, we only refresh the
		// location cache for short period of time, and after the initial grace period that we keep retrying
		// resolving key location, we will slow it down to resolve it only once every
		// `LOCATION_CACHE_FAILED_ENDPOINT_RETRY_INTERVAL`.
		cx->setFailedEndpointOnHealthyServer(endpoint);
		const auto& failureInfo = cx->getEndpointFailureInfo(endpoint);
		ASSERT(failureInfo.present());
		if (now() - failureInfo.get().startTime < CLIENT_KNOBS->LOCATION_CACHE_ENDPOINT_FAILURE_GRACE_PERIOD ||
		    now() - failureInfo.get().lastRefreshTime > CLIENT_KNOBS->LOCATION_CACHE_FAILED_ENDPOINT_RETRY_INTERVAL) {
			cx->updateFailedEndpointRefreshTime(endpoint);
			return true;
		}
	} else {
		cx->clearFailedEndpointOnHealthyServer(endpoint);
	}
	return false;
}

template <class F>
Future<KeyRangeLocationInfo> getKeyLocation(Database const& cx,