This function checks if a given node should be marked as FAIL. * It happens if the following conditions are met: * * 1) We received enough failure reports from other master nodes via gossip. * Enough means that the majority of the masters signaled the node is * down recently. * 2) We believe this node is in PFAIL state. * * If a failure is detected we also inform the whole cluster ab
| 1281 | * FAIL flag will be cleared after some time. |
| 1282 | */ |
| 1283 | void markNodeAsFailingIfNeeded(clusterNode *node) { |
| 1284 | int failures; |
| 1285 | int needed_quorum = (server.cluster->size / 2) + 1; |
| 1286 | |
| 1287 | if (!nodeTimedOut(node)) return; /* We can reach it. */ |
| 1288 | if (nodeFailed(node)) return; /* Already FAILing. */ |
| 1289 | |
| 1290 | failures = clusterNodeFailureReportsCount(node); |
| 1291 | /* Also count myself as a voter if I'm a master. */ |
| 1292 | if (nodeIsMaster(myself)) failures++; |
| 1293 | if (failures < needed_quorum) return; /* No weak agreement from masters. */ |
| 1294 | |
| 1295 | serverLog(LL_NOTICE, |
| 1296 | "Marking node %.40s as failing (quorum reached).", node->name); |
| 1297 | |
| 1298 | /* Mark the node as failing. */ |
| 1299 | node->flags &= ~CLUSTER_NODE_PFAIL; |
| 1300 | node->flags |= CLUSTER_NODE_FAIL; |
| 1301 | node->fail_time = mstime(); |
| 1302 | |
| 1303 | /* Broadcast the failing node name to everybody, forcing all the other |
| 1304 | * reachable nodes to flag the node as FAIL. |
| 1305 | * We do that even if this node is a replica and not a master: anyway |
| 1306 | * the failing state is triggered collecting failure reports from masters, |
| 1307 | * so here the replica is only helping propagating this status. */ |
| 1308 | clusterSendFail(node->name); |
| 1309 | clusterDoBeforeSleep(CLUSTER_TODO_UPDATE_STATE|CLUSTER_TODO_SAVE_CONFIG); |
| 1310 | } |
| 1311 | |
| 1312 | /* This function is called only if a node is marked as FAIL, but we are able |
| 1313 | * to reach it again. It checks if there are the conditions to undo the FAIL |
no test coverage detected