| 69 | } |
| 70 | |
| 71 | void initialize() override |
| 72 | { |
| 73 | // There are two different timeouts here: |
| 74 | // |
| 75 | // (1) `sessionTimeout` is the client's proposed value for the |
| 76 | // ZooKeeper session timeout. |
| 77 | // |
| 78 | // (2) `initLoopTimeout` is how long we are prepared to wait, |
| 79 | // calling `zookeeper_init` in a loop, until a call succeeds. |
| 80 | // |
| 81 | // `sessionTimeout` is used to determine the liveness of our |
| 82 | // ZooKeeper session. `initLoopTimeout` determines how long to |
| 83 | // retry erroneous calls to `zookeeper_init`, because there are |
| 84 | // cases when temporary DNS outages cause `zookeeper_init` to |
| 85 | // return failure. ZooKeeper masks EAI_AGAIN as EINVAL and a name |
| 86 | // resolution timeout may be upwards of 30 seconds. As such, a 10 |
| 87 | // second timeout (the default `sessionTimeout`) is not enough. We |
| 88 | // hardcode `initLoopTimeout` to 10 minutes ensure we're trying |
| 89 | // again in the face of temporary name resolution failures. See |
| 90 | // MESOS-1523 for more information. |
| 91 | // |
| 92 | // Note that there are cases where `zookeeper_init` returns |
| 93 | // success but we don't see a subsequent ZooKeeper event |
| 94 | // indicating that our connection has been established. A common |
| 95 | // cause for this situation is that the ZK hostname list resolves |
| 96 | // to unreachable IP addresses. ZooKeeper will continue looping, |
| 97 | // trying to connect to the list of IPs but never attempting to |
| 98 | // re-resolve the input hostnames. Since DNS may have changed, we |
| 99 | // close the ZK handle and create a new handle to ensure that ZK |
| 100 | // will try to re-resolve the configured list of hostnames. |
| 101 | // However, since we can't easily check if the `connected` ZK |
| 102 | // event has been fired for this session yet, we implement this |
| 103 | // timeout in `GroupProcess`. See MESOS-4546 for more information. |
| 104 | const Timeout initLoopTimeout = Timeout::in(Minutes(10)); |
| 105 | |
| 106 | while (!initLoopTimeout.expired()) { |
| 107 | zh = zookeeper_init( |
| 108 | servers.c_str(), |
| 109 | event, |
| 110 | static_cast<int>(sessionTimeout.ms()), |
| 111 | nullptr, |
| 112 | &callback, |
| 113 | 0); |
| 114 | |
| 115 | // Unfortunately, EINVAL is highly overloaded in zookeeper_init |
| 116 | // and can correspond to: |
| 117 | // (1) Empty / invalid 'host' string format. |
| 118 | // (2) Any getaddrinfo error other than EAI_NONAME, |
| 119 | // EAI_NODATA, and EAI_MEMORY are mapped to EINVAL. |
| 120 | // The errors EAI_NONAME and EAI_NODATA are mapped to ENOENT. |
| 121 | // Either way, retrying is not problematic. |
| 122 | if (zh == nullptr && (errno == EINVAL || errno == ENOENT)) { |
| 123 | ErrnoError error("zookeeper_init failed"); |
| 124 | LOG(WARNING) << error.message << " ; retrying in 1 second"; |
| 125 | os::sleep(Seconds(1)); |
| 126 | continue; |
| 127 | } |
| 128 | |