| 3177 | } |
| 3178 | |
| 3179 | [[noreturn]] void ImpalaServer::UnresponsiveBackendThread() { |
| 3180 | int64_t max_lag_ms = FLAGS_status_report_max_retry_s * 1000 |
| 3181 | * (1 + FLAGS_status_report_cancellation_padding / 100.0); |
| 3182 | DCHECK_GT(max_lag_ms, 0); |
| 3183 | VLOG(1) << "Queries will be cancelled if a backend has not reported its status in " |
| 3184 | << "more than " << max_lag_ms << "ms."; |
| 3185 | while (true) { |
| 3186 | vector<CancellationWork> to_cancel; |
| 3187 | query_driver_map_.DoFuncForAllEntries( |
| 3188 | [&](const std::shared_ptr<QueryDriver>& query_driver) { |
| 3189 | ClientRequestState* request_state = query_driver->GetActiveClientRequestState(); |
| 3190 | Coordinator* coord = request_state->GetCoordinator(); |
| 3191 | if (coord != nullptr) { |
| 3192 | NetworkAddressPB address; |
| 3193 | int64_t lag_time_ms = coord->GetMaxBackendStateLagMs(&address); |
| 3194 | if (lag_time_ms > max_lag_ms) { |
| 3195 | to_cancel.push_back( |
| 3196 | CancellationWork::TerminatedByServer(request_state->query_id(), |
| 3197 | Status(TErrorCode::UNRESPONSIVE_BACKEND, |
| 3198 | PrintId(request_state->query_id()), |
| 3199 | NetworkAddressPBToString(address), lag_time_ms, max_lag_ms), |
| 3200 | false /* unregister */)); |
| 3201 | } |
| 3202 | } |
| 3203 | }); |
| 3204 | |
| 3205 | // We call Offer() outside of DoFuncForAllEntries() to ensure that if the |
| 3206 | // cancellation_thread_pool_ queue is full, we're not blocked while holding one of the |
| 3207 | // 'query_driver_map_' shard locks. |
| 3208 | for (auto cancellation_work : to_cancel) { |
| 3209 | cancellation_thread_pool_->Offer(cancellation_work); |
| 3210 | } |
| 3211 | SleepForMs(max_lag_ms * 0.1); |
| 3212 | } |
| 3213 | } |
| 3214 | |
| 3215 | [[noreturn]] void ImpalaServer::AdmissionHeartbeatThread() { |
| 3216 | while (true) { |
nothing calls this directly
no test coverage detected