| 409 | } |
| 410 | |
| 411 | void KrpcDataStreamMgr::Maintenance() { |
| 412 | const int32_t sleep_time_ms = |
| 413 | min(max(1, FLAGS_datastream_sender_timeout_ms / 2), 10000); |
| 414 | while (true) { |
| 415 | const int64_t now = MonotonicMillis(); |
| 416 | |
| 417 | // Notify any senders that have been waiting too long for their receiver to |
| 418 | // appear. Keep lock_ held for only a short amount of time. |
| 419 | vector<EarlySendersList> timed_out_senders; |
| 420 | { |
| 421 | lock_guard<mutex> l(lock_); |
| 422 | auto it = early_senders_map_.begin(); |
| 423 | while (it != early_senders_map_.end()) { |
| 424 | if (now - it->second.arrival_time > FLAGS_datastream_sender_timeout_ms) { |
| 425 | timed_out_senders.emplace_back(move(it->second)); |
| 426 | it = early_senders_map_.erase(it); |
| 427 | } else { |
| 428 | ++it; |
| 429 | } |
| 430 | } |
| 431 | } |
| 432 | |
| 433 | // Send responses to all timed-out senders. We need to propagate the time-out errors |
| 434 | // to senders which sent EOS RPC so all query fragments will eventually be cancelled. |
| 435 | // Otherwise, the receiver may hang when it eventually gets created as the timed-out |
| 436 | // EOS will be lost forever. |
| 437 | for (const EarlySendersList& senders_queue : timed_out_senders) { |
| 438 | for (const unique_ptr<TransmitDataCtx>& ctx : senders_queue.waiting_sender_ctxs) { |
| 439 | RespondToTimedOutSender<TransmitDataCtx, TransmitDataRequestPB>(ctx); |
| 440 | } |
| 441 | for (const unique_ptr<EndDataStreamCtx>& ctx : senders_queue.closed_sender_ctxs) { |
| 442 | RespondToTimedOutSender<EndDataStreamCtx, EndDataStreamRequestPB>(ctx); |
| 443 | } |
| 444 | } |
| 445 | |
| 446 | // Remove any closed streams that have been in the cache for more than |
| 447 | // STREAM_EXPIRATION_TIME_MS. |
| 448 | { |
| 449 | lock_guard<mutex> l(lock_); |
| 450 | ClosedStreamMap::iterator it = closed_stream_expirations_.begin(); |
| 451 | int32_t before = closed_stream_cache_.size(); |
| 452 | while (it != closed_stream_expirations_.end() && it->first < now) { |
| 453 | closed_stream_cache_.erase(it->second); |
| 454 | closed_stream_expirations_.erase(it++); |
| 455 | } |
| 456 | DCHECK_EQ(closed_stream_cache_.size(), closed_stream_expirations_.size()); |
| 457 | int32_t after = closed_stream_cache_.size(); |
| 458 | if (before != after) { |
| 459 | VLOG_QUERY << "Reduced stream ID cache from " << before << " items, to " << after |
| 460 | << ", eviction took: " |
| 461 | << PrettyPrinter::Print(MonotonicMillis() - now, TUnit::TIME_MS); |
| 462 | } |
| 463 | } |
| 464 | |
| 465 | bool timed_out = false; |
| 466 | shutdown_promise_.Get(sleep_time_ms, &timed_out); |
| 467 | if (!timed_out) return; |
| 468 | } |