| 76 | } |
| 77 | |
| 78 | void ThreadMonitor::monitoring_loop() { |
| 79 | if (m_clock_ticks_per_sec <= 0 || m_page_size <= 0) { |
| 80 | derr << "Failed to retrieve system configuration " |
| 81 | << "(clock ticks per second or page size). Monitoring will not start." << dendl; |
| 82 | running = false; |
| 83 | return; |
| 84 | } |
| 85 | |
| 86 | while (running) { |
| 87 | // Phase 1: under lock — update RSS counters and copy all state needed for CPU computation. |
| 88 | long long process_rss_pages = 0; |
| 89 | if (!read_process_statm(process_rss_pages)) { |
| 90 | derr << "Failed to read process memory info from /proc/self/statm." << dendl; |
| 91 | std::this_thread::sleep_for(monitoring_interval); |
| 92 | continue; |
| 93 | } |
| 94 | |
| 95 | std::vector<ThreadEntry> entries; |
| 96 | { |
| 97 | std::lock_guard<std::mutex> lock(monitored_threads_mutex); |
| 98 | for (auto& [tid, info] : monitored_threads) { |
| 99 | if (info.py_module && info.py_module->perfcounter) { |
| 100 | long long rss_bytes = process_rss_pages * m_page_size; |
| 101 | long long rss_change = rss_bytes - info.last_snapshot.rss_pages * m_page_size; |
| 102 | info.py_module->perfcounter->set(info.py_module->l_pym_mem_rss_current, rss_bytes); |
| 103 | info.py_module->perfcounter->set(info.py_module->l_pym_mem_rss_change, rss_change); |
| 104 | info.last_snapshot.rss_pages = process_rss_pages; |
| 105 | dout(20) << "Module '" << info.name << "' (TID: " << tid << "): " |
| 106 | << "Memory RSS: " << rss_bytes << " bytes" |
| 107 | << ", Change: " << rss_change << " bytes" << dendl; |
| 108 | } |
| 109 | entries.push_back({tid, info.serve_thread_id, info.name, |
| 110 | info.py_module, info.last_snapshot, info.last_serve_snapshot}); |
| 111 | } |
| 112 | } |
| 113 | |
| 114 | // Phase 2: no lock — do slow /proc reads and CPU calculations using copied snapshots. |
| 115 | std::vector<ThreadResult> results; |
| 116 | results.reserve(entries.size()); |
| 117 | for (const auto& e : entries) { |
| 118 | results.push_back(process_thread_stats(e)); |
| 119 | } |
| 120 | |
| 121 | // Phase 3: under lock — write results back; remove dead threads. |
| 122 | { |
| 123 | std::lock_guard<std::mutex> lock(monitored_threads_mutex); |
| 124 | for (const auto& r : results) { |
| 125 | auto it = monitored_threads.find(r.tid); |
| 126 | if (it == monitored_threads.end()) { |
| 127 | continue; // deregistered between phase 1 and 3 |
| 128 | } |
| 129 | MonitoredThreadInfo& info = it->second; |
| 130 | if (!r.main_ok) { |
| 131 | dout(0) << "Removing dead thread '" << info.name << "' (TID: " << r.tid << ")" << dendl; |
| 132 | monitored_threads.erase(it); |
| 133 | continue; |
| 134 | } |
| 135 | info.last_snapshot.utime = r.new_utime; |