(self)
| 146 | raise |
| 147 | |
| 148 | def loop(self) -> None: |
| 149 | check_interval = 20 |
| 150 | min_interval = 20 |
| 151 | max_interval = 300 |
| 152 | backoff_factor = 1.5 |
| 153 | consecutive_failures = 0 |
| 154 | heartbeat_interval = 300 |
| 155 | last_heartbeat = time.monotonic() |
| 156 | |
| 157 | while not self.stop: |
| 158 | try: |
| 159 | for thread in [self.system, self.reporter_agent]: |
| 160 | status = thread.check_status() |
| 161 | label = "Ok" if status else "Critical" |
| 162 | self.log.debug(f"{thread} status: {label}") |
| 163 | consecutive_failures = 0 |
| 164 | check_interval = min_interval |
| 165 | self.log.debug( |
| 166 | "All threads are alive, next check in %ds.", check_interval |
| 167 | ) |
| 168 | now = time.monotonic() |
| 169 | if now - last_heartbeat >= heartbeat_interval: |
| 170 | self.log.info( |
| 171 | "node-proxy running (heartbeat), next check in %ds.", |
| 172 | heartbeat_interval, |
| 173 | ) |
| 174 | last_heartbeat = now |
| 175 | except Exception as e: |
| 176 | consecutive_failures += 1 |
| 177 | self.log.error( |
| 178 | f"{consecutive_failures} failure(s): thread not running: " |
| 179 | f"{e.__class__.__name__}: {e}" |
| 180 | ) |
| 181 | for thread in [self.system, self.reporter_agent]: |
| 182 | thread.shutdown() |
| 183 | self.init_system() |
| 184 | self.init_reporter() |
| 185 | check_interval = min(int(check_interval * backoff_factor), max_interval) |
| 186 | self.log.info("Next check in %ds (backoff).", check_interval) |
| 187 | time.sleep(check_interval) |
| 188 | |
| 189 | def shutdown(self) -> None: |
| 190 | self.stop = True |
no test coverage detected