| 267 | self.stop() |
| 268 | |
| 269 | def run(self): |
| 270 | # Don't fetch logs or adjust timing: just ping the watchdog. |
| 271 | # |
| 272 | # If we hit an exception, reset our session as it is likely broken. |
| 273 | while self._running: |
| 274 | try: |
| 275 | self._worker_manager.ping(request=None) |
| 276 | time.sleep(self.ping_interval) |
| 277 | except errors.OpError as e: |
| 278 | # Catch any TF errors that occur so we don't stop sending heartbeats |
| 279 | logging.debug('Caught error while sending heartbeat: %s', e) |
| 280 | self._reset_manager() |
| 281 | |
| 282 | |
| 283 | def start_worker_watchdog(session, |