Function watchdog

apps/cloud/scripts/dev-db.ts:209–262

()

};

const watchdog = async () => {
  let consecutiveFailures = 0;
  let restarts = 0;
  for (;;) {
    await sleep(WATCHDOG_INTERVAL_MS);
    if (stopping) return;
    try {
      await probe();
      consecutiveFailures = 0;
    } catch (cause) {
      consecutiveFailures += 1;
      console.error(
        `[dev-db][watchdog] probe failed (${consecutiveFailures}/${WATCHDOG_FAILURES_TO_RESTART}): ${String(cause)}`,
      );
      if (consecutiveFailures < WATCHDOG_FAILURES_TO_RESTART) continue;
      console.error(
        `[dev-db][watchdog] socket server wedged; stats: ${JSON.stringify(server.getStats())}`,
      );
      if (restarts >= WATCHDOG_MAX_RESTARTS) {
        console.error(
          `[dev-db][watchdog] still wedged after ${restarts} restarts — giving up so the boot supervisor reports it`,
        );
        process.exit(1);
      }
      restarts += 1;
      consecutiveFailures = 0;
      console.error(
        `[dev-db][watchdog] restarting socket server (${restarts}/${WATCHDOG_MAX_RESTARTS})`,
      );
      // stop() itself goes through the query queue (detach rolls back open
      // transactions), so a wedge deep enough can hang the restart too —
      // bound it and treat that as fatal rather than hanging the watchdog.
      const restart = async () => {
        await server.stop();
        server = makeServer();
        await server.start();
      };
      try {
        await Promise.race([
          restart(),
          sleep(15_000).then(() => {
            throw new Error("restart timed out after 15s");
          }),
        ]);
        console.error(`[dev-db][watchdog] socket server restarted`);
      } catch (restartCause) {
        console.error(
          `[dev-db][watchdog] restart failed (${String(restartCause)}) — exiting so the boot supervisor reports it`,
        );
        process.exit(1);
      }
    }
  }
};

void watchdog();