| 207 | }; |
| 208 | |
| 209 | const watchdog = async () => { |
| 210 | let consecutiveFailures = 0; |
| 211 | let restarts = 0; |
| 212 | // Heartbeat: the CI e2e cascade of 2026-08-28 (run 33129376530) showed the |
| 213 | // app starving on CONNECT_TIMEOUT for 100+ seconds while every watchdog |
| 214 | // probe silently PASSED — the stall was on the app's side of the socket, |
| 215 | // not this server's. A silent-when-healthy watchdog cannot distinguish |
| 216 | // "healthy" from "not running", and it discards the one signal that would |
| 217 | // test the leading theory (workerd leaking dev-db connections until its |
| 218 | // socket layer starves): the active connection count over time. Log stats |
| 219 | // periodically and whenever the count jumps a bucket. |
| 220 | let lastHeartbeatAt = Date.now(); |
| 221 | let lastLoggedBucket = 0; |
| 222 | for (;;) { |
| 223 | await sleep(WATCHDOG_INTERVAL_MS); |
| 224 | if (stopping) return; |
| 225 | try { |
| 226 | await probe(); |
| 227 | consecutiveFailures = 0; |
| 228 | const stats = server.getStats(); |
| 229 | const bucket = Math.floor(stats.activeConnections / 50); |
| 230 | if (bucket !== lastLoggedBucket || Date.now() - lastHeartbeatAt >= 60_000) { |
| 231 | lastLoggedBucket = bucket; |
| 232 | lastHeartbeatAt = Date.now(); |
| 233 | console.log(`[dev-db][watchdog] healthy; stats: ${JSON.stringify(stats)}`); |
| 234 | } |
| 235 | } catch (cause) { |
| 236 | consecutiveFailures += 1; |
| 237 | console.error( |
| 238 | `[dev-db][watchdog] probe failed (${consecutiveFailures}/${WATCHDOG_FAILURES_TO_RESTART}): ${String(cause)}`, |
| 239 | ); |
| 240 | if (consecutiveFailures < WATCHDOG_FAILURES_TO_RESTART) continue; |
| 241 | console.error( |
| 242 | `[dev-db][watchdog] socket server wedged; stats: ${JSON.stringify(server.getStats())}`, |
| 243 | ); |
| 244 | if (restarts >= WATCHDOG_MAX_RESTARTS) { |
| 245 | console.error( |
| 246 | `[dev-db][watchdog] still wedged after ${restarts} restarts — giving up so the boot supervisor reports it`, |
| 247 | ); |
| 248 | process.exit(1); |
| 249 | } |
| 250 | restarts += 1; |
| 251 | consecutiveFailures = 0; |
| 252 | console.error( |
| 253 | `[dev-db][watchdog] restarting socket server (${restarts}/${WATCHDOG_MAX_RESTARTS})`, |
| 254 | ); |
| 255 | // stop() itself goes through the query queue (detach rolls back open |
| 256 | // transactions), so a wedge deep enough can hang the restart too — |
| 257 | // bound it and treat that as fatal rather than hanging the watchdog. |
| 258 | const restart = async () => { |
| 259 | await server.stop(); |
| 260 | server = makeServer(); |
| 261 | await server.start(); |
| 262 | }; |
| 263 | try { |
| 264 | await Promise.race([ |
| 265 | restart(), |
| 266 | sleep(15_000).then(() => { |