runaway script detection ensures that no single vm can block the thread for more than the allowed interval
(
shutdown_handle: VmShutdownHandle,
ping_send: mpsc::Sender<oneshot::Sender<()>>,
)
| 69 | // runaway script detection ensures that no single vm can |
| 70 | // block the thread for more than the allowed interval |
| 71 | async fn monitor_vm_runaway( |
| 72 | shutdown_handle: VmShutdownHandle, |
| 73 | ping_send: mpsc::Sender<oneshot::Sender<()>>, |
| 74 | ) { |
| 75 | let ping_interval = Duration::from_secs(10); |
| 76 | loop { |
| 77 | let (send, rcv) = oneshot::channel(); |
| 78 | match ping_send.send(send).await { |
| 79 | Ok(_) => { |
| 80 | let last_ping = Instant::now(); |
| 81 | match tokio::time::timeout(ping_interval, rcv).await { |
| 82 | Ok(_) => { |
| 83 | // sleep until the next ping |
| 84 | let remaining = ping_interval - last_ping.elapsed(); |
| 85 | tokio::time::sleep(remaining).await; |
| 86 | } |
| 87 | Err(_) => { |
| 88 | // we hit a timeout, meaning there's a runaway script |
| 89 | // |
| 90 | // note that this logic is currently very flawed, |
| 91 | // you could make a vm that takes a 1 less millisecond than the ping interval |
| 92 | // then the next vm takes move than 1 millisecond and this logic |
| 93 | // will think the cause is the latter even though it was only responsible for 1ms |
| 94 | // |
| 95 | // in the future we will have to actively track the cpu usage of vm's to shut down the proper vm |
| 96 | // if there's a bad actor |
| 97 | shutdown_handle.shutdown_vm(ShutdownReason::Runaway, true); |
| 98 | } |
| 99 | } |
| 100 | } |
| 101 | Err(_) => { |
| 102 | // receiver was dropped meaning the thread has shut down |
| 103 | return; |
| 104 | } |
| 105 | } |
| 106 | } |
| 107 | } |
no test coverage detected