Background cleanup: stop supervisor process, wait for it to exit, remove from supervisor, optionally delete workdir, and free CID. `delete_workdir`: true for user-initiated removal, false for orphan cleanup.
(&self, id: &str, delete_workdir: bool)
| 325 | /// |
| 326 | /// `delete_workdir`: true for user-initiated removal, false for orphan cleanup. |
| 327 | async fn finish_remove_vm(&self, id: &str, delete_workdir: bool) -> Result<()> { |
| 328 | // Stop the supervisor process (idempotent if already stopped) |
| 329 | if let Err(err) = self.supervisor.stop(id).await { |
| 330 | debug!("supervisor.stop({id}) during removal: {err:?}"); |
| 331 | } |
| 332 | |
| 333 | // Poll until the process is no longer running, then remove it. |
| 334 | // Some VMs take a long time to stop (e.g. 2+ hours), so we wait indefinitely. |
| 335 | let mut poll_count: u64 = 0; |
| 336 | loop { |
| 337 | match self.supervisor.info(id).await { |
| 338 | Ok(Some(info)) if info.state.status.is_running() => { |
| 339 | tokio::time::sleep(std::time::Duration::from_secs(2)).await; |
| 340 | poll_count += 1; |
| 341 | if poll_count.is_multiple_of(30) { |
| 342 | info!( |
| 343 | "VM {id} still running after {}m during removal, waiting...", |
| 344 | poll_count * 2 / 60 |
| 345 | ); |
| 346 | } |
| 347 | } |
| 348 | Ok(Some(_)) => { |
| 349 | // Not running — remove from supervisor |
| 350 | if let Err(err) = self.supervisor.remove(id).await { |
| 351 | warn!("supervisor.remove({id}) failed: {err:?}"); |
| 352 | } |
| 353 | break; |
| 354 | } |
| 355 | Ok(None) => { |
| 356 | // Already gone from supervisor |
| 357 | break; |
| 358 | } |
| 359 | Err(err) => { |
| 360 | warn!("supervisor.info({id}) failed during removal: {err:?}"); |
| 361 | tokio::time::sleep(std::time::Duration::from_secs(5)).await; |
| 362 | } |
| 363 | } |
| 364 | } |
| 365 | |
| 366 | // Only delete the workdir for user-initiated removal or if .removing marker exists. |
| 367 | // Orphaned supervisor processes without the marker keep their data intact. |
| 368 | let vm_path = self.work_dir(id); |
| 369 | if delete_workdir || vm_path.is_removing() { |
| 370 | if vm_path.path().exists() { |
| 371 | if let Err(err) = fs::remove_dir_all(&vm_path) { |
| 372 | error!("failed to remove VM directory for {id}: {err:?}"); |
| 373 | } |
| 374 | } |
| 375 | } else if vm_path.path().exists() { |
| 376 | info!( |
| 377 | "VM {id} workdir preserved (orphan cleanup): {}", |
| 378 | vm_path.path().display() |
| 379 | ); |
| 380 | } |
| 381 | |
| 382 | // Free CID and remove from memory (last step) |
| 383 | { |
| 384 | let mut state = self.lock(); |
no test coverage detected