Resume sandboxes whose store records say they should be running. Drivers that do not auto-restart compute resources across gateway restarts (currently only Docker) implement `StartupResume`. For each sandbox in the store whose phase is not `Deleting` or `Error`, we ask the driver to resume the underlying resource. If the driver reports that the resource no longer exists or fails to start, the sand
(&self)
| 748 | /// Should be called once at gateway startup, before watchers spawn, |
| 749 | /// so the watch loop sees the post-resume state on its first poll. |
| 750 | pub async fn resume_persisted_sandboxes(&self) -> Result<(), String> { |
| 751 | let Some(resume) = &self.startup_resume else { |
| 752 | return Ok(()); |
| 753 | }; |
| 754 | |
| 755 | let records = self |
| 756 | .store |
| 757 | .list(Sandbox::object_type(), 1000, 0) |
| 758 | .await |
| 759 | .map_err(|e| e.to_string())?; |
| 760 | |
| 761 | let mut resumed = 0usize; |
| 762 | let mut missing = 0usize; |
| 763 | let mut failed = 0usize; |
| 764 | |
| 765 | for record in records { |
| 766 | let sandbox = match Sandbox::decode(record.payload.as_slice()) { |
| 767 | Ok(sandbox) => sandbox, |
| 768 | Err(err) => { |
| 769 | warn!(error = %err, "Failed to decode sandbox record during startup resume"); |
| 770 | continue; |
| 771 | } |
| 772 | }; |
| 773 | |
| 774 | let phase = SandboxPhase::try_from(sandbox.phase()).unwrap_or(SandboxPhase::Unknown); |
| 775 | if !sandbox_phase_should_be_running(phase) { |
| 776 | continue; |
| 777 | } |
| 778 | |
| 779 | match resume |
| 780 | .resume_sandbox(sandbox.object_id(), sandbox.object_name()) |
| 781 | .await |
| 782 | { |
| 783 | Ok(true) => { |
| 784 | info!( |
| 785 | sandbox_id = %sandbox.object_id(), |
| 786 | sandbox_name = %sandbox.object_name(), |
| 787 | ?phase, |
| 788 | "Resumed sandbox during gateway startup" |
| 789 | ); |
| 790 | resumed += 1; |
| 791 | } |
| 792 | Ok(false) => { |
| 793 | // Backend resource is gone but the store still |
| 794 | // remembers the sandbox. Mark Error so the UI |
| 795 | // surfaces the inconsistency; the reconcile loop |
| 796 | // will eventually prune it after the orphan grace |
| 797 | // period. |
| 798 | warn!( |
| 799 | sandbox_id = %sandbox.object_id(), |
| 800 | sandbox_name = %sandbox.object_name(), |
| 801 | "Cannot resume sandbox: backend resource is missing" |
| 802 | ); |
| 803 | self.mark_sandbox_error( |
| 804 | &sandbox, |
| 805 | "BackendResourceMissing", |
| 806 | "Sandbox container disappeared while the gateway was offline", |
| 807 | ) |
no test coverage detected