diff --git a/crates/openshell-cli/src/run.rs b/crates/openshell-cli/src/run.rs index eb224685ed..39334ca53c 100644 --- a/crates/openshell-cli/src/run.rs +++ b/crates/openshell-cli/src/run.rs @@ -2942,18 +2942,20 @@ pub async fn sandbox_list( // Print header if all_workspaces { println!( - "{: &str { + sandbox + .status + .as_ref() + .and_then(|status| { + status.conditions.iter().find(|condition| { + condition.r#type == "Ready" && condition.reason == "ComputeDriverMismatch" + }) + }) + .map_or("-", |condition| condition.message.as_str()) +} + fn configuration_failure_message(sandbox: &Sandbox) -> Option<&str> { sandbox .status diff --git a/crates/openshell-server/src/compute/image_update.rs b/crates/openshell-server/src/compute/image_update.rs index 4a19ee2a53..27d583abc3 100644 --- a/crates/openshell-server/src/compute/image_update.rs +++ b/crates/openshell-server/src/compute/image_update.rs @@ -61,6 +61,7 @@ impl ComputeRuntime { .await .map_err(|_| Status::internal("fetch sandbox failed"))? .ok_or_else(|| Status::not_found("sandbox not found"))?; + self.validate_compute_driver_binding(¤t)?; ensure_image_update_stopped(¤t)?; if current.get_resource_version() != expected_resource_version { return Err(crate::grpc::persistence_error_to_status( @@ -454,6 +455,9 @@ impl ComputeRuntime { self.update_recovery_backoff.lock().await.remove(id); return Ok(()); }; + if self.other_driver_owner(&sandbox).is_some() { + return Ok(()); + } if sandbox.phase() == SandboxPhase::Deleting as i32 { self.update_recovery_backoff.lock().await.remove(id); return Ok(()); @@ -691,6 +695,79 @@ mod tests { .collect() } + #[tokio::test] + async fn foreign_driver_update_is_not_sent_or_recovered() { + let (runtime, driver, original) = fixture(true).await; + let mut foreign = original.clone(); + foreign.metadata.as_mut().unwrap().annotations.insert( + crate::compute::CREATED_BY_DRIVER_ANNOTATION.into(), + "vm".into(), + ); + runtime.store.put_message(&foreign).await.unwrap(); + let current = runtime + .store + .get_message::(foreign.object_id()) + .await + .unwrap() + .unwrap(); + driver.clear_calls(); + + let error = runtime + .update_sandbox_image( + current.object_id(), + "example/workload:new".into(), + current.get_resource_version(), + ) + .await + .unwrap_err(); + assert_eq!(error.code(), Code::FailedPrecondition); + assert!(error.message().contains("compute driver 'vm'")); + assert!(image_calls(&driver).is_empty()); + + runtime + .store + .begin_sandbox_image_update( + current.object_id(), + current.get_resource_version(), + 0, + &SandboxImageUpdate { + operation_id: "foreign-operation".into(), + previous_image: "example/workload:old".into(), + image: "example/workload:new".into(), + revision: 1, + ..Default::default() + }, + ) + .await + .unwrap(); + let pending = runtime + .store + .get_message::(foreign.object_id()) + .await + .unwrap() + .unwrap(); + runtime.recover_pending_sandbox_updates().await.unwrap(); + drop(runtime.update_recovery_sweep.acquire().await.unwrap()); + let retained = runtime + .store + .get_message::(foreign.object_id()) + .await + .unwrap() + .unwrap(); + assert_eq!(retained, pending); + assert!(image_calls(&driver).is_empty()); + assert!( + runtime + .store + .get_sandbox_image_state(foreign.object_id()) + .await + .unwrap() + .unwrap() + .pending + .is_some() + ); + } + #[tokio::test] async fn recovery_drains_multiple_pending_pages_with_updates_disabled() { let (mut runtime, driver, original) = fixture(true).await; diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 2039ef4f7f..95a9a5f3be 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -72,6 +72,12 @@ pub const COMPUTE_DRIVER_ANNOTATION: &str = "internal.openshell.ai/compute-drive const ISOLATION_BACKEND_ANNOTATION: &str = "internal.openshell.ai/isolation-backend"; pub const COMPUTE_RUNTIME_IDENTITY_ANNOTATION: &str = "internal.openshell.ai/compute-runtime-identity"; +/// Configured name of the compute driver that created the sandbox. Unlike +/// `COMPUTE_DRIVER_ANNOTATION`, which is the sandbox-authentication binding and +/// is written only for drivers that authenticate sandboxes, this is recorded +/// for every sandbox so that a gateway whose compute driver was changed leaves +/// the other driver's records alone. +pub const CREATED_BY_DRIVER_ANNOTATION: &str = "internal.openshell.ai/created-by-compute-driver"; /// Durable intent of an in-flight `Starting` transition that resumes retained /// state. Bound to the runtime generation it was written for so retries and /// gateway-restart recovery send the same driver request, and so a stale value @@ -1244,6 +1250,53 @@ impl ComputeRuntime { Ok(()) } + /// The configured compute driver that owns `sandbox`, when that is not this + /// gateway's driver. Records that name no creating driver (created before + /// it was recorded) fall back to the sandbox-authentication binding and + /// otherwise count as this driver's. + fn other_driver_owner<'a>(&self, sandbox: &'a Sandbox) -> Option<&'a str> { + let annotations = &sandbox.metadata.as_ref()?.annotations; + annotations + .get(CREATED_BY_DRIVER_ANNOTATION) + .or_else(|| annotations.get(COMPUTE_DRIVER_ANNOTATION)) + .map(String::as_str) + .filter(|owner| !owner.is_empty() && *owner != self.configured_driver_name()) + } + + /// Add a read-only availability condition to an API response. The + /// persisted phase and conditions must be available again after switching + /// back to the sandbox's creating driver. + pub(crate) fn project_driver_availability(&self, sandbox: &mut Sandbox) { + if let Some(owner) = self.other_driver_owner(sandbox) { + let message = format!( + "Created on compute driver '{owner}'; this gateway runs '{}'", + self.configured_driver_name() + ); + upsert_ready_condition( + &mut sandbox.status, + SandboxCondition { + r#type: "Ready".to_string(), + status: "False".to_string(), + reason: "ComputeDriverMismatch".to_string(), + message, + transition_time: None, + }, + ); + } + } + + /// Refuses lifecycle work on a sandbox another compute driver created: this + /// gateway's driver does not own its resources. + pub(crate) fn validate_compute_driver_binding(&self, sandbox: &Sandbox) -> Result<(), Status> { + if let Some(owner) = self.other_driver_owner(sandbox) { + return Err(Status::failed_precondition(format!( + "sandbox was created by compute driver '{owner}', but this gateway uses '{}'; switch the gateway back to '{owner}' to use it, or delete it", + self.configured_driver_name() + ))); + } + Ok(()) + } + fn driver_sandbox_for_launch(&self, sandbox: &Sandbox) -> Result { let mut driver_sandbox = driver_sandbox_from_public(sandbox, &self.driver_info.name) .map_err(|status| *status)?; @@ -1259,6 +1312,7 @@ impl ComputeRuntime { launch_authentication: Vec, expected_runtime_identity: String, ) -> Result { + self.validate_compute_driver_binding(sandbox)?; self.validate_isolation_backend_binding(sandbox)?; if self.isolation_backend.is_some() { self.validate_launch_signer_configured(!launch_authentication.is_empty())?; @@ -1621,6 +1675,10 @@ impl ComputeRuntime { ISOLATION_BACKEND_ANNOTATION.into(), self.isolation_backend_name().into(), ); + metadata.annotations.insert( + CREATED_BY_DRIVER_ANNOTATION.into(), + self.configured_driver_name().into(), + ); // Strip the staging token from the public sandbox before anything // persists it: the object store copy is readable by every member of the @@ -2042,7 +2100,6 @@ impl ComputeRuntime { workspace: &str, name: &str, ) -> Result { - self.require_suspend_resume()?; self.quiesce_sandbox(QuiesceOperation::Suspend, workspace, name) .await } @@ -2075,9 +2132,12 @@ impl ComputeRuntime { "sandbox name changed while the {verb} request was waiting; retry explicitly" ))); } - let mut attempt = 1; let (previous, transition) = loop { + self.validate_compute_driver_binding(¤t)?; + if operation == QuiesceOperation::Suspend { + self.require_suspend_resume()?; + } provisioning_operation::ensure_operation_settled(¤t)?; match operation.admit(¤t)? { QuiesceAdmission::Settled => { @@ -2332,6 +2392,7 @@ impl ComputeRuntime { .await .map_err(|e| Status::internal(format!("fetch sandbox failed: {e}")))? .ok_or_else(|| Status::not_found("sandbox not found"))?; + self.validate_compute_driver_binding(&candidate)?; self.validate_isolation_backend_binding(&candidate)?; if self.isolation_backend.is_some() { self.validate_launch_signer_configured(authority.is_some())?; @@ -3445,25 +3506,38 @@ impl ComputeRuntime { self.sandbox_watch_bus.notify(&target.sandbox_id); drop(guard); - let result = self - .driver - .call( - openshell_otel::rpc::DELETE_SANDBOX, - Some(transition.deleting.object_id()), - |driver| { - let sandbox_id = transition.deleting.object_id().to_string(); - let sandbox_name = transition.deleting.object_name().to_string(); - async move { - driver - .delete_sandbox(Request::new(DeleteSandboxRequest { - sandbox_id, - name: sandbox_name, - })) - .await - } - }, - ) - .await; + let result = if let Some(owner) = self.other_driver_owner(&transition.deleting) { + // The active driver does not own this sandbox's resources, so do + // not hand it an ID it never created. Remove only the record. + info!( + sandbox_id = %target.sandbox_id, + owner_driver = owner, + driver = self.configured_driver_name(), + "Deleting the record of a sandbox created by another compute driver; resources held by that driver are not released" + ); + Ok(tonic::Response::new( + openshell_core::proto::compute::v1::DeleteSandboxResponse { deleted: false }, + )) + } else { + self.driver + .call( + openshell_otel::rpc::DELETE_SANDBOX, + Some(transition.deleting.object_id()), + |driver| { + let sandbox_id = transition.deleting.object_id().to_string(); + let sandbox_name = transition.deleting.object_name().to_string(); + async move { + driver + .delete_sandbox(Request::new(DeleteSandboxRequest { + sandbox_id, + name: sandbox_name, + })) + .await + } + }, + ) + .await + }; match result { Ok(response) => { @@ -3522,22 +3596,27 @@ impl ComputeRuntime { .map_err(|e| Status::internal(format!("fetch sandbox failed: {e}")))? .ok_or_else(|| Status::not_found("sandbox not found"))?, }; - // Explicit deletion can supersede a configuration update: the - // driver only patches an existing immutable resource and cannot - // recreate it after deletion. Keep the interlock until removal so - // a failed delete can safely restore the pending update. - let pending_update = self - .store - .get_sandbox_image_state(sandbox_id) - .await - .map_err(|_| Status::unavailable("cannot read pending sandbox update"))? - .and_then(|state| state.pending); - if failed_create.is_none() - && let Some(intent) = pending_update - { - image_update::ensure_image_update_interlock(&sandbox, &intent)?; - } else { - provisioning_operation::ensure_operation_settled(&sandbox)?; + // A foreign driver's pending operation cannot be settled by this + // gateway. Explicit deletion only removes the local record. + // Failed-create compensation still owns its original operation. + if failed_create.is_some() || self.other_driver_owner(&sandbox).is_none() { + // Explicit deletion can supersede a configuration update: the + // driver only patches an existing immutable resource and cannot + // recreate it after deletion. Keep the interlock until removal so + // a failed delete can safely restore the pending update. + let pending_update = self + .store + .get_sandbox_image_state(sandbox_id) + .await + .map_err(|_| Status::unavailable("cannot read pending sandbox update"))? + .and_then(|state| state.pending); + if failed_create.is_none() + && let Some(intent) = pending_update + { + image_update::ensure_image_update_interlock(&sandbox, &intent)?; + } else { + provisioning_operation::ensure_operation_settled(&sandbox)?; + } } // Failed-create compensation owns only its original attempt. @@ -3694,9 +3773,10 @@ impl ComputeRuntime { let mut delete_resource_version = record.resource_version; let mut sandbox = decode_sandbox_record(&record)?; - if self - .pending_driver_operation_blocks_cleanup(&sandbox) - .await? + if self.other_driver_owner(&sandbox).is_none() + && self + .pending_driver_operation_blocks_cleanup(&sandbox) + .await? { return Ok(false); } @@ -4092,6 +4172,9 @@ impl ComputeRuntime { } }; + if self.other_driver_owner(&sandbox).is_some() { + return (0, 0); + } let phase = SandboxPhase::try_from(sandbox.phase()).unwrap_or(SandboxPhase::Unknown); // A driver stop discards retained suspend state. An unresolved @@ -4254,6 +4337,15 @@ impl ComputeRuntime { if !sandbox_phase_should_be_running(phase) && !recoverable_error { continue; } + if let Some(owner) = self.other_driver_owner(&sandbox) { + info!( + sandbox_id, + owner_driver = owner, + driver = self.configured_driver_name(), + "Not starting sandbox created by another compute driver" + ); + continue; + } if let Err(error) = self.validate_isolation_backend_binding(&sandbox) { warn!(sandbox_id, %error, "Rejected recovered isolation backend selection"); failed += 1; @@ -4494,6 +4586,9 @@ impl ComputeRuntime { continue; } }; + if self.other_driver_owner(&sandbox).is_some() { + continue; + } if self .pending_driver_operation_blocks_cleanup(&sandbox) .await? @@ -4970,6 +5065,9 @@ impl ComputeRuntime { continue; } }; + if self.other_driver_owner(&sandbox).is_some() { + continue; + } let phase = SandboxPhase::try_from(sandbox.phase()).unwrap_or(SandboxPhase::Unknown); let due = sandbox @@ -5016,6 +5114,9 @@ impl ComputeRuntime { else { return Ok(()); }; + if self.other_driver_owner(¤t).is_some() { + return Ok(()); + } let phase = SandboxPhase::try_from(current.phase()).unwrap_or(SandboxPhase::Unknown); let due = current .status @@ -5753,6 +5854,15 @@ impl ComputeRuntime { terminal_delivery_finalized: bool, ) -> Result<(), String> { let _guard = self.lock_sandbox_local(sandbox_id).await; + let existing = self + .store + .get_message::(sandbox_id) + .await + .map_err(|error| error.to_string())?; + if let Some(sandbox) = existing.as_ref() { + self.validate_compute_driver_binding(sandbox) + .map_err(|error| error.message().to_string())?; + } // A replacement session may already belong to another gateway. Do not // let cleanup from this replica overwrite the replacement's Ready state. @@ -5760,11 +5870,6 @@ impl ComputeRuntime { return Ok(()); } - let existing = self - .store - .get_message::(sandbox_id) - .await - .map_err(|error| error.to_string())?; self.set_supervisor_session_state_from_snapshot( sandbox_id, SupervisorSessionStateUpdate::Disconnected { @@ -5863,6 +5968,8 @@ impl ComputeRuntime { let Some(current) = existing else { return Ok(()); }; + self.validate_compute_driver_binding(¤t) + .map_err(|error| error.message().to_string())?; let current_phase = SandboxPhase::try_from(current.phase()).unwrap_or(SandboxPhase::Unknown); if connected @@ -6026,6 +6133,8 @@ impl ComputeRuntime { else { return Ok(()); }; + self.validate_compute_driver_binding(&existing) + .map_err(|error| error.message().to_string())?; if provisioning_deadline::timed_out(&existing) { return Ok(()); } @@ -6123,6 +6232,8 @@ impl ComputeRuntime { else { return Ok(()); }; + self.validate_compute_driver_binding(&sandbox) + .map_err(|error| error.message().to_string())?; let phase = SandboxPhase::try_from(sandbox.phase()).unwrap_or(SandboxPhase::Unknown); if matches!( phase, @@ -6188,6 +6299,8 @@ impl ComputeRuntime { else { return Ok(()); }; + self.validate_compute_driver_binding(&sandbox) + .map_err(|error| error.message().to_string())?; let phase = SandboxPhase::try_from(sandbox.phase()).unwrap_or(SandboxPhase::Unknown); if phase != SandboxPhase::Completed && !is_failed_main_process_result(&sandbox) { return Ok(()); @@ -6271,6 +6384,7 @@ impl ComputeRuntime { sandbox_resource_version(sandbox), ) .await? + && self.other_driver_owner(sandbox).is_none() { self.spawn_driver_sandbox_cleanup(sandbox.object_id(), sandbox.object_name()); } @@ -6621,6 +6735,19 @@ impl ComputeRuntime { } let sandbox = decode_sandbox_record(¤t_record)?; + if let Some(owner) = self.other_driver_owner(&sandbox) { + // The active driver cannot list what another driver created, so + // its absence from the snapshot says nothing about the + // sandbox. Keep the record unchanged for its own driver. + debug!( + sandbox_id = %sandbox.object_id(), + sandbox_name = %sandbox.object_name(), + owner_driver = owner, + driver = self.configured_driver_name(), + "Keeping sandbox created by another compute driver" + ); + return Ok(()); + } if self .pending_driver_operation_blocks_cleanup(&sandbox) .await? @@ -15321,6 +15448,536 @@ mod tests { ); } + fn sandbox_record_created_by( + id: &str, + name: &str, + phase: SandboxPhase, + driver_name: &str, + ) -> Sandbox { + let mut sandbox = sandbox_record(id, name, phase); + sandbox.metadata.as_mut().unwrap().annotations.insert( + CREATED_BY_DRIVER_ANNOTATION.to_string(), + driver_name.to_string(), + ); + sandbox + } + + async fn assert_no_driver_delete(driver: &ControlledDriver) { + assert!( + tokio::time::timeout(Duration::from_millis(100), driver.delete_started.notified()) + .await + .is_err(), + "no driver delete may be sent" + ); + assert_eq!(driver.delete_calls(), 0); + } + + #[tokio::test] + async fn prune_missing_sandbox_keeps_records_of_another_compute_driver() { + // Regression test for #4769: after the gateway's compute driver is + // changed (vm -> docker), the new driver cannot list what the old one + // created. Those records must survive the sweep unchanged. + let driver = ControlledDriver::new(); + driver.set_get_outcome(ControlledGetOutcome::Missing); + let runtime = test_runtime_for_driver(driver.clone(), "docker").await; + let kept = [ + ("sb-stopped", SandboxPhase::Stopped), + ("sb-error", SandboxPhase::Error), + ("sb-provisioning", SandboxPhase::Provisioning), + ]; + for (id, phase) in kept { + runtime + .store + .put_message(&sandbox_record_created_by(id, id, phase, "vm")) + .await + .unwrap(); + } + // Records created before the creating driver was recorded still name + // their driver through the sandbox-authentication binding. + let mut bound = sandbox_record("sb-bound", "sb-bound", SandboxPhase::Stopped); + bound.metadata.as_mut().unwrap().annotations.insert( + COMPUTE_DRIVER_ANNOTATION.to_string(), + "kubernetes".to_string(), + ); + runtime.store.put_message(&bound).await.unwrap(); + + // Without the fix the first sweep moves Stopped to Error and the + // second removes the record and sends DeleteSandbox. + for _ in 0..2 { + runtime + .reconcile_store_with_backend(Duration::ZERO) + .await + .unwrap(); + } + + for (id, phase) in kept + .into_iter() + .chain([("sb-bound", SandboxPhase::Stopped)]) + { + let stored = runtime + .store + .get_message::(id) + .await + .unwrap() + .unwrap_or_else(|| panic!("{id} must be kept")); + assert_eq!(SandboxPhase::try_from(stored.phase()).unwrap(), phase); + assert!( + stored + .status + .as_ref() + .is_none_or(|status| status.conditions.is_empty()), + "{id} must not gain a condition" + ); + } + assert_no_driver_delete(&driver).await; + } + + #[tokio::test] + async fn prune_missing_sandbox_still_prunes_records_of_the_configured_driver() { + let driver = ControlledDriver::new(); + driver.set_get_outcome(ControlledGetOutcome::Missing); + let runtime = test_runtime_for_driver(driver.clone(), "docker").await; + let sandbox = + sandbox_record_created_by("sb-1", "sandbox-a", SandboxPhase::Stopped, "docker"); + runtime.store.put_message(&sandbox).await.unwrap(); + + runtime + .reconcile_store_with_backend(Duration::ZERO) + .await + .unwrap(); + let stored = runtime + .store + .get_message::("sb-1") + .await + .unwrap() + .unwrap(); + assert_eq!( + SandboxPhase::try_from(stored.phase()).unwrap(), + SandboxPhase::Error + ); + + runtime + .reconcile_store_with_backend(Duration::ZERO) + .await + .unwrap(); + assert!( + runtime + .store + .get_message::("sb-1") + .await + .unwrap() + .is_none() + ); + tokio::time::timeout(Duration::from_secs(1), driver.delete_started.notified()) + .await + .expect("background driver cleanup did not run"); + assert_eq!( + driver.delete_requests(), + vec![("sb-1".to_string(), "sandbox-a".to_string())] + ); + } + + #[tokio::test] + async fn prune_missing_sandbox_treats_records_without_a_driver_as_the_configured_drivers() { + // Records written before the creating driver was recorded keep the + // earlier behaviour: a missing retained resource moves to Error. + let driver = ControlledDriver::new(); + driver.set_get_outcome(ControlledGetOutcome::Missing); + let runtime = test_runtime_for_driver(driver.clone(), "docker").await; + let sandbox = sandbox_record("sb-1", "sandbox-a", SandboxPhase::Stopped); + runtime.store.put_message(&sandbox).await.unwrap(); + + runtime + .reconcile_store_with_backend(Duration::ZERO) + .await + .unwrap(); + + let stored = runtime + .store + .get_message::("sb-1") + .await + .unwrap() + .unwrap(); + assert_eq!( + SandboxPhase::try_from(stored.phase()).unwrap(), + SandboxPhase::Error + ); + let ready = stored + .status + .as_ref() + .and_then(|status| status.conditions.iter().find(|c| c.r#type == "Ready")) + .expect("Ready condition present"); + assert_eq!(ready.reason, "ComputeResourceMissing"); + } + + #[tokio::test] + async fn foreign_driver_deadlines_leave_records_unchanged() { + let driver = ControlledDriver::new(); + let runtime = test_runtime_for_driver(driver.clone(), "docker").await; + let now = openshell_core::time::now_ms(); + for (id, phase) in [ + ("sb-provisioning", SandboxPhase::Provisioning), + ("sb-starting", SandboxPhase::Starting), + ] { + let mut sandbox = sandbox_record_created_by(id, id, phase, "vm"); + sandbox.status.as_mut().unwrap().provisioning = + Some(provisioning_deadline::new_preparation_record(now, 1)); + runtime.store.put_message(&sandbox).await.unwrap(); + } + let mut timed_out = + sandbox_record_created_by("sb-timed-out", "timed-out", SandboxPhase::Error, "vm"); + let mut deadline = provisioning_deadline::new_preparation_record(now, 1); + deadline.timeout_time = openshell_core::time::timestamp_from_millis(now).ok(); + timed_out.status.as_mut().unwrap().provisioning = Some(deadline); + runtime.store.put_message(&timed_out).await.unwrap(); + + let mut originals = Vec::new(); + for id in ["sb-provisioning", "sb-starting", "sb-timed-out"] { + let original = runtime + .store + .get_message::(id) + .await + .unwrap() + .unwrap(); + originals.push(original); + } + + runtime + .reconcile_provisioning_deadlines(now + 1_000) + .await + .unwrap(); + for original in &originals { + let current = runtime + .store + .get_message::(original.object_id()) + .await + .unwrap() + .unwrap(); + assert_eq!(¤t, original); + } + let gate = runtime.lifecycle_gates.lock_for("sb-provisioning").await; + let guard = runtime.lock_sandbox_for_lifecycle(&gate).await; + assert!( + runtime + .claim_provisioning_timeout(&originals[0], now + 1_000) + .await + .unwrap() + .is_none() + ); + drop(guard); + drop(gate); + let gate = runtime.lifecycle_gates.lock_for("sb-timed-out").await; + runtime + .reclaim_provisioning_timeout(&originals[2], &gate) + .await + .unwrap(); + let current = runtime + .store + .get_message::("sb-timed-out") + .await + .unwrap() + .unwrap(); + assert_eq!(current, originals[2]); + assert_eq!(driver.stop_calls(), 0); + } + + #[tokio::test] + async fn foreign_driver_due_restart_never_claims_or_stops() { + let driver = ControlledDriver::new(); + let runtime = test_runtime_for_driver(driver.clone(), "docker").await; + let mut sandbox = sandbox_record_created_by( + "sb-foreign-restart", + "foreign-restart", + SandboxPhase::Starting, + "vm", + ); + sandbox.spec = Some(SandboxSpec { + restart_policy: SandboxRestartPolicy::OnFailure as i32, + ..Default::default() + }); + sandbox.status = Some(SandboxStatus { + phase: SandboxPhase::Starting as i32, + exit_code: Some(9), + restart_count: 1, + next_restart_time: restart_timestamp(openshell_core::time::now_ms() - 1), + ..Default::default() + }); + runtime.store.put_message(&sandbox).await.unwrap(); + let original = runtime + .store + .get_message::(sandbox.object_id()) + .await + .unwrap() + .unwrap(); + + runtime.restart_due_sandboxes().await.unwrap(); + runtime + .restart_sandbox_runtime(sandbox.object_id()) + .await + .unwrap(); + + let current = runtime + .store + .get_message::(sandbox.object_id()) + .await + .unwrap() + .unwrap(); + assert_eq!(current, original); + assert_eq!(driver.stop_calls(), 0); + assert_eq!(driver.start_calls(), 0); + } + + #[tokio::test] + async fn foreign_driver_shutdown_and_recovery_skip_quiesce() { + let driver = ControlledDriver::new(); + let runtime = test_runtime_with_gateway_managed_lifecycle(driver.clone(), "docker").await; + for (id, phase) in [ + ("sb-ready", SandboxPhase::Ready), + ("sb-stopping", SandboxPhase::Stopping), + ("sb-suspending", SandboxPhase::Suspending), + ] { + runtime + .store + .put_message(&sandbox_record_created_by(id, id, phase, "vm")) + .await + .unwrap(); + } + let mut originals = Vec::new(); + for id in ["sb-ready", "sb-stopping", "sb-suspending"] { + originals.push( + runtime + .store + .get_message::(id) + .await + .unwrap() + .unwrap(), + ); + } + + runtime + .stop_persisted_sandboxes_on_shutdown() + .await + .unwrap(); + runtime + .recover_persisted_lifecycle_transitions() + .await + .unwrap(); + + for original in originals { + let current = runtime + .store + .get_message::(original.object_id()) + .await + .unwrap() + .unwrap(); + assert_eq!(current, original); + } + assert_eq!(driver.stop_calls(), 0); + assert_eq!(driver.suspend_calls(), 0); + } + + #[tokio::test] + async fn create_sandbox_records_the_creating_compute_driver() { + let runtime = test_runtime_for_driver(Arc::new(TestDriver::default()), "vm").await; + let sandbox = sandbox_record("sb-1", "sandbox-a", SandboxPhase::Provisioning); + + runtime.create_sandbox(sandbox, None, false).await.unwrap(); + + let stored = runtime + .store + .get_message::("sb-1") + .await + .unwrap() + .unwrap(); + let annotations = &stored.metadata.as_ref().unwrap().annotations; + assert_eq!( + annotations + .get(CREATED_BY_DRIVER_ANNOTATION) + .map(String::as_str), + Some("vm") + ); + // The sandbox-authentication binding is unchanged for drivers that do + // not authenticate sandboxes. + assert!(!annotations.contains_key(COMPUTE_DRIVER_ANNOTATION)); + } + + #[tokio::test] + async fn delete_sandbox_of_another_compute_driver_removes_only_the_record() { + let driver = ControlledDriver::new(); + let runtime = test_runtime_for_driver(driver.clone(), "docker").await; + let sandbox = sandbox_record_created_by("sb-1", "sandbox-a", SandboxPhase::Stopped, "vm"); + runtime.store.put_message(&sandbox).await.unwrap(); + runtime.sandbox_index.update_from_sandbox(&sandbox); + + let result = runtime + .delete_sandbox("default", "sandbox-a") + .await + .unwrap(); + + assert_eq!( + result.outcome, + openshell_core::proto::DeletionOutcome::Completed + ); + assert!( + runtime + .store + .get_message::("sb-1") + .await + .unwrap() + .is_none() + ); + assert_no_driver_delete(&driver).await; + } + + #[tokio::test] + async fn foreign_driver_pending_operation_does_not_block_record_delete() { + let driver = ControlledDriver::new(); + let runtime = test_runtime_for_driver(driver.clone(), "docker").await; + let mut sandbox = + sandbox_record_created_by("sb-1", "sandbox-a", SandboxPhase::Starting, "vm"); + let mut operation = + provisioning_deadline::new_preparation_record(openshell_core::time::now_ms(), 60); + operation.driver_operation_pending = true; + sandbox.status.as_mut().unwrap().provisioning = Some(operation); + runtime.store.put_message(&sandbox).await.unwrap(); + runtime.sandbox_index.update_from_sandbox(&sandbox); + let stored = runtime + .store + .get_message::("sb-1") + .await + .unwrap() + .unwrap(); + + let error = runtime + .delete_sandbox_with_preconditions( + "default", + "sandbox-a", + SandboxDeletePreconditions { + expected_sandbox_id: Some("sb-1".to_string()), + expected_resource_version: Some(sandbox_resource_version(&stored) + 1), + }, + ) + .await + .unwrap_err(); + assert_eq!(error.code(), Code::Aborted); + assert_eq!( + runtime.store.get_message::("sb-1").await.unwrap(), + Some(stored.clone()), + ); + + let result = runtime + .delete_sandbox_with_preconditions( + "default", + "sandbox-a", + SandboxDeletePreconditions { + expected_sandbox_id: Some("sb-1".to_string()), + expected_resource_version: Some(sandbox_resource_version(&stored)), + }, + ) + .await + .unwrap(); + assert_eq!( + result.outcome, + openshell_core::proto::DeletionOutcome::Completed + ); + assert!( + runtime + .store + .get_message::("sb-1") + .await + .unwrap() + .is_none() + ); + assert_no_driver_delete(&driver).await; + } + + #[tokio::test] + async fn start_persisted_sandboxes_skips_records_of_another_compute_driver() { + let driver = ControlledDriver::new(); + let runtime = test_runtime_with_gateway_managed_lifecycle(driver.clone(), "docker").await; + let sandbox = sandbox_record_created_by("sb-1", "sandbox-a", SandboxPhase::Ready, "vm"); + runtime.store.put_message(&sandbox).await.unwrap(); + + runtime.start_persisted_sandboxes().await.unwrap(); + + assert_eq!(driver.start_calls(), 0); + let stored = runtime + .store + .get_message::("sb-1") + .await + .unwrap() + .unwrap(); + assert_eq!( + SandboxPhase::try_from(stored.phase()).unwrap(), + SandboxPhase::Ready + ); + } + + #[tokio::test] + async fn start_sandbox_of_another_compute_driver_fails_precondition() { + let driver = ControlledDriver::new(); + let runtime = test_runtime_for_driver(driver.clone(), "docker").await; + let sandbox = sandbox_record_created_by("sb-1", "sandbox-a", SandboxPhase::Stopped, "vm"); + runtime.store.put_message(&sandbox).await.unwrap(); + runtime.sandbox_index.update_from_sandbox(&sandbox); + + let error = runtime + .start_sandbox("default", "sandbox-a") + .await + .unwrap_err(); + + assert_eq!(error.code(), Code::FailedPrecondition); + assert!(error.message().contains("compute driver 'vm'")); + assert_eq!(driver.start_calls(), 0); + let stored = runtime + .store + .get_message::("sb-1") + .await + .unwrap() + .unwrap(); + assert_eq!( + SandboxPhase::try_from(stored.phase()).unwrap(), + SandboxPhase::Stopped + ); + } + + #[tokio::test] + async fn stop_and_suspend_foreign_sandbox_fail_before_driver_calls() { + let driver = ControlledDriver::new(); + let runtime = test_runtime_for_driver(driver.clone(), "docker").await; + let sandbox = sandbox_record_created_by("sb-1", "sandbox-a", SandboxPhase::Ready, "vm"); + runtime.store.put_message(&sandbox).await.unwrap(); + let original = runtime + .store + .get_message::(sandbox.object_id()) + .await + .unwrap() + .unwrap(); + + let stop = runtime + .stop_sandbox("default", "sandbox-a") + .await + .unwrap_err(); + let suspend = runtime + .suspend_sandbox("default", "sandbox-a") + .await + .unwrap_err(); + + assert_eq!(stop.code(), Code::FailedPrecondition); + assert_eq!(suspend.code(), Code::FailedPrecondition); + assert!(stop.message().contains("compute driver 'vm'")); + assert!(suspend.message().contains("compute driver 'vm'")); + let current = runtime + .store + .get_message::(sandbox.object_id()) + .await + .unwrap() + .unwrap(); + assert_eq!(current, original); + assert_eq!(driver.stop_calls(), 0); + assert_eq!(driver.suspend_calls(), 0); + } + #[tokio::test] async fn prune_sweep_does_not_block_on_a_stuck_driver_delete_call() { // Regression test: the prune sweep's driver cleanup must not be @@ -17340,7 +17997,7 @@ mod tests { let driver = ControlledDriver::new(); driver.set_runtime_identity("new-runtime-identity"); let mut runtime = - test_runtime_with_gateway_managed_lifecycle(driver.clone(), "arbitrary").await; + test_runtime_with_gateway_managed_lifecycle(driver.clone(), "test-driver").await; enable_runtime_identity_binding(&mut runtime); let mut sandbox = sandbox_record("sb-1", "sandbox", SandboxPhase::Ready); set_compute_runtime_binding(&mut sandbox, "previous-runtime-identity"); diff --git a/crates/openshell-server/src/compute/provisioning_deadline.rs b/crates/openshell-server/src/compute/provisioning_deadline.rs index cb351e64d0..80a25cc5c5 100644 --- a/crates/openshell-server/src/compute/provisioning_deadline.rs +++ b/crates/openshell-server/src/compute/provisioning_deadline.rs @@ -470,6 +470,9 @@ impl super::ComputeRuntime { for record in records { let candidate = Sandbox::decode(record.payload.as_slice()).map_err(|error| error.to_string())?; + if self.other_driver_owner(&candidate).is_some() { + continue; + } if !matches!( SandboxPhase::try_from(candidate.phase()), Ok(SandboxPhase::Provisioning | SandboxPhase::Starting) @@ -517,6 +520,9 @@ impl super::ComputeRuntime { else { continue; }; + if self.other_driver_owner(¤t).is_some() { + continue; + } let previous = current.clone(); refresh_configuration(&self.store, &mut current, now_ms).await?; if current != previous { @@ -574,6 +580,9 @@ impl super::ComputeRuntime { ) -> Result, String> { use openshell_core::ObjectId; use openshell_core::proto::{Sandbox, SandboxCondition, SandboxPhase}; + if self.other_driver_owner(current).is_some() { + return Ok(None); + } if !matches!( SandboxPhase::try_from(current.phase()), Ok(SandboxPhase::Provisioning | SandboxPhase::Starting) @@ -668,6 +677,9 @@ impl super::ComputeRuntime { let Some(current) = current else { return Ok(()); }; + if self.other_driver_owner(¤t).is_some() { + return Ok(()); + } let expected_attempt = expired .status .as_ref() diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index 279329c666..19314dc067 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -5062,6 +5062,7 @@ pub(super) async fn handle_report_sandbox_configuration( .await .map_err(|e| Status::internal(format!("fetch sandbox failed: {e}")))? .ok_or_else(|| Status::not_found("sandbox not found"))?; + state.compute.validate_compute_driver_binding(&sandbox)?; let sandbox_name = sandbox.object_name().to_string(); let workspace = sandbox.object_workspace().to_string(); crate::compute::provisioning_deadline::refresh_configuration( @@ -5263,6 +5264,13 @@ pub async fn record_policy_apply_result( .collect::() }); let version_i64 = i64::from(version); + let sandbox = state + .store + .get_message::(sandbox_id) + .await + .map_err(|error| Status::internal(format!("fetch sandbox failed: {error}")))? + .ok_or_else(|| Status::not_found("sandbox not found"))?; + state.compute.validate_compute_driver_binding(&sandbox)?; let updated = state .store .update_policy_status( diff --git a/crates/openshell-server/src/grpc/policy/endpoint_status.rs b/crates/openshell-server/src/grpc/policy/endpoint_status.rs index 680aa369d6..3a781786f7 100644 --- a/crates/openshell-server/src/grpc/policy/endpoint_status.rs +++ b/crates/openshell-server/src/grpc/policy/endpoint_status.rs @@ -159,6 +159,7 @@ async fn handle_report_endpoint_status_inner( .await .map_err(|error| Status::internal(format!("fetch sandbox failed: {error}")))? .ok_or_else(|| Status::not_found("sandbox not found"))?; + state.compute.validate_compute_driver_binding(&sandbox)?; let context = active_endpoint_context(state.as_ref(), &sandbox).await?; if req.policy_hash != context.policy_hash || req.provider_env_revision != context.provider_env_revision diff --git a/crates/openshell-server/src/grpc/provider_readiness.rs b/crates/openshell-server/src/grpc/provider_readiness.rs index 8a690c6c74..0313cb0a0e 100644 --- a/crates/openshell-server/src/grpc/provider_readiness.rs +++ b/crates/openshell-server/src/grpc/provider_readiness.rs @@ -662,6 +662,7 @@ async fn handle_report_provider_readiness_inner( .await .map_err(|_| registry_unavailable())? .ok_or_else(|| Status::not_found("sandbox not found"))?; + state.compute.validate_compute_driver_binding(&sandbox)?; if sandbox.phase() != SandboxPhase::Ready as i32 { return Err(Status::failed_precondition( "sandbox supervisor is not ready", diff --git a/crates/openshell-server/src/grpc/sandbox.rs b/crates/openshell-server/src/grpc/sandbox.rs index a797da2589..7e29581cfd 100644 --- a/crates/openshell-server/src/grpc/sandbox.rs +++ b/crates/openshell-server/src/grpc/sandbox.rs @@ -1036,7 +1036,7 @@ pub(super) async fn handle_get_sandbox( let principal = super::extract_principal(&request)?; let wants_owner = owner_requested(&request); let req = request.into_inner(); - let sandbox = resolve_and_authorize_sandbox_name( + let mut sandbox = resolve_and_authorize_sandbox_name( state, &principal, &req.name, @@ -1049,6 +1049,7 @@ pub(super) async fn handle_get_sandbox( } else { None }; + state.compute.project_driver_availability(&mut sandbox); let mut response = Response::new(SandboxResponse { sandbox: Some(sandbox), service_urls: HashMap::new(), @@ -1112,8 +1113,12 @@ pub(super) async fn handle_list_sandboxes( .await .map_err(|e| Status::internal(format!("list sandboxes failed: {e}")))?; let next_page_token = pagination.next_object_token(page.next_cursor.as_ref()); + let mut sandboxes = page.messages; + for sandbox in &mut sandboxes { + state.compute.project_driver_availability(sandbox); + } Ok(Response::new(ListSandboxesResponse { - sandboxes: page.messages, + sandboxes, next_page_token, })) } @@ -2219,11 +2224,15 @@ pub(super) async fn handle_watch_sandbox( match state.store.get_message::(&sandbox_id).await { Ok(Some(sandbox)) => { state.sandbox_index.update_from_sandbox(&sandbox); + let mut response_sandbox = sandbox.clone(); + state + .compute + .project_driver_availability(&mut response_sandbox); let _ = tx .send(Ok(SandboxStreamEvent { payload: Some( openshell_core::proto::sandbox_stream_event::Payload::Sandbox( - sandbox.clone(), + response_sandbox, ), ), // Status snapshots are re-read, not resumed by cursor. @@ -2503,7 +2512,9 @@ pub(super) async fn handle_watch_sandbox( match state.store.get_message::(&sandbox_id).await { Ok(Some(sandbox)) => { state.sandbox_index.update_from_sandbox(&sandbox); - if tx.send(Ok(SandboxStreamEvent { payload: Some(openshell_core::proto::sandbox_stream_event::Payload::Sandbox(sandbox.clone())), cursor: String::new() })).await.is_err() { + let mut response_sandbox = sandbox.clone(); + state.compute.project_driver_availability(&mut response_sandbox); + if tx.send(Ok(SandboxStreamEvent { payload: Some(openshell_core::proto::sandbox_stream_event::Payload::Sandbox(response_sandbox)), cursor: String::new() })).await.is_err() { return; } if stop_on_terminal { @@ -2738,6 +2749,7 @@ pub(super) async fn handle_exec_sandbox( completion.ensure_target(sandbox.object_id())?; } + state.compute.validate_compute_driver_binding(&sandbox)?; require_ready_sandbox(&sandbox)?; // Open a relay channel through the supervisor session. Use a 15s @@ -2868,6 +2880,7 @@ pub(super) async fn handle_forward_tcp( MinWorkspaceRole::User, ) .await?; + state.compute.validate_compute_driver_binding(&sandbox)?; // The main process may finish between minting the SSH token and opening // its transport. Keep the relay reachable until terminal delivery is @@ -3317,6 +3330,7 @@ pub(super) async fn handle_exec_sandbox_interactive_start( completion.ensure_target(sandbox.object_id())?; } + state.compute.validate_compute_driver_binding(&sandbox)?; require_ready_sandbox(&sandbox)?; let (channel_id, relay_rx) = crate::supervisor_session::open_routed_relay_with_target( @@ -3407,6 +3421,7 @@ pub(super) async fn handle_create_ssh_session( ) .await?; let sandbox_id = sandbox.object_id().to_string(); + state.compute.validate_compute_driver_binding(&sandbox)?; if !sandbox_relay_reachable(state, &sandbox) { return Err(Status::failed_precondition("sandbox is not ready")); @@ -10161,6 +10176,186 @@ mod tests { ); } + #[tokio::test] + async fn foreign_driver_blocks_ssh_session_and_tcp_forward() { + use openshell_core::proto::open_shell_client::OpenShellClient; + use openshell_core::proto::open_shell_server::OpenShellServer; + use tokio_stream::wrappers::TcpListenerStream; + + let state = test_server_state_with_driver("docker").await; + let mut sandbox = test_sandbox("foreign", Vec::new()); + sandbox.metadata.as_mut().unwrap().annotations.insert( + crate::compute::CREATED_BY_DRIVER_ANNOTATION.to_string(), + "vm".to_string(), + ); + state.store.put_message(&sandbox).await.unwrap(); + let stored = state + .store + .get_message::(sandbox.object_id()) + .await + .unwrap() + .unwrap(); + + let error = handle_create_ssh_session( + &state, + authed_request(CreateSshSessionRequest { + sandbox: "foreign".into(), + workspace_scope: Some(openshell_core::proto::workspace_selector("default")), + }), + ) + .await + .unwrap_err(); + assert_eq!(error.code(), tonic::Code::FailedPrecondition); + assert!(error.message().contains("compute driver 'vm'")); + + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let principal = authed_request(()) + .extensions_mut() + .remove::() + .unwrap(); + let server = tokio::spawn( + tonic::transport::Server::builder() + .add_service(OpenShellServer::with_interceptor( + crate::grpc::OpenShellService::new(state.clone()), + move |mut request: Request<()>| { + request.extensions_mut().insert(principal.clone()); + Ok(request) + }, + )) + .serve_with_incoming(TcpListenerStream::new(listener)), + ); + let mut client = OpenShellClient::connect(format!("http://{address}")) + .await + .unwrap(); + let frame = TcpForwardFrame { + payload: Some(openshell_core::proto::tcp_forward_frame::Payload::Init( + TcpForwardInit { + sandbox: "foreign".into(), + workspace: "default".into(), + target: Some(loopback_tcp_target()), + ..Default::default() + }, + )), + }; + let error = client + .forward_tcp(tokio_stream::iter([frame])) + .await + .unwrap_err(); + assert_eq!(error.code(), tonic::Code::FailedPrecondition); + assert!(error.message().contains("compute driver 'vm'")); + server.abort(); + + assert_eq!( + state + .store + .get_message::(sandbox.object_id()) + .await + .unwrap(), + Some(stored), + ); + } + + #[tokio::test] + async fn foreign_driver_responses_show_unavailable_without_changing_store() { + use tokio_stream::StreamExt as _; + + let state = test_server_state_with_driver("docker").await; + let mut sandbox = test_sandbox("foreign", Vec::new()); + sandbox.metadata.as_mut().unwrap().annotations.insert( + crate::compute::CREATED_BY_DRIVER_ANNOTATION.to_string(), + "vm".to_string(), + ); + state.store.put_message(&sandbox).await.unwrap(); + let stored = state + .store + .get_message::(sandbox.object_id()) + .await + .unwrap() + .unwrap(); + let scope = Some(openshell_core::proto::workspace_selector( + "default".to_string(), + )); + + let got = handle_get_sandbox( + &state, + authed_request(GetSandboxRequest { + name: "foreign".to_string(), + workspace_scope: scope.clone(), + }), + ) + .await + .unwrap() + .into_inner() + .sandbox + .unwrap(); + let listed = handle_list_sandboxes( + &state, + authed_request(ListSandboxesRequest { + workspace_scope: scope.clone(), + ..Default::default() + }), + ) + .await + .unwrap() + .into_inner(); + let mut watched = handle_watch_sandbox( + &state, + authed_request(WatchSandboxRequest { + sandbox: "foreign".to_string(), + workspace_scope: scope.clone(), + ..Default::default() + }), + ) + .await + .unwrap() + .into_inner(); + let event = watched.next().await.unwrap().unwrap(); + let openshell_core::proto::sandbox_stream_event::Payload::Sandbox(watch_sandbox) = + event.payload.unwrap() + else { + panic!("watch must send the sandbox snapshot first"); + }; + watched.disconnect_and_wait().await; + assert_eq!(listed.sandboxes.len(), 1); + for response in [&got, &listed.sandboxes[0], &watch_sandbox] { + assert_eq!(response.phase(), SandboxPhase::Ready as i32); + let condition = response + .status + .as_ref() + .unwrap() + .conditions + .iter() + .find(|condition| condition.r#type == "Ready") + .unwrap(); + assert_eq!(condition.status, "False"); + assert_eq!(condition.reason, "ComputeDriverMismatch"); + assert!(condition.message.contains("'vm'")); + assert!(condition.message.contains("'docker'")); + } + let current = state + .store + .get_message::(sandbox.object_id()) + .await + .unwrap() + .unwrap(); + assert_eq!(current, stored); + + let error = handle_exec_sandbox( + &state, + authed_request(ExecSandboxRequest { + sandbox: "foreign".to_string(), + command: vec!["true".to_string()], + workspace_scope: scope, + ..Default::default() + }), + ) + .await + .unwrap_err(); + assert_eq!(error.code(), tonic::Code::FailedPrecondition); + assert!(error.message().contains("compute driver 'vm'")); + } + #[tokio::test] async fn sandbox_naming_another_sandbox_is_recorded_as_cross_sandbox() { let state = test_server_state().await; diff --git a/crates/openshell-server/src/supervisor_session.rs b/crates/openshell-server/src/supervisor_session.rs index 248f5fab7b..0340d6d3e8 100644 --- a/crates/openshell-server/src/supervisor_session.rs +++ b/crates/openshell-server/src/supervisor_session.rs @@ -2552,6 +2552,7 @@ pub async fn handle_connect_supervisor( // record, so registration detects publications the bootstrap may miss. let captured_seq = pushes_config.then(|| state.config_delivery.publications()); let sandbox = require_persisted_sandbox(&state.store, &sandbox_id).await?; + state.compute.validate_compute_driver_binding(&sandbox)?; // Validate readiness identities before replacing a healthy session. let provider_readiness = ProviderReadinessEvidence::from_hello(&hello)?; @@ -3173,6 +3174,89 @@ mod tests { ); } + #[tokio::test] + async fn foreign_supervisor_reconnect_and_exit_callbacks_keep_record_unchanged() { + use crate::grpc::test_support::{ + reconnect_supervisor_stream, test_server_state_with_driver, + }; + + let state = test_server_state_with_driver("docker").await; + let mut sandbox = sandbox_record("sb-foreign", "sb-foreign"); + sandbox.set_phase(SandboxPhase::Ready as i32); + sandbox.metadata.as_mut().unwrap().annotations.insert( + crate::compute::CREATED_BY_DRIVER_ANNOTATION.to_string(), + "vm".to_string(), + ); + sandbox + .status + .get_or_insert_with(Default::default) + .main_process_instance_id = "instance".to_string(); + state.store.put_message(&sandbox).await.unwrap(); + let stored = state + .store + .get_message::("sb-foreign") + .await + .unwrap() + .unwrap(); + + let error = reconnect_supervisor_stream(&state, "sb-foreign", StreamFeatures::Legacy) + .await + .err() + .expect("foreign supervisor reconnect must be refused"); + assert_eq!(error.code(), tonic::Code::FailedPrecondition); + assert!(error.message().contains("compute driver 'vm'")); + + let error = handle_report_main_process_exit( + &state, + Request::new(ReportMainProcessExitRequest { + sandbox_id: "sb-foreign".into(), + instance_id: "instance".into(), + exit_code: 0, + }), + ) + .await + .unwrap_err(); + assert_eq!(error.code(), tonic::Code::FailedPrecondition); + assert!(error.message().contains("compute driver 'vm'")); + + let error = handle_finalize_main_process_exit( + &state, + Request::new(openshell_core::proto::FinalizeMainProcessExitRequest { + sandbox_id: "sb-foreign".into(), + instance_id: "instance".into(), + }), + ) + .await + .unwrap_err(); + assert_eq!(error.code(), tonic::Code::FailedPrecondition); + assert!(error.message().contains("compute driver 'vm'")); + + for result in [ + state + .compute + .supervisor_session_connected("sb-foreign", "instance") + .await, + state + .compute + .supervisor_session_disconnected("sb-foreign", false) + .await, + state + .compute + .cleanup_finalized_ephemeral_sandbox("sb-foreign", "instance") + .await, + ] { + assert!(result.unwrap_err().contains("compute driver 'vm'")); + } + assert_eq!( + state + .store + .get_message::("sb-foreign") + .await + .unwrap(), + Some(stored), + ); + } + use tokio::io::{AsyncReadExt, AsyncWriteExt}; async fn test_store() -> Arc { diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index 3d5680301d..f5ed27b377 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -102,6 +102,8 @@ version = 2 The gateway selector is `compute_driver = ""`. It accepts one scalar driver name. Omit it to retain auto-detection. +Changing the compute driver keeps the sandboxes the previous driver created. The gateway records the creating driver on each sandbox and leaves those sandboxes in their last phase. `openshell sandbox list` shows why those sandboxes are unavailable; start, stop, exec, and configuration updates refuse them. Switch back to the creating driver to use them again. `openshell sandbox delete` removes the gateway record only; resources that the other driver holds, such as a VM disk, stay until you delete the sandbox from a gateway that runs its driver. Legacy sandboxes without either a creating-driver annotation or a sandbox-authentication driver binding are treated as belonging to the current driver. + ## Additional Destination CAs Use the global supervisor network table to trust private certificate authorities diff --git a/docs/how-it-works/sandboxes/runtimes.mdx b/docs/how-it-works/sandboxes/runtimes.mdx index 70f6891ce1..8573893042 100644 --- a/docs/how-it-works/sandboxes/runtimes.mdx +++ b/docs/how-it-works/sandboxes/runtimes.mdx @@ -26,6 +26,7 @@ the sandbox record and driver-supported persistent storage. ## Select a Compute Driver Set `compute_driver` in the gateway TOML file. A gateway uses one driver at a time. +When changing the driver, see [Gateway Configuration File](/how-it-works/gateways/configuration) for how the gateway retains sandboxes created by the previous driver. ```toml [openshell]