Skip to content

feat(docker): scope managed resources by host UID #5136

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
vineethsai7 wants to merge 2 commits into
base: main
Choose a base branch
from
Open

feat(docker): scope managed resources by host UID #5136

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
aace71f
feat(docker): scope managed resources by host UID
vineethsai7 Nov 2, 2026
06d162c
fix(docker): scope container names and verify pending deletes
vineethsai7 Nov 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions crates/openshell-driver-docker/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,18 @@ label resolver and are denied under enforcement. GPU devices are temporarily
exempt. Inspect labels again before launch, restart, and during reconciliation.
See [resource admission configuration](../../docs/how-it-works/gateways/configuration.mdx#external-resource-admission).

`scope_by_host_uid = true` under `[openshell.drivers.docker]` derives the
driver's existing sandbox namespace from the gateway process's effective host
UID and `sandbox_label` (`uid-<uid>-<sandbox_label>`). The same namespace flows
through workload, supervisor, staging, and identity container labels; container
queries and startup cleanup; container and volume names and labels; and
private host state. The default remains unscoped for compatibility. Opt-in
fails before driver reconciliation when OpenShell containers or volumes still
have the unscoped label, avoiding an invisible legacy sandbox after a config
change. Delete old sandboxes through the original gateway, then stop that
gateway before enabling the option. The scope separates OS users through
OpenShell, but Docker daemon access can bypass it.

The driver creates two containers for each sandbox:

- `openshell-sandbox` is PID 1 in the workload container. It owns the workload
Expand Down
210 changes: 161 additions & 49 deletions crates/openshell-driver-docker/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,9 +195,13 @@ pub struct DockerComputeConfig {
/// Image pull policy for sandbox images.
pub image_pull_policy: ImagePullPolicy,

/// Value of the `openshell.sandbox_namespace` label applied to Docker sandboxes.
/// Value of the `openshell.ai/sandbox-namespace` label applied to Docker sandboxes.
pub sandbox_label: String,

/// Scope Docker-managed resources to the gateway process's host UID.
/// Enable only after removing resources in the unscoped namespace.
pub scope_by_host_uid: bool,

/// Gateway gRPC endpoint the sandbox connects back to.
pub grpc_endpoint: String,

Expand Down Expand Up @@ -323,6 +327,7 @@ impl Default for DockerComputeConfig {
default_image: openshell_core::image::default_sandbox_image(),
image_pull_policy: ImagePullPolicy::default(),
sandbox_label: "default".to_string(),
scope_by_host_uid: false,
grpc_endpoint: String::new(),
supervisor_otlp_endpoint: None,
supervisor_network_mode: DockerSupervisorNetworkMode::Host,
Expand All @@ -347,18 +352,113 @@ impl Default for DockerComputeConfig {
}
}

fn docker_sandbox_namespace(config: &DockerComputeConfig, host_uid: u32) -> String {
if config.scope_by_host_uid {
format!("uid-{host_uid}-{}", config.sandbox_label)
} else {
config.sandbox_label.clone()
}
}

async fn reject_unscoped_docker_resources(docker: &Docker, namespace: &str) -> CoreResult<()> {
let filters = managed_resource_label_filters(namespace, []);
let containers = docker
.list_containers(Some(
ListContainersOptionsBuilder::default()
.all(true)
.filters(&filters)
.build(),
))
.await
.map_err(|error| {
Error::execution(format!(
"failed to list unscoped Docker containers: {error}"
))
})?;
let volumes = docker
.list_volumes(Some(
ListVolumesOptionsBuilder::default()
.filters(&filters)
.build(),
))
.await
.map_err(|error| {
Error::execution(format!("failed to list unscoped Docker volumes: {error}"))
})?;
if !containers.is_empty() || volumes.volumes.is_some_and(|volumes| !volumes.is_empty()) {
return Err(Error::config(format!(
"scope_by_host_uid requires an empty unscoped Docker namespace '{namespace}'; remove its OpenShell containers and volumes before enabling it"
)));
}
Ok(())
}

async fn remove_owned_pending_container(
docker: &Docker,
name: &str,
sandbox_id: &str,
namespace: &str,
) -> Result<bool, Status> {
let inspected = match docker.inspect_container(name, None).await {
Ok(container) => container,
Err(error) if is_not_found_error(&error) => return Ok(false),
Err(error) => return Err(internal_status("inspect pending Docker container", error)),
};
let labels = inspected
.config
.as_ref()
.and_then(|config| config.labels.as_ref());
let owned = labels.is_some_and(|labels| {
labels.get(LABEL_MANAGED_BY).map(String::as_str) == Some(LABEL_MANAGED_BY_VALUE)
&& labels.get(LABEL_SANDBOX_NAMESPACE).map(String::as_str) == Some(namespace)
&& labels.get(LABEL_SANDBOX_ID).map(String::as_str) == Some(sandbox_id)
&& labels.get(LABEL_ISOLATION_ROLE).map(String::as_str)
== Some(LABEL_ISOLATION_ROLE_SANDBOX)
&& labels.get(LABEL_ISOLATION_BACKEND).map(String::as_str)
== Some(LABEL_ISOLATION_BACKEND_OPEN_SHELL)
});
if !owned {
return Err(Status::failed_precondition(format!(
"Docker container '{name}' is not owned by this sandbox namespace; refusing to remove it"
)));
}
let container_id = inspected
.id
.as_deref()
.filter(|id| !id.is_empty())
.ok_or_else(|| Status::failed_precondition("pending Docker container has no ID"))?;
match docker
.remove_container(
container_id,
Some(RemoveContainerOptionsBuilder::default().force(true).build()),
)
.await
{
Ok(()) => Ok(true),
Err(error) if is_not_found_error(&error) => Ok(false),
Err(error) => Err(internal_status("delete pending Docker container", error)),
}
}

#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct DockerGuestTlsPaths {
pub(crate) ca: PathBuf,
}

#[derive(Debug, Clone, Copy)]
enum DockerContainerNaming {
Legacy,
ScopedByHostUid,
}

#[derive(Debug, Clone)]
struct DockerDriverRuntimeConfig {
allow_driver_config: bool,
resource_admission: openshell_core::resource_admission::ResourceAdmissionConfig,
default_image: String,
image_pull_policy: ImagePullPolicy,
sandbox_namespace: String,
container_naming: DockerContainerNaming,
stop_timeout_secs: u32,
log_level: String,
sandbox_binary: Arc<Vec<u8>>,
Expand Down Expand Up @@ -386,6 +486,17 @@ struct DockerDriverRuntimeConfig {
network_trust: Option<NetworkSupervisorTrustBundle>,
}

impl DockerDriverRuntimeConfig {
fn container_name(&self, sandbox: &DriverSandbox) -> String {
if matches!(self.container_naming, DockerContainerNaming::Legacy) {
return container_name_for_sandbox(sandbox);
}
let digest = format!("{:x}", Sha256::digest(self.sandbox_namespace.as_bytes()));
let prefix = format!("{CONTAINER_NAME_PREFIX}{}-", &digest[..24]);
container_name_for_sandbox_with_prefix(sandbox, &prefix)
}
}

#[derive(Debug, Clone, Copy)]
struct DockerGpuRuntimeCapabilities {
cdi_supported: bool,
Expand Down Expand Up @@ -914,6 +1025,8 @@ impl DockerComputeDriver {
network_trust: Option<NetworkSupervisorTrustBundle>,
) -> CoreResult<Self> {
docker_config.validate_configuration(gateway_bind_address)?;
let sandbox_namespace =
docker_sandbox_namespace(docker_config, rustix::process::geteuid().as_raw());
let socket_path = docker_config
.socket_path
.clone()
Expand All @@ -933,6 +1046,9 @@ impl DockerComputeDriver {
docker.version().await.map_err(|err| {
Error::execution(format!("failed to query Docker daemon version: {err}"))
})?;
if docker_config.scope_by_host_uid {
reject_unscoped_docker_resources(&docker, &docker_config.sandbox_label).await?;
}
Comment on lines +1049 to +1051

@chatgpt-codex-connector chatgpt-codex-connector Bot Nov 3, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Guard the reverse migration out of UID scope

When an operator turns scope_by_host_uid off after creating scoped sandboxes, this condition skips the only migration guard. The runtime namespace then reverts to the bare sandbox_label, while list, delete, and reconciliation paths filter that namespace exactly (for example in list_managed_container_summaries), so the existing uid-<uid>-... containers, volumes, and private state become invisible to the gateway and may remain running or orphaned. Reject startup while the current UID-scoped namespace is nonempty, or provide an explicit reverse migration.

Useful? React with 👍 / 👎.

let info = docker.info().await.map_err(|err| {
Error::execution(format!("failed to query Docker daemon info: {err}"))
})?;
Expand Down Expand Up @@ -986,7 +1102,7 @@ impl DockerComputeDriver {
matches!(endpoint.host(), Some(url::Host::Domain(host)) if host.eq_ignore_ascii_case(HOST_DOCKER_INTERNAL))
})
{
Some(resolve_docker_host_gateway(&docker, &supervisor_image_id, &docker_config.sandbox_label).await?)
Some(resolve_docker_host_gateway(&docker, &supervisor_image_id, &sandbox_namespace).await?)
} else {
docker_supervisor_host_address(&supervisor_grpc_endpoint)
};
Expand Down Expand Up @@ -1021,7 +1137,12 @@ impl DockerComputeDriver {
config: DockerDriverRuntimeConfig {
default_image: docker_config.default_image.clone(),
image_pull_policy: docker_config.image_pull_policy,
sandbox_namespace: docker_config.sandbox_label.clone(),
sandbox_namespace,
container_naming: if docker_config.scope_by_host_uid {
DockerContainerNaming::ScopedByHostUid
} else {
DockerContainerNaming::Legacy
},
stop_timeout_secs: DEFAULT_STOP_TIMEOUT_SECS,
log_level: gateway_log_level.to_string(),
sandbox_binary,
Expand Down Expand Up @@ -1719,7 +1840,7 @@ impl DockerComputeDriver {
));
}

let container_name = container_name_for_sandbox(sandbox);
let container_name = self.config.container_name(sandbox);
let gpu_devices = match self
.resolve_gpu_cdi_devices(
validated.gpu_requirements,
Expand Down Expand Up @@ -2299,41 +2420,32 @@ impl DockerComputeDriver {
.await?
else {
if let Some(record) = pending {
let container_name = container_name_for_sandbox(&record.sandbox);
match self
.docker
.remove_container(
&container_name,
Some(RemoveContainerOptionsBuilder::default().force(true).build()),
let container_name = self.config.container_name(&record.sandbox);
let removed = remove_owned_pending_container(
&self.docker,
&container_name,
&record.sandbox.id,
&self.config.sandbox_namespace,
)
.await?;
self.clear_runtime_failure(&record.sandbox.id).await;
if removed {
remove_docker_channel_volume_by_id(
&self.docker,
&record.sandbox.id,
&self.config,
)
.await
{
Ok(()) => {
self.clear_runtime_failure(&record.sandbox.id).await;
remove_docker_channel_volume_by_id(
&self.docker,
&record.sandbox.id,
&self.config,
)
.await?;
cleanup_docker_boundary_state(&record.sandbox, &self.config);
return Ok(true);
}
Err(err) if is_not_found_error(&err) => {
self.clear_runtime_failure(&record.sandbox.id).await;
let _ = remove_docker_channel_volume_by_id(
&self.docker,
&record.sandbox.id,
&self.config,
)
.await;
cleanup_docker_boundary_state(&record.sandbox, &self.config);
return Ok(true);
}
Err(err) => {
return Err(internal_status("delete docker sandbox container", err));
}
.await?;
} else {
let _ = remove_docker_channel_volume_by_id(
&self.docker,
&record.sandbox.id,
&self.config,
)
.await;
}
cleanup_docker_boundary_state(&record.sandbox, &self.config);
return Ok(true);
}
if !sandbox_id.is_empty() {
self.stop_control_process(sandbox_id).await;
Expand Down Expand Up @@ -5061,7 +5173,7 @@ async fn stage_docker_supervisor_bundle(
config: &DockerDriverRuntimeConfig,
archive: Vec<u8>,
) -> Result<(), Status> {
let stager_name = format!("{}-supervisor-stage", container_name_for_sandbox(sandbox));
let stager_name = format!("{}-supervisor-stage", config.container_name(sandbox));
let _ = docker
.remove_container(
&stager_name,
Expand Down Expand Up @@ -5371,7 +5483,7 @@ async fn spawn_docker_control_process(
let workspace_root = tokio::fs::read_to_string(directory.join(WORKSPACE_ROOT_FILE))
.await
.map_err(|error| Status::internal(format!("read Docker workspace root: {error}")))?;
let supervisor_name = format!("{}-supervisor", container_name_for_sandbox(sandbox));
let supervisor_name = format!("{}-supervisor", config.container_name(sandbox));
let _ = docker
.remove_container(
&supervisor_name,
Expand Down Expand Up @@ -6746,27 +6858,27 @@ const MAX_CONTAINER_NAME_LEN: usize = 200;
const CONTAINER_NAME_PREFIX: &str = "openshell-";

fn container_name_for_sandbox(sandbox: &DriverSandbox) -> String {
container_name_for_sandbox_with_prefix(sandbox, CONTAINER_NAME_PREFIX)
}

fn container_name_for_sandbox_with_prefix(sandbox: &DriverSandbox, prefix: &str) -> String {
let id_suffix = sanitize_docker_name(&sandbox.id);
let workspace = sanitize_docker_name(&sandbox.workspace);
let name = sanitize_docker_name(&sandbox.name);

// Format: openshell-{workspace}--{name}-{id}
// The workspace and id are never truncated — they ensure uniqueness.
// Only the sandbox name portion is truncated when the total exceeds
// MAX_CONTAINER_NAME_LEN.

// The sandbox name portion is truncated to fit Docker's name limit.
if name.is_empty() {
let mut base = format!("{CONTAINER_NAME_PREFIX}{workspace}---{id_suffix}");
let mut base = format!("{prefix}{workspace}---{id_suffix}");
if base.len() > MAX_CONTAINER_NAME_LEN {
base.truncate(MAX_CONTAINER_NAME_LEN);
}
return trim_container_name_tail(base);
}

// Reserve space for fixed parts: prefix + workspace + "--" + "-" + id
let reserved = CONTAINER_NAME_PREFIX.len() + workspace.len() + 2 + 1 + id_suffix.len();
// Reserve space for fixed parts: prefix + workspace + "--" + "-" + id.
let reserved = prefix.len() + workspace.len() + 2 + 1 + id_suffix.len();
if reserved >= MAX_CONTAINER_NAME_LEN {
let mut base = format!("{CONTAINER_NAME_PREFIX}{workspace}---{id_suffix}");
let mut base = format!("{prefix}{workspace}---{id_suffix}");
base.truncate(MAX_CONTAINER_NAME_LEN);
return trim_container_name_tail(base);
}
Expand All @@ -6777,7 +6889,7 @@ fn container_name_for_sandbox(sandbox: &DriverSandbox) -> String {
} else {
name
};
format!("{CONTAINER_NAME_PREFIX}{workspace}--{truncated_name}-{id_suffix}")
format!("{prefix}{workspace}--{truncated_name}-{id_suffix}")
}

/// Normalize an image reference for a pull request by appending `:latest`
Expand Down
Loading
Loading