Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions crates/openshell-gateway/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ tracing = { workspace = true, optional = true }

[target.'cfg(target_os = "windows")'.dependencies]
openshell-driver-mxc = { path = "../openshell-driver-mxc", optional = true }
windows = { workspace = true, features = ["Win32_System_JobObjects"] }

[features]
default = ["telemetry", "in-tree-compute-drivers"]
Expand Down
2 changes: 2 additions & 0 deletions crates/openshell-gateway/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ compile_error!(
build a telemetry-free gateway with `--no-default-features --features defaults-without-telemetry`"
);

pub mod process_lifetime;

#[cfg(all(not(target_os = "windows"), feature = "compute-driver-vm"))]
mod vm;

Expand Down
6 changes: 6 additions & 0 deletions crates/openshell-gateway/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,12 @@

#[tokio::main]
async fn main() -> miette::Result<()> {
// Must run before anything is spawned so every sandbox process joins the job.
if let Err(error) = openshell_gateway::process_lifetime::kill_children_on_exit() {
eprintln!(
"warning: could not tie sandbox processes to the gateway lifetime; they will outlive it: {error}"
);
}
openshell_server::cli::run_cli_with_compute_drivers(
openshell_gateway::install_default_compute_drivers(),
)
Expand Down
114 changes: 114 additions & 0 deletions crates/openshell-gateway/src/process_lifetime.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

//! Tie the lifetime of every sandbox process to the gateway process.
//!
//! With the MXC driver built into the gateway, the gateway starts `wxc-exec`,
//! `openshell-supervisor` and `openshell-windows-sandbox` for each sandbox. If
//! the gateway is killed (crash, `taskkill`, logoff), those processes used to
//! survive with no owner and, because they inherited the gateway's listening
//! socket, kept its port bound so a new gateway could not start.
//!
//! A Windows *job object* with `JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE` fixes the
//! first half: the gateway puts itself in the job, every child it spawns
//! inherits membership, and Windows ends all members when the last handle to
//! the job closes, which happens when the gateway process ends for any reason.
//! (The listening sockets are also made non-inheritable; see
//! `openshell_server::gateway_listener`.)

/// Put the current process into a new kill-on-close job object.
///
/// Call this first thing in `main`, before anything is spawned. The job handle
/// is deliberately never closed: it must stay open for as long as the process
/// lives, and closing it (which only happens when the process exits) is what
/// triggers the clean-up. On other platforms this does nothing.
///
/// # Errors
///
/// Returns a description of the failing Win32 call. The caller decides whether
/// that is fatal; nested jobs are supported on Windows 8 and later, so this
/// normally only fails on unusual hosts.
pub fn kill_children_on_exit() -> Result<(), String> {
#[cfg(windows)]
{
windows_impl::kill_children_on_exit()
}
#[cfg(not(windows))]
{
Ok(())
}
}

#[cfg(windows)]
#[allow(unsafe_code)] // Win32 job object calls.
mod windows_impl {
use std::mem::size_of;

use windows::Win32::System::JobObjects::{
AssignProcessToJobObject, CreateJobObjectW, JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE,
JOBOBJECT_EXTENDED_LIMIT_INFORMATION, JobObjectExtendedLimitInformation,
SetInformationJobObject,
};
use windows::Win32::System::Threading::GetCurrentProcess;
use windows::core::PCWSTR;

pub(super) fn kill_children_on_exit() -> Result<(), String> {
// SAFETY: plain Win32 calls with valid arguments; the info struct
// outlives the call and `size_of` matches its type.
unsafe {
let job = CreateJobObjectW(None, PCWSTR::null())
.map_err(|e| format!("CreateJobObjectW failed: {e}"))?;
let mut info = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default();
info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE;
SetInformationJobObject(
job,
JobObjectExtendedLimitInformation,
(&raw const info).cast(),
u32::try_from(size_of::<JOBOBJECT_EXTENDED_LIMIT_INFORMATION>())
.map_err(|e| format!("job info size: {e}"))?,
)
.map_err(|e| format!("SetInformationJobObject failed: {e}"))?;
AssignProcessToJobObject(job, GetCurrentProcess())
.map_err(|e| format!("AssignProcessToJobObject failed: {e}"))?;
// `job` is intentionally leaked (see the module docs).
let _ = job;
}
Ok(())
}
}

#[cfg(all(test, windows))]
#[allow(unsafe_code)] // Win32 job object query.
mod tests {
use std::mem::size_of;

use windows::Win32::System::JobObjects::{
JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, JOBOBJECT_EXTENDED_LIMIT_INFORMATION,
JobObjectExtendedLimitInformation, QueryInformationJobObject,
};

#[test]
fn puts_the_process_in_a_kill_on_close_job() {
super::kill_children_on_exit().expect("job object");

// A null job handle asks about the job the calling process is in; with
// nested jobs that is the innermost one, i.e. the one just created.
let mut info = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default();
// SAFETY: `info` outlives the call and the size matches its type.
unsafe {
QueryInformationJobObject(
None,
JobObjectExtendedLimitInformation,
(&raw mut info).cast(),
u32::try_from(size_of::<JOBOBJECT_EXTENDED_LIMIT_INFORMATION>()).unwrap(),
None,
)
}
.expect("QueryInformationJobObject");
assert_ne!(
info.BasicLimitInformation.LimitFlags.0 & JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE.0,
0,
"the process job must kill its members when closed"
);
}
}
4 changes: 4 additions & 0 deletions crates/openshell-server/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,10 @@ x509-parser = "0.16"
arc-swap = "1"
notify = "8"

[target.'cfg(target_os = "windows")'.dependencies]
# Clears the handle-inherit flag on the gateway's listening sockets (see gateway_listener.rs).
windows = { workspace = true }

[features]
default = ["telemetry"]
## Compile in anonymous telemetry emission (forwards to openshell-core/telemetry).
Expand Down
88 changes: 88 additions & 0 deletions crates/openshell-server/src/gateway_listener.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ use openshell_core::{Error, Result};
use std::net::SocketAddr;
use tokio::net::TcpListener;
use tracing::info;
#[cfg(windows)]
use tracing::warn;

pub struct BoundGatewayListener {
pub listener: TcpListener,
Expand All @@ -16,10 +18,96 @@ pub async fn bind_gateway_listener(address: SocketAddr) -> Result<BoundGatewayLi
let listener = TcpListener::bind(address)
.await
.map_err(|error| Error::transport(format!("failed to bind to {address}: {error}")))?;
disable_handle_inheritance(&listener, "gateway");
let local_addr = listener.local_addr().unwrap_or(address);
info!(address = %local_addr, "Gateway listener bound");
Ok(BoundGatewayListener {
listener,
address: local_addr,
})
}

/// Keep a listening socket private to the gateway process.
///
/// On Windows, `mio` (underneath tokio) creates sockets with plain `socket()`,
/// which leaves the handle inheritable, and `std::process::Command` starts
/// children with handle inheritance enabled. Every sandbox process the gateway
/// spawns therefore keeps its own copy of the listening socket. If the gateway
/// is killed while a sandbox is still running, that copy keeps the port bound
/// ("Only one usage of each socket address") and a new gateway cannot start.
/// Clearing the inherit flag right after `bind` prevents the leak.
///
/// A failure is logged and otherwise ignored: the listener still works, it just
/// keeps the old behaviour. This is a no-op on other platforms.
pub fn disable_handle_inheritance(listener: &TcpListener, name: &str) {
#[cfg(windows)]
if let Err(error) = windows_impl::disable_handle_inheritance(listener) {
warn!(
listener = name,
%error,
"could not make the listening socket non-inheritable; child processes may keep the port bound after a gateway crash"
);
}
#[cfg(not(windows))]
let _ = (listener, name);
}

#[cfg(windows)]
#[allow(unsafe_code)] // Win32 handle flags.
mod windows_impl {
use std::ffi::c_void;
use std::os::windows::io::AsRawSocket;

use tokio::net::TcpListener;
use windows::Win32::Foundation::{
HANDLE, HANDLE_FLAG_INHERIT, HANDLE_FLAGS, SetHandleInformation,
};

fn handle_of(listener: &TcpListener) -> HANDLE {
// A SOCKET is a pointer-sized handle value; this is an integer-to-pointer cast.
HANDLE(listener.as_raw_socket() as *mut c_void)
}

pub(super) fn disable_handle_inheritance(listener: &TcpListener) -> windows::core::Result<()> {
// SAFETY: `listener` owns a live socket handle for the whole call.
unsafe { SetHandleInformation(handle_of(listener), HANDLE_FLAG_INHERIT.0, HANDLE_FLAGS(0)) }
}

#[cfg(test)]
pub(super) fn is_inheritable(listener: &TcpListener) -> bool {
use windows::Win32::Foundation::GetHandleInformation;

let mut flags = 0u32;
// SAFETY: `listener` owns a live socket handle and `flags` outlives the call.
unsafe { GetHandleInformation(handle_of(listener), &raw mut flags) }
.expect("GetHandleInformation");
flags & HANDLE_FLAG_INHERIT.0 != 0
}
}

#[cfg(all(test, windows))]
mod tests {
use super::*;

#[tokio::test]
async fn bound_gateway_listener_is_not_inheritable() {
let bound = bind_gateway_listener("127.0.0.1:0".parse().unwrap())
.await
.unwrap();
assert!(
!windows_impl::is_inheritable(&bound.listener),
"child processes would inherit the gateway's listening socket"
);
}

#[tokio::test]
async fn listener_still_accepts_connections_after_disabling_inheritance() {
let bound = bind_gateway_listener("127.0.0.1:0".parse().unwrap())
.await
.unwrap();
let client = tokio::net::TcpStream::connect(bound.address);
let (accepted, connected) = tokio::join!(bound.listener.accept(), client);
accepted.expect("accept");
connected.expect("connect");
}
}
4 changes: 3 additions & 1 deletion crates/openshell-server/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ pub(crate) fn install_jsonwebtoken_crypto_provider() {
}

use compute::ComputeRuntime;
use gateway_listener::{BoundGatewayListener, bind_gateway_listener};
use gateway_listener::{BoundGatewayListener, bind_gateway_listener, disable_handle_inheritance};
pub use grpc::OpenShellService;
pub use http::{health_router, http_router, metrics_router, service_http_router};

Expand Down Expand Up @@ -854,6 +854,7 @@ pub(crate) async fn run_server(
"failed to bind health port {health_bind_address}: {e}"
))
})?;
disable_handle_inheritance(&health_listener, "health");
info!(address = %health_bind_address, "Health server listening");
// `health_router` returns immediately; the listener serves
// `Initializing → 503` until the background monitor publishes the
Expand All @@ -880,6 +881,7 @@ pub(crate) async fn run_server(
"failed to bind metrics port {metrics_bind_address}: {e}",
))
})?;
disable_handle_inheritance(&metrics_listener, "metrics");
info!(address = %metrics_bind_address, "Metrics server listening");
tokio::spawn(async move {
if let Err(e) = axum::serve(
Expand Down
Loading