Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
105 changes: 105 additions & 0 deletions crates/openshell-driver-kubernetes/src/driver.rs
Original file line number Diff line number Diff line change
Expand Up @@ -485,6 +485,7 @@ const SANDBOX_VERSIONS: &[&str] = &[SANDBOX_VERSION_V1BETA1, SANDBOX_VERSION_V1A
pub const SANDBOX_KIND: &str = "Sandbox";
const SANDBOX_POD_NAME_ANNOTATION: &str = "agents.x-k8s.io/pod-name";
const SANDBOX_SUSPENDED_CONDITION: &str = "Suspended";
const POD_SCHEDULED_CONDITION: &str = "PodScheduled";
const SANDBOX_SUSPENDED_POD_NOT_OWNED_REASON: &str = "PodNotOwned";
const SANDBOX_TOKEN_AUDIENCE: &str = "openshell-gateway";
const POD_NAME_EXTRA: &str = "authentication.kubernetes.io/pod-name";
Expand Down Expand Up @@ -6625,6 +6626,43 @@ async fn sandbox_runtime_workload_generation_matches(
}
}

/// The scheduler's verdict for a pod it cannot place, as a sandbox condition.
fn pod_unschedulable_condition(pod: &Pod) -> Option<SandboxCondition> {
let condition = pod
.status
.as_ref()?
.conditions
.as_ref()?
.iter()
.find(|condition| {
condition.type_ == POD_SCHEDULED_CONDITION
&& condition.status == "False"
&& condition.reason.as_deref() == Some("Unschedulable")
})?;
Some(SandboxCondition {
r#type: POD_SCHEDULED_CONDITION.to_string(),
status: "False".to_string(),
reason: "Unschedulable".to_string(),
message: condition.message.clone().unwrap_or_default(),
transition_time: condition.last_transition_time.as_ref().and_then(|time| {
openshell_core::time::timestamp_from_millis(time.0.timestamp_millis()).ok()
}),
})
}

/// Best-effort: a failed read only omits the scheduling hint.
async fn workload_unschedulable_condition(
client: &Client,
namespace: &str,
pod_name: &str,
) -> Option<SandboxCondition> {
let pods = Api::<Pod>::namespaced(client.clone(), namespace);
match tokio::time::timeout(KUBE_API_TIMEOUT, pods.get_opt(pod_name)).await {
Ok(Ok(Some(pod))) => pod_unschedulable_condition(&pod),
_ => None,
}
}

async fn sandbox_runtime_supervisor_generation_matches(
client: &Client,
namespace: &str,
Expand Down Expand Up @@ -6720,6 +6758,18 @@ async fn sandbox_from_object_with_sandbox_runtime_readiness(
mark_sandbox_runtime_control_unavailable(&mut sandbox);
}
}
if let Some(status) = sandbox.status.as_mut()
&& !status.conditions.iter().any(|condition| {
condition.r#type == "Ready" && condition.status.eq_ignore_ascii_case("true")
})
&& let Some(condition) =
workload_unschedulable_condition(client, &object_namespace, &name).await
{
status
.conditions
.retain(|condition| condition.r#type != POD_SCHEDULED_CONDITION);
status.conditions.push(condition);
}
Ok((name, sandbox))
}

Expand Down Expand Up @@ -6882,6 +6932,9 @@ fn attach_kube_progress_metadata(
mark_progress_complete(metadata, PROGRESS_STEP_PULLING_IMAGE, label);
mark_progress_active(metadata, PROGRESS_STEP_STARTING_SANDBOX);
}
"FailedScheduling" => {
mark_progress_detail(metadata, format!("Waiting to be scheduled: {message}"));
}
_ => {}
}
}
Expand Down Expand Up @@ -11295,6 +11348,58 @@ mod tests {
);
}

#[test]
fn kube_failed_scheduling_event_adds_waiting_detail() {
let mut metadata = std::collections::HashMap::new();

attach_kube_progress_metadata(
&mut metadata,
"FailedScheduling",
"0/3 nodes are available: 3 Insufficient memory.",
);

assert_eq!(
metadata.get(PROGRESS_ACTIVE_DETAIL_KEY).map(String::as_str),
Some("Waiting to be scheduled: 0/3 nodes are available: 3 Insufficient memory.")
);
assert!(!metadata.contains_key(PROGRESS_COMPLETE_STEP_KEY));
assert!(!metadata.contains_key(PROGRESS_ACTIVE_STEP_KEY));
}

#[test]
fn unschedulable_pod_reports_scheduler_message() {
let pod_with = |reason: &str, status: &str| Pod {
status: Some(
serde_json::from_value(serde_json::json!({
"conditions": [{
"type": "PodScheduled",
"status": status,
"reason": reason,
"message": "0/3 nodes are available: 3 Insufficient memory.",
"lastTransitionTime": "2026-10-24T08:00:00Z"
}]
}))
.expect("valid Pod status"),
),
..Pod::default()
};

let condition = pod_unschedulable_condition(&pod_with("Unschedulable", "False"))
.expect("unschedulable pod");
assert_eq!(condition.r#type, "PodScheduled");
assert_eq!(condition.status, "False");
assert_eq!(condition.reason, "Unschedulable");
assert_eq!(
condition.message,
"0/3 nodes are available: 3 Insufficient memory."
);
assert!(condition.transition_time.is_some());

assert!(pod_unschedulable_condition(&pod_with("SchedulingGated", "False")).is_none());
assert!(pod_unschedulable_condition(&pod_with("", "True")).is_none());
assert!(pod_unschedulable_condition(&Pod::default()).is_none());
}

#[test]
fn sandbox_runtime_renders_credential_free_boundary_workload() {
let params = SandboxPodParams {
Expand Down
7 changes: 7 additions & 0 deletions docs/how-it-works/sandboxes/overview.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -1077,6 +1077,13 @@ After a gateway restart, an existing sandbox can return to `Provisioning`
temporarily while its supervisor reconnects. Wait for the phase to return to
`Ready` before you connect to the sandbox or execute commands.

On the Kubernetes driver, a sandbox pod that the scheduler cannot place keeps
the sandbox in `Provisioning` with a `PodScheduled=False` condition. Its reason
is `Unschedulable` and its message is the scheduler's explanation, such as
`0/3 nodes are available: 3 Insufficient memory`. `openshell sandbox create`
shows the same message while it waits, and `openshell sandbox get` reports the
condition. The condition clears once the pod is scheduled.

The gateway records a successful canonical main-process exit as `Ready=False`
with reason `MainProcessCompleted`. Nonzero and signal-normalized results use
`MainProcessFailed` and the `Error` phase. It also sets `status.exit_code`;
Expand Down
Loading