fix(execution): name the signal exit code of a failed sandbox process (#9523)

* fix(execution): name the signal that killed a sandbox process in its error

* fix(execution): describe a signal's exit code without claiming a kill

Author: waleedlatif1

apps/sim/lib/execution/remote-sandbox/code-failure.test.ts

@@ -1,5 +1,8 @@
 import { describe, expect, it } from 'vitest'
-import { processCodeFailure } from '@/lib/execution/remote-sandbox/code-failure'
+import {
+  describeProcessExit,
+  processCodeFailure,
+} from '@/lib/execution/remote-sandbox/code-failure'
 
 /** Node 22's crash report when `fetch` loses its socket before any response bytes. */
 const FETCH_SOCKET_CLOSED = `node:internal/deps/undici/undici:13510
@@ -196,6 +199,34 @@ describe('processCodeFailure', () => {
     expect(error).toEqual({ name, value, traceback: stderr })
   })
 
+  it('keeps the printed headline and adds the signal exit code beside it', () => {
+    const warning = 'Warning: You are sending unauthenticated requests to the HF Hub.'
+    const killed = processCodeFailure({ stdout: '', stderr: warning, exitCode: 137 })
+    const exited = processCodeFailure({ stdout: '', stderr: warning, exitCode: 1 })
+
+    expect(killed.error).toEqual({
+      name: 'Warning',
+      value:
+        'You are sending unauthenticated requests to the HF Hub. [Process exited with code 137 (the exit code of SIGKILL, usually out of memory)]',
+      traceback: warning,
+    })
+    expect(exited.error).toEqual({
+      name: 'Warning',
+      value: 'You are sending unauthenticated requests to the HF Hub.',
+      traceback: warning,
+    })
+  })
+
+  it.each([
+    [1, 'Process exited with code 1'],
+    [128, 'Process exited with code 128'],
+    [129, 'Process exited with code 129 (the exit code of signal 1)'],
+    [143, 'Process exited with code 143 (the exit code of SIGTERM)'],
+    [193, 'Process exited with code 193'],
+  ])('describes exit code %i', (exitCode, description) => {
+    expect(describeProcessExit(exitCode)).toBe(description)
+  })
+
   it('reads the traceback from stdout when stderr is empty', () => {
     const { error } = processCodeFailure({ stdout: PYTHON_TRACEBACK, stderr: '', exitCode: 1 })
 

apps/sim/lib/execution/remote-sandbox/code-failure.ts

@@ -14,9 +14,37 @@ import type { SandboxCodeResult, SandboxCommandResult } from '@/lib/execution/re
 const ERROR_HEADLINE =
   /^(?:\[(?:[A-Za-z_$][\w.$]*)?(?:Error|Exception)|\[?[A-Za-z_$][\w.$]* \[[\w.$-]+\]|[A-Za-z_$][\w.$]*):(?:\s|$)/
 
+const SIGNAL_NAMES: Readonly<Record<number, string>> = {
+  6: 'SIGABRT',
+  9: 'SIGKILL',
+  11: 'SIGSEGV',
+  15: 'SIGTERM',
+}
+
+/**
+ * `Process exited with code N`, naming the signal a code above 128 stands for. A process a signal
+ * killed often printed only a warning first, so the code must be visible beside whatever it
+ * printed. The code alone cannot prove a kill (a program may exit 137 itself), so it is described
+ * as the signal's code; SIGKILL's (137) usually means the out-of-memory killer.
+ */
+export function describeProcessExit(exitCode: number): string {
+  const note = signalCodeNote(exitCode)
+  return `Process exited with code ${exitCode}${note ? ` (${note})` : ''}`
+}
+
+/** What a shell's `128 + n` exit code stands for, if the code is one. */
+function signalCodeNote(exitCode: number): string | undefined {
+  if (exitCode <= 128 || exitCode > 128 + 64) return undefined
+  const signal = exitCode - 128
+  const name = SIGNAL_NAMES[signal] ?? `signal ${signal}`
+  return `the exit code of ${name}${signal === 9 ? ', usually out of memory' : ''}`
+}
+
 /**
  * Converts a failed code command into a code result, naming the error from the
- * last error headline of the traceback (stderr, else stdout).
+ * last error headline of the traceback (stderr, else stdout). A signal's exit code
+ * is appended to the value, since a killed process's last printed line (often a
+ * warning) is not necessarily why it stopped.
  */
 export function processCodeFailure(result: SandboxCommandResult): SandboxCodeResult {
   const traceback = result.stderr || result.stdout
@@ -31,10 +59,13 @@ export function processCodeFailure(result: SandboxCommandResult): SandboxCodeRes
   const separator = errorLine?.indexOf(':') ?? -1
   const parsedErrorLine = errorLine ?? ''
   const name = separator > 0 ? parsedErrorLine.slice(0, separator) : 'Error'
-  const value =
+  const headlineValue =
     separator > 0
       ? parsedErrorLine.slice(separator + 1).trim()
       : parsedErrorLine || 'Execution failed'
+  const value = signalCodeNote(result.exitCode)
+    ? `${headlineValue} [${describeProcessExit(result.exitCode)}]`
+    : headlineValue
   return {
     text: '',
     stdout: result.stdout,

apps/sim/lib/execution/remote-sandbox/conformance.test.ts

@@ -1261,6 +1261,18 @@ describe.each(PROVIDERS)('sandbox conformance [%s]', (provider) => {
     })
   })
 
+  it("leads a failed command's error with its exit code and the signal it stands for", async () => {
+    const warning = 'Warning: You are sending unauthenticated requests to the HF Hub.'
+    stubShellCommand(provider, provider === 'daytona' ? warning : '', warning, 137)
+
+    const res = await executeShellInSandbox({ code: 'python train.py', envs: {}, timeoutMs: 1000 })
+
+    expect(res.error).toBe(
+      `Process exited with code 137 (the exit code of SIGKILL, usually out of memory): ${warning}`
+    )
+    expect(res.stdout).toContain(warning)
+  })
+
   it('terminates shell execution when streamed process output exceeds the byte budget', async () => {
     const oversized = 'x'.repeat(MAX_SANDBOX_PROCESS_OUTPUT_BYTES + 1)
     if (provider === 'e2b') {

apps/sim/lib/execution/remote-sandbox/index.ts

@@ -11,6 +11,7 @@ import {
 import { recordSandboxTeardownFailure } from '@/lib/core/execution-limits/metrics'
 import { buildJavaScriptRuntimeBindingsSource } from '@/lib/execution/code-placeholders/javascript-runtime'
 import { SANDBOX_SYSTEM_PATH } from '@/lib/execution/remote-sandbox/cli-tools.server'
+import { describeProcessExit } from '@/lib/execution/remote-sandbox/code-failure'
 import {
   type CreatedSandbox,
   createSandbox,
@@ -1152,8 +1153,10 @@ async function executeShellInSandboxWithinBudget(
     if (result.exitCode !== 0) {
       // Daytona merges both streams into stdout (stderr is always empty), so fall
       // back to stdout for the real command output before the generic message.
-      const errorMessage =
-        result.stderr || result.stdout || `Process exited with code ${result.exitCode}`
+      const output = result.stderr || result.stdout
+      const errorMessage = output
+        ? `${describeProcessExit(result.exitCode)}: ${output}`
+        : describeProcessExit(result.exitCode)
       logger.error('Sandbox shell execution error', {
         sandboxId,
         exitCode: result.exitCode,