Skip to content

Commit 03f7752

Browse files
committed
fix(execution): name the signal that killed a sandbox process in its error
1 parent d3f4739 commit 03f7752

4 files changed

Lines changed: 80 additions & 4 deletions

File tree

‎apps/sim/lib/execution/remote-sandbox/code-failure.test.ts‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
import { describe, expect, it } from 'vitest'
2-
import { processCodeFailure } from '@/lib/execution/remote-sandbox/code-failure'
2+
import {
3+
describeProcessExit,
4+
processCodeFailure,
5+
} from '@/lib/execution/remote-sandbox/code-failure'
36

47
/** Node 22's crash report when `fetch` loses its socket before any response bytes. */
58
const FETCH_SOCKET_CLOSED = `node:internal/deps/undici/undici:13510
@@ -196,6 +199,29 @@ describe('processCodeFailure', () => {
196199
expect(error).toEqual({ name, value, traceback: stderr })
197200
})
198201

202+
it('names the signal for a process killed after printing only a warning', () => {
203+
const warning = 'Warning: You are sending unauthenticated requests to the HF Hub.'
204+
const killed = processCodeFailure({ stdout: '', stderr: warning, exitCode: 137 })
205+
const exited = processCodeFailure({ stdout: '', stderr: warning, exitCode: 1 })
206+
207+
expect(killed.error).toEqual({
208+
name: 'Error',
209+
value: 'Process exited with code 137 (killed by SIGKILL, usually out of memory)',
210+
traceback: warning,
211+
})
212+
expect(exited.error?.name).toBe('Warning')
213+
})
214+
215+
it.each([
216+
[1, 'Process exited with code 1'],
217+
[128, 'Process exited with code 128'],
218+
[129, 'Process exited with code 129 (killed by signal 1)'],
219+
[143, 'Process exited with code 143 (killed by SIGTERM)'],
220+
[193, 'Process exited with code 193'],
221+
])('describes exit code %i', (exitCode, description) => {
222+
expect(describeProcessExit(exitCode)).toBe(description)
223+
})
224+
199225
it('reads the traceback from stdout when stderr is empty', () => {
200226
const { error } = processCodeFailure({ stdout: PYTHON_TRACEBACK, stderr: '', exitCode: 1 })
201227

‎apps/sim/lib/execution/remote-sandbox/code-failure.ts‎

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,47 @@ import type { SandboxCodeResult, SandboxCommandResult } from '@/lib/execution/re
1414
const ERROR_HEADLINE =
1515
/^(?:\[(?:[A-Za-z_$][\w.$]*)?(?:Error|Exception)|\[?[A-Za-z_$][\w.$]* \[[\w.$-]+\]|[A-Za-z_$][\w.$]*):(?:\s|$)/
1616

17+
const SIGNAL_NAMES: Readonly<Record<number, string>> = {
18+
6: 'SIGABRT',
19+
9: 'SIGKILL',
20+
11: 'SIGSEGV',
21+
15: 'SIGTERM',
22+
}
23+
24+
/**
25+
* `Process exited with code N`, naming the signal behind a code above 128. A killed process often
26+
* printed only a warning first, and that warning must not read as the reason it failed; SIGKILL
27+
* (137) is usually the out-of-memory killer.
28+
*/
29+
export function describeProcessExit(exitCode: number): string {
30+
const described = `Process exited with code ${exitCode}`
31+
const signal = killingSignal(exitCode)
32+
if (signal === undefined) return described
33+
const name = SIGNAL_NAMES[signal] ?? `signal ${signal}`
34+
return `${described} (killed by ${name}${signal === 9 ? ', usually out of memory' : ''})`
35+
}
36+
37+
/** The signal a shell's `128 + n` exit code reports, if the code is one. */
38+
function killingSignal(exitCode: number): number | undefined {
39+
return exitCode > 128 && exitCode <= 128 + 64 ? exitCode - 128 : undefined
40+
}
41+
1742
/**
1843
* Converts a failed code command into a code result, naming the error from the
19-
* last error headline of the traceback (stderr, else stdout).
44+
* last error headline of the traceback (stderr, else stdout). A process a signal
45+
* killed raised nothing: anything it printed (a warning) is not the cause, so the
46+
* signal names the error and the output stays as the traceback.
2047
*/
2148
export function processCodeFailure(result: SandboxCommandResult): SandboxCodeResult {
2249
const traceback = result.stderr || result.stdout
50+
if (killingSignal(result.exitCode) !== undefined) {
51+
return {
52+
text: '',
53+
stdout: result.stdout,
54+
stderr: result.stderr,
55+
error: { name: 'Error', value: describeProcessExit(result.exitCode), traceback },
56+
}
57+
}
2358
const headline = traceback
2459
.split('\n')
2560
.reverse()

‎apps/sim/lib/execution/remote-sandbox/conformance.test.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1261,6 +1261,18 @@ describe.each(PROVIDERS)('sandbox conformance [%s]', (provider) => {
12611261
})
12621262
})
12631263

1264+
it("leads a killed command's error with its signal, not the warning it printed first", async () => {
1265+
const warning = 'Warning: You are sending unauthenticated requests to the HF Hub.'
1266+
stubShellCommand(provider, provider === 'daytona' ? warning : '', warning, 137)
1267+
1268+
const res = await executeShellInSandbox({ code: 'python train.py', envs: {}, timeoutMs: 1000 })
1269+
1270+
expect(res.error).toBe(
1271+
`Process exited with code 137 (killed by SIGKILL, usually out of memory): ${warning}`
1272+
)
1273+
expect(res.stdout).toContain(warning)
1274+
})
1275+
12641276
it('terminates shell execution when streamed process output exceeds the byte budget', async () => {
12651277
const oversized = 'x'.repeat(MAX_SANDBOX_PROCESS_OUTPUT_BYTES + 1)
12661278
if (provider === 'e2b') {

‎apps/sim/lib/execution/remote-sandbox/index.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import {
1111
import { recordSandboxTeardownFailure } from '@/lib/core/execution-limits/metrics'
1212
import { buildJavaScriptRuntimeBindingsSource } from '@/lib/execution/code-placeholders/javascript-runtime'
1313
import { SANDBOX_SYSTEM_PATH } from '@/lib/execution/remote-sandbox/cli-tools.server'
14+
import { describeProcessExit } from '@/lib/execution/remote-sandbox/code-failure'
1415
import {
1516
type CreatedSandbox,
1617
createSandbox,
@@ -1152,8 +1153,10 @@ async function executeShellInSandboxWithinBudget(
11521153
if (result.exitCode !== 0) {
11531154
// Daytona merges both streams into stdout (stderr is always empty), so fall
11541155
// back to stdout for the real command output before the generic message.
1155-
const errorMessage =
1156-
result.stderr || result.stdout || `Process exited with code ${result.exitCode}`
1156+
const output = result.stderr || result.stdout
1157+
const errorMessage = output
1158+
? `${describeProcessExit(result.exitCode)}: ${output}`
1159+
: describeProcessExit(result.exitCode)
11571160
logger.error('Sandbox shell execution error', {
11581161
sandboxId,
11591162
exitCode: result.exitCode,

0 commit comments

Comments
 (0)