diff --git a/.ci/readability-baseline.env b/.ci/readability-baseline.env index 9de17adc4..6832b1d7e 100644 --- a/.ci/readability-baseline.env +++ b/.ci/readability-baseline.env @@ -6,6 +6,6 @@ PROD_FILES_GT1000=22 PROD_MAX_FILE_LINES=2829 PROD_MAX_FILE_PATH=crates/temper-server/src/storage/mod.rs ALLOW_CLIPPY_COUNT=35 -ALLOW_DEAD_CODE_COUNT=14 +ALLOW_DEAD_CODE_COUNT=16 PROD_PRINTLN_COUNT=247 PROD_UNWRAP_CI_OK_COUNT=132 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2c29f86dc..75eb855ce 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -265,6 +265,20 @@ jobs: - name: cargo test --workspace -- --skip dst_ run: cargo test --workspace -- --skip dst_ + # spec_validate_endpoint declares required-features = ["observe"]. The + # workspace run above usually still covers it via feature unification + # (temper-cli/temper-mcp enable temper-server's observe feature), but + # that coverage is incidental — it evaporates if those edges change. + # Run the observe-gated tests explicitly so coverage is guaranteed. + - name: cargo test observe-gated tests + run: | + cargo test -p temper-server --features observe --test spec_validate_endpoint + cargo test -p temper-server --features observe --lib observe:: + # api::repl tests (incl. server_repl_config_never_allows_host_ops, + # the ARN-166 host-op isolation guard) compile only under `observe`; + # run them explicitly so coverage never depends on feature unification. + cargo test -p temper-server --features observe --lib api::repl:: + # ───────────────────────────────────────────────────── # Gate 3b: DST/platform coverage (matrixed) # ───────────────────────────────────────────────────── diff --git a/Cargo.lock b/Cargo.lock index 5b9d7ed73..76fae172b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,22 +2,13 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "addr2line" -version = "0.24.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" -dependencies = [ - "gimli 0.31.1", -] - [[package]] name = "addr2line" version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" dependencies = [ - "gimli 0.32.3", + "gimli", ] [[package]] @@ -182,7 +173,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7eb93bbb63b9c227414f6eb3a0adfddca591a8ce1e9b60661bb08969b87e340b" dependencies = [ - "object 0.37.3", + "object", ] [[package]] @@ -231,6 +222,150 @@ dependencies = [ "serde_json", ] +[[package]] +name = "async-channel" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" +dependencies = [ + "concurrent-queue", + "event-listener 2.5.3", + "futures-core", +] + +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-executor" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96bf972d85afc50bf5ab8fe2d54d1586b4e0b46c97c50a0c9e71e2f7bcd812a" +dependencies = [ + "async-task", + "concurrent-queue", + "fastrand", + "futures-lite", + "pin-project-lite", + "slab", +] + +[[package]] +name = "async-global-executor" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b1b633a2115cd122d73b955eadd9916c18c8f510ec9cd1686404c60ad1c29c" +dependencies = [ + "async-channel 2.5.0", + "async-executor", + "async-io", + "async-lock", + "blocking", + "futures-lite", + "once_cell", +] + +[[package]] +name = "async-io" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "456b8a8feb6f42d237746d4b3e9a178494627745c3c56c6ea55d92ba50d026fc" +dependencies = [ + "autocfg", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite", + "parking", + "polling", + "rustix 1.1.4", + "slab", + "windows-sys 0.61.2", +] + +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener 5.4.1", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "async-process" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc50921ec0055cdd8a16de48773bfeec5c972598674347252c0399676be7da75" +dependencies = [ + "async-channel 2.5.0", + "async-io", + "async-lock", + "async-signal", + "async-task", + "blocking", + "cfg-if", + "event-listener 5.4.1", + "futures-lite", + "rustix 1.1.4", +] + +[[package]] +name = "async-signal" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52b5aaafa020cf5053a01f2a60e8ff5dccf550f0f77ec54a4e47285ac2bab485" +dependencies = [ + "async-io", + "async-lock", + "atomic-waker", + "cfg-if", + "futures-core", + "futures-io", + "rustix 1.1.4", + "signal-hook-registry", + "slab", + "windows-sys 0.61.2", +] + +[[package]] +name = "async-std" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c8e079a4ab67ae52b7403632e4618815d6db36d2a010cfe41b02c1b1578f93b" +dependencies = [ + "async-channel 1.9.0", + "async-global-executor", + "async-io", + "async-lock", + "async-process", + "crossbeam-utils", + "futures-channel", + "futures-core", + "futures-io", + "futures-lite", + "gloo-timers", + "kv-log-macro", + "log", + "memchr", + "once_cell", + "pin-project-lite", + "pin-utils", + "slab", + "wasm-bindgen-futures", +] + [[package]] name = "async-stream" version = "0.3.6" @@ -253,6 +388,12 @@ dependencies = [ "syn", ] +[[package]] +name = "async-task" +version = "4.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" + [[package]] name = "async-trait" version = "0.1.89" @@ -427,11 +568,11 @@ version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" dependencies = [ - "addr2line 0.25.1", + "addr2line", "cfg-if", "libc", "miniz_oxide", - "object 0.37.3", + "object", "rustc-demangle", "windows-link", ] @@ -561,6 +702,19 @@ dependencies = [ "generic-array", ] +[[package]] +name = "blocking" +version = "1.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e83f8d02be6967315521be875afa792a316e28d57b5a2d401897e2a7921b7f21" +dependencies = [ + "async-channel 2.5.0", + "async-task", + "futures-io", + "futures-lite", + "piper", +] + [[package]] name = "bollard" version = "0.18.1" @@ -667,9 +821,9 @@ dependencies = [ [[package]] name = "cap-fs-ext" -version = "3.4.5" +version = "3.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5528f85b1e134ae811704e41ef80930f56e795923f866813255bc342cc20654" +checksum = "476f0d0003a760918ed4b1e039a59e11769030416f79c8222551d22785f7f70d" dependencies = [ "cap-primitives", "cap-std", @@ -679,9 +833,9 @@ dependencies = [ [[package]] name = "cap-net-ext" -version = "3.4.5" +version = "3.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20a158160765c6a7d0d8c072a53d772e4cb243f38b04bfcf6b4939cfbe7482e7" +checksum = "150941cefd3df4de2fea24604ba4949371576f62e527410298333f7d431a1bc6" dependencies = [ "cap-primitives", "cap-std", @@ -691,9 +845,9 @@ dependencies = [ [[package]] name = "cap-primitives" -version = "3.4.5" +version = "3.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6cf3aea8a5081171859ef57bc1606b1df6999df4f1110f8eef68b30098d1d3a" +checksum = "8e0bf07d379916947be6c4a07f43684153d710a2896c31f9e97781362895596c" dependencies = [ "ambient-authority", "fs-set-times", @@ -709,9 +863,9 @@ dependencies = [ [[package]] name = "cap-rand" -version = "3.4.5" +version = "3.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8144c22e24bbcf26ade86cb6501a0916c46b7e4787abdb0045a467eb1645a1d" +checksum = "6ec6a5b75f54547c579a6b117c6fdd5f04f4ab7598de747b9f440a53592b3a4a" dependencies = [ "ambient-authority", "rand 0.8.5", @@ -719,9 +873,9 @@ dependencies = [ [[package]] name = "cap-std" -version = "3.4.5" +version = "3.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6dc3090992a735d23219de5c204927163d922f42f575a0189b005c62d37549a" +checksum = "a59e59fa26472d29680ece6a9f8ee8b0551a719a33df2f5240bde065ecbddfd7" dependencies = [ "cap-primitives", "io-extras", @@ -731,9 +885,9 @@ dependencies = [ [[package]] name = "cap-time-ext" -version = "3.4.5" +version = "3.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "def102506ce40c11710a9b16e614af0cde8e76ae51b1f48c04b8d79f4b671a80" +checksum = "b54c289326c70f1c697ebf0a31842a480932e5942b5fac92fcc46e87286b48e2" dependencies = [ "ambient-authority", "cap-primitives", @@ -1148,20 +1302,38 @@ dependencies = [ "libc", ] +[[package]] +name = "cranelift-assembler-x64" +version = "0.123.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6835dba958b2ab7ab523e7e99296e0524317f60430a00cf5850562ef78ea7001" +dependencies = [ + "cranelift-assembler-x64-meta", +] + +[[package]] +name = "cranelift-assembler-x64-meta" +version = "0.123.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b6e4ce8ee6d899381fbdd9e6561336c651189d46cecaeee09b29e8d80aa786e" +dependencies = [ + "cranelift-srcgen", +] + [[package]] name = "cranelift-bforest" -version = "0.116.1" +version = "0.123.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e15d04a0ce86cb36ead88ad68cf693ffd6cda47052b9e0ac114bc47fd9cd23c4" +checksum = "0cb6d37015df7ea4b60450c1229ad5f5819a1fb27434b063f8e6216dfbd0c42a" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-bitset" -version = "0.116.1" +version = "0.123.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c6e3969a7ce267259ce244b7867c5d3bc9e65b0a87e81039588dfdeaede9f34" +checksum = "986bea0b0858b55192782120032ce9c15943fa073f186f6e479653c59e62c329" dependencies = [ "serde", "serde_derive", @@ -1169,11 +1341,12 @@ dependencies = [ [[package]] name = "cranelift-codegen" -version = "0.116.1" +version = "0.123.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c22032c4cb42558371cf516bb47f26cdad1819d3475c133e93c49f50ebf304e" +checksum = "9f30aeb2de7f97d6f26b4a1642615834daad58e2e4d7c027810010a3a32f22be" dependencies = [ "bumpalo", + "cranelift-assembler-x64", "cranelift-bforest", "cranelift-bitset", "cranelift-codegen-meta", @@ -1181,45 +1354,51 @@ dependencies = [ "cranelift-control", "cranelift-entity", "cranelift-isle", - "gimli 0.31.1", - "hashbrown 0.14.5", + "gimli", + "hashbrown 0.15.5", "log", + "pulley-interpreter", "regalloc2", "rustc-hash 2.1.1", "serde", "smallvec", "target-lexicon", + "wasmtime-internal-math", ] [[package]] name = "cranelift-codegen-meta" -version = "0.116.1" +version = "0.123.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c904bc71c61b27fc57827f4a1379f29de64fe95653b620a3db77d59655eee0b8" +checksum = "cd5dd137fcdedef33b6fd40edf1ced024460d764ceb75833e8198a843395945c" dependencies = [ + "cranelift-assembler-x64-meta", "cranelift-codegen-shared", + "cranelift-srcgen", + "heck", + "pulley-interpreter", ] [[package]] name = "cranelift-codegen-shared" -version = "0.116.1" +version = "0.123.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40180f5497572f644ce88c255480981ae2ec1d7bb4d8e0c0136a13b87a2f2ceb" +checksum = "ab54b260ef23a8f0f536679b9fc3b3b3e05353e8d1448f3ab83df02078e8be9b" [[package]] name = "cranelift-control" -version = "0.116.1" +version = "0.123.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d132c6d0bd8a489563472afc171759da0707804a65ece7ceb15a8c6d7dd5ef" +checksum = "3f3e569779ad70537f34a670d444ee3d75ae583b2023913f4682814b0979f7e8" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.116.1" +version = "0.123.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b2d0d9618275474fbf679dd018ac6e009acbd6ae6850f6a67be33fb3b00b323" +checksum = "2ff53acc85f5c5f7d9315ff133a6671d329a0f04aa2d1a8a2e81d59709ccddcb" dependencies = [ "cranelift-bitset", "serde", @@ -1228,9 +1407,9 @@ dependencies = [ [[package]] name = "cranelift-frontend" -version = "0.116.1" +version = "0.123.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fac41e16729107393174b0c9e3730fb072866100e1e64e80a1a963b2e484d57" +checksum = "ab5976c0ff5bfadf61cd8bda81fea78ee5a07018b9cd03e66c0952c56684928b" dependencies = [ "cranelift-codegen", "log", @@ -1240,21 +1419,27 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.116.1" +version = "0.123.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ca20d576e5070044d0a72a9effc2deacf4d6aa650403189d8ea50126483944d" +checksum = "77b4f73d2288e9480fd2d1d9ab576394dce4805443d6148c6d819dbf78865ce4" [[package]] name = "cranelift-native" -version = "0.116.1" +version = "0.123.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8dee82f3f1f2c4cba9177f1cc5e350fe98764379bcd29340caa7b01f85076c7" +checksum = "fe9650c2baf22fa1e2542a5bdd8152616ec2023d929c4cbb450ff677ad8d9c21" dependencies = [ "cranelift-codegen", "libc", "target-lexicon", ] +[[package]] +name = "cranelift-srcgen" +version = "0.123.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ad4f61ae701d73c326d3df08c366b29ad10f1ba06c245092f217b8d2306746b" + [[package]] name = "crc" version = "3.4.0" @@ -1448,6 +1633,7 @@ dependencies = [ "reqwest", "serde", "serde_json", + "temper-authz", "temper-jit", "temper-runtime", "temper-server", @@ -1730,33 +1916,13 @@ dependencies = [ "dirs-sys-next", ] -[[package]] -name = "dirs" -version = "4.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" -dependencies = [ - "dirs-sys 0.3.7", -] - [[package]] name = "dirs" version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" dependencies = [ - "dirs-sys 0.5.0", -] - -[[package]] -name = "dirs-sys" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" -dependencies = [ - "libc", - "redox_users 0.4.6", - "winapi", + "dirs-sys", ] [[package]] @@ -1992,6 +2158,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + [[package]] name = "event-listener" version = "5.4.1" @@ -2003,6 +2175,16 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener 5.4.1", + "pin-project-lite", +] + [[package]] name = "fallible-iterator" version = "0.2.0" @@ -2084,6 +2266,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + [[package]] name = "fixedbitset" version = "0.5.7" @@ -2260,6 +2448,19 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" +[[package]] +name = "futures-lite" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + [[package]] name = "futures-macro" version = "0.3.32" @@ -2418,27 +2619,33 @@ dependencies = [ [[package]] name = "gimli" -version = "0.31.1" +version = "0.32.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" dependencies = [ "fallible-iterator 0.3.0", "indexmap 2.13.0", "stable_deref_trait", ] -[[package]] -name = "gimli" -version = "0.32.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" - [[package]] name = "glob" version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +[[package]] +name = "gloo-timers" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "h2" version = "0.3.27" @@ -3247,6 +3454,15 @@ dependencies = [ "cpufeatures 0.2.17", ] +[[package]] +name = "kv-log-macro" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" +dependencies = [ + "log", +] + [[package]] name = "lalrpop" version = "0.22.2" @@ -3258,7 +3474,7 @@ dependencies = [ "ena", "itertools 0.14.0", "lalrpop-util", - "petgraph", + "petgraph 0.7.1", "pico-args", "regex", "regex-syntax", @@ -3565,6 +3781,9 @@ name = "log" version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +dependencies = [ + "value-bag", +] [[package]] name = "logos" @@ -3814,6 +4033,12 @@ dependencies = [ "strum", ] +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + [[package]] name = "native-tls" version = "0.2.18" @@ -4009,9 +4234,9 @@ dependencies = [ [[package]] name = "object" -version = "0.36.7" +version = "0.37.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" dependencies = [ "crc32fast", "hashbrown 0.15.5", @@ -4019,15 +4244,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "object" -version = "0.37.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" -dependencies = [ - "memchr", -] - [[package]] name = "oncall-reference" version = "0.1.0" @@ -4209,6 +4425,7 @@ version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84dfad6042089c7fc1f6118b7040dc2eb4ab520abbf410b79dc481032af39570" dependencies = [ + "async-std", "async-trait", "futures-channel", "futures-executor", @@ -4219,6 +4436,8 @@ dependencies = [ "rand 0.8.5", "serde_json", "thiserror 2.0.18", + "tokio", + "tokio-stream", "tracing", ] @@ -4291,12 +4510,6 @@ dependencies = [ "syn", ] -[[package]] -name = "paste" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" - [[package]] name = "pathdiff" version = "0.2.3" @@ -4324,13 +4537,23 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "petgraph" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" +dependencies = [ + "fixedbitset 0.4.2", + "indexmap 2.13.0", +] + [[package]] name = "petgraph" version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" dependencies = [ - "fixedbitset", + "fixedbitset 0.5.7", "indexmap 2.13.0", ] @@ -4430,6 +4653,17 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "piper" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c835479a4443ded371d6c535cbfd8d31ad92c5d23ae9770a61bc155e4992a3c1" +dependencies = [ + "atomic-waker", + "fastrand", + "futures-io", +] + [[package]] name = "pkcs1" version = "0.7.5" @@ -4491,6 +4725,20 @@ dependencies = [ "plotters-backend", ] +[[package]] +name = "polling" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" +dependencies = [ + "cfg-if", + "concurrent-queue", + "hermit-abi", + "pin-project-lite", + "rustix 1.1.4", + "windows-sys 0.61.2", +] + [[package]] name = "polyval" version = "0.6.2" @@ -4584,8 +4832,10 @@ dependencies = [ "log", "nix", "once_cell", - "protobuf", - "protobuf-codegen-pure", + "prost 0.12.6", + "prost-build", + "prost-derive 0.12.6", + "sha2 0.10.9", "smallvec", "spin 0.10.0", "symbolic-demangle", @@ -4698,6 +4948,27 @@ dependencies = [ "prost-derive 0.14.3", ] +[[package]] +name = "prost-build" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" +dependencies = [ + "bytes", + "heck", + "itertools 0.10.5", + "log", + "multimap", + "once_cell", + "petgraph 0.6.5", + "prettyplease", + "prost 0.12.6", + "prost-types 0.12.6", + "regex", + "syn", + "tempfile", +] + [[package]] name = "prost-derive" version = "0.12.6" @@ -4739,36 +5010,20 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.14.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" -dependencies = [ - "prost 0.14.3", -] - -[[package]] -name = "protobuf" -version = "2.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" - -[[package]] -name = "protobuf-codegen" -version = "2.28.0" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "033460afb75cf755fcfc16dfaed20b86468082a2ea24e05ac35ab4a099a017d6" +checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0" dependencies = [ - "protobuf", + "prost 0.12.6", ] [[package]] -name = "protobuf-codegen-pure" -version = "2.28.0" +name = "prost-types" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95a29399fc94bcd3eeaa951c715f7bea69409b2445356b00519740bcd6ddd865" +checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" dependencies = [ - "protobuf", - "protobuf-codegen", + "prost 0.14.3", ] [[package]] @@ -4783,14 +5038,25 @@ dependencies = [ [[package]] name = "pulley-interpreter" -version = "29.0.1" +version = "36.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62d95f8575df49a2708398182f49a888cf9dc30210fb1fd2df87c889edcee75d" +checksum = "eb0a4b56042e461cc64456650182938e2d1ede98fa0c8a975027416a2809c414" dependencies = [ "cranelift-bitset", "log", - "sptr", - "wasmtime-math", + "pulley-macros", + "wasmtime-internal-math", +] + +[[package]] +name = "pulley-macros" +version = "36.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "244667bea2e214273442a71f26adb12b88a41f66718fb2c6eea47c00f0dc325f" +dependencies = [ + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -4810,9 +5076,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quick-xml" -version = "0.37.5" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" dependencies = [ "memchr", "serde", @@ -5117,9 +5383,9 @@ dependencies = [ [[package]] name = "regalloc2" -version = "0.11.2" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc06e6b318142614e4a48bc725abbf08ff166694835c43c9dae5a9009704639a" +checksum = "5216b1837de2149f8bc8e6d5f88a9326b63b8c836ed58ce4a0a29ec736a59734" dependencies = [ "allocator-api2", "bumpalo", @@ -5775,15 +6041,6 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc6fe69c597f9c37bfeeeeeb33da3530379845f10be461a66d16d03eca2ded77" -[[package]] -name = "shellexpand" -version = "2.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ccc8076840c4da029af4f87e4e8daeb0fca6b87bbb02e10cb60b791450e11e4" -dependencies = [ - "dirs 4.0.0", -] - [[package]] name = "shlex" version = "1.3.0" @@ -5910,12 +6167,6 @@ dependencies = [ "der", ] -[[package]] -name = "sptr" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b9b39299b249ad65f3b7e96443bad61c02ca5cd3589f46cb6d610a0fd6c0d6a" - [[package]] name = "sqlx" version = "0.8.6" @@ -5941,7 +6192,7 @@ dependencies = [ "crc", "crossbeam-queue", "either", - "event-listener", + "event-listener 5.4.1", "futures-core", "futures-intrusive", "futures-io", @@ -6354,7 +6605,7 @@ dependencies = [ "chrono", "deadpool-postgres", "prost 0.14.3", - "prost-types", + "prost-types 0.14.3", "serde", "serde_json", "temper-jit", @@ -6414,7 +6665,7 @@ dependencies = [ "console", "deadpool-postgres", "dialoguer", - "dirs 6.0.0", + "dirs", "dotenvy", "indicatif", "open", @@ -6511,6 +6762,7 @@ dependencies = [ "serde_json", "sha2 0.10.9", "temper-ots", + "temper-platform", "temper-runtime", "temper-sandbox", "temper-server", @@ -6591,6 +6843,7 @@ dependencies = [ "axum 0.8.8", "base64 0.22.1", "chrono", + "futures", "hyper 1.8.1", "opentelemetry", "reqwest", @@ -6645,6 +6898,7 @@ dependencies = [ "serde_json", "sha2 0.10.9", "tokio", + "tracing", "uuid", ] @@ -6697,6 +6951,7 @@ dependencies = [ "temper-jit", "temper-observe", "temper-odata", + "temper-ots", "temper-platform", "temper-runtime", "temper-sandbox", @@ -7547,17 +7802,6 @@ dependencies = [ "tracing-serde", ] -[[package]] -name = "trait-variant" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70977707304198400eb4835a78f6a9f928bf41bba420deb8fdb175cd965d77a7" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "try-lock" version = "0.2.5" @@ -7781,6 +8025,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "value-bag" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "068e763e8279de7ab94b6afebded2cb701678af094feb1c12ccb061b4783c1be" + [[package]] name = "vcpkg" version = "0.2.15" @@ -7930,12 +8180,12 @@ dependencies = [ [[package]] name = "wasm-encoder" -version = "0.221.3" +version = "0.236.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc8444fe4920de80a4fe5ab564fff2ae58b6b73166b89751f8c6c93509da32e5" +checksum = "724fccfd4f3c24b7e589d333fc0429c68042897a7e8a5f8694f31792471841e7" dependencies = [ - "leb128", - "wasmparser 0.221.3", + "leb128fmt", + "wasmparser 0.236.1", ] [[package]] @@ -7985,9 +8235,9 @@ dependencies = [ [[package]] name = "wasmparser" -version = "0.221.3" +version = "0.236.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d06bfa36ab3ac2be0dee563380147a5b81ba10dd8885d7fbbc9eb574be67d185" +checksum = "a9b1e81f3eb254cf7404a82cee6926a4a3ccc5aad80cc3d43608a070c67aa1d7" dependencies = [ "bitflags 2.11.0", "hashbrown 0.15.5", @@ -8021,22 +8271,22 @@ dependencies = [ [[package]] name = "wasmprinter" -version = "0.221.3" +version = "0.236.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7343c42a97f2926c7819ff81b64012092ae954c5d83ddd30c9fcdefd97d0b283" +checksum = "2df225df06a6df15b46e3f73ca066ff92c2e023670969f7d50ce7d5e695abbb1" dependencies = [ "anyhow", "termcolor", - "wasmparser 0.221.3", + "wasmparser 0.236.1", ] [[package]] name = "wasmtime" -version = "29.0.1" +version = "36.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11976a250672556d1c4c04c6d5d7656ac9192ac9edc42a4587d6c21460010e69" +checksum = "7d05c745dc0978e589ef295958f3130122afc33d96af6bad3f0f06dbe7ac43a8" dependencies = [ - "addr2line 0.24.2", + "addr2line", "anyhow", "async-trait", "bitflags 2.11.0", @@ -8045,104 +8295,128 @@ dependencies = [ "cfg-if", "encoding_rs", "fxprof-processed-profile", - "gimli 0.31.1", - "hashbrown 0.14.5", + "gimli", + "hashbrown 0.15.5", "indexmap 2.13.0", "ittapi", "libc", "log", "mach2", "memfd", - "object 0.36.7", + "object", "once_cell", - "paste", "postcard", - "psm", "pulley-interpreter", "rayon", - "rustix 0.38.44", + "rustix 1.1.4", "semver", "serde", "serde_derive", "serde_json", "smallvec", - "sptr", "target-lexicon", - "trait-variant", - "wasm-encoder 0.221.3", - "wasmparser 0.221.3", - "wasmtime-asm-macros", - "wasmtime-cache", - "wasmtime-component-macro", - "wasmtime-component-util", - "wasmtime-cranelift", + "wasm-encoder 0.236.1", + "wasmparser 0.236.1", "wasmtime-environ", - "wasmtime-fiber", - "wasmtime-jit-debug", - "wasmtime-jit-icache-coherence", - "wasmtime-math", - "wasmtime-slab", - "wasmtime-versioned-export-macros", - "wasmtime-winch", + "wasmtime-internal-asm-macros", + "wasmtime-internal-cache", + "wasmtime-internal-component-macro", + "wasmtime-internal-component-util", + "wasmtime-internal-cranelift", + "wasmtime-internal-fiber", + "wasmtime-internal-jit-debug", + "wasmtime-internal-jit-icache-coherence", + "wasmtime-internal-math", + "wasmtime-internal-slab", + "wasmtime-internal-unwinder", + "wasmtime-internal-versioned-export-macros", + "wasmtime-internal-winch", "wat", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] -name = "wasmtime-asm-macros" -version = "29.0.1" +name = "wasmtime-environ" +version = "36.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f178b0d125201fbe9f75beaf849bd3e511891f9e45ba216a5b620802ccf64f2" +checksum = "9fd1d43cfaa1a0859d2f4fccc15e7e571e2a88b357e81bc88ba6c501b83d925d" +dependencies = [ + "anyhow", + "cpp_demangle", + "cranelift-bitset", + "cranelift-entity", + "gimli", + "indexmap 2.13.0", + "log", + "object", + "postcard", + "rustc-demangle", + "semver", + "serde", + "serde_derive", + "smallvec", + "target-lexicon", + "wasm-encoder 0.236.1", + "wasmparser 0.236.1", + "wasmprinter", + "wasmtime-internal-component-util", +] + +[[package]] +name = "wasmtime-internal-asm-macros" +version = "36.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "515dd7158bf1719b41290cd2e6a2a46ec944484146816992f195af3720e49b3f" dependencies = [ "cfg-if", ] [[package]] -name = "wasmtime-cache" -version = "29.0.1" +name = "wasmtime-internal-cache" +version = "36.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b1161c8f62880deea07358bc40cceddc019f1c81d46007bc390710b2fe24ffc" +checksum = "12a53145473629ea40f445235ed95182b76940f00a67c3c9c6c4857dae0ad823" dependencies = [ "anyhow", - "base64 0.21.7", + "base64 0.22.1", "directories-next", "log", "postcard", - "rustix 0.38.44", + "rustix 1.1.4", "serde", "serde_derive", "sha2 0.10.9", "toml", - "windows-sys 0.59.0", + "windows-sys 0.60.2", "zstd", ] [[package]] -name = "wasmtime-component-macro" -version = "29.0.1" +name = "wasmtime-internal-component-macro" +version = "36.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d74de6592ed945d0a602f71243982a304d5d02f1e501b638addf57f42d57dfaf" +checksum = "dfca017b7daa80ff217c66f105ee20e674d87b7c11dca85bbb9d0146e9f443fb" dependencies = [ "anyhow", "proc-macro2", "quote", "syn", - "wasmtime-component-util", - "wasmtime-wit-bindgen", - "wit-parser 0.221.3", + "wasmtime-internal-component-util", + "wasmtime-internal-wit-bindgen", + "wit-parser 0.236.1", ] [[package]] -name = "wasmtime-component-util" -version = "29.0.1" +name = "wasmtime-internal-component-util" +version = "36.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707dc7b3c112ab5a366b30cfe2fb5b2f8e6a0f682f16df96a5ec582bfe6f056e" +checksum = "1c3e218b51d2ef9eb181499e42c691512c1bc11dd6dc7746807a9b2b9290369c" [[package]] -name = "wasmtime-cranelift" -version = "29.0.1" +name = "wasmtime-internal-cranelift" +version = "36.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "366be722674d4bf153290fbcbc4d7d16895cc82fb3e869f8d550ff768f9e9e87" +checksum = "5ba1736927b58e50e741e407da7c037c0250f3e213833a09c89dcd8f73ae2eac" dependencies = [ "anyhow", "cfg-if", @@ -8151,114 +8425,134 @@ dependencies = [ "cranelift-entity", "cranelift-frontend", "cranelift-native", - "gimli 0.31.1", - "itertools 0.12.1", + "gimli", + "itertools 0.14.0", "log", - "object 0.36.7", + "object", + "pulley-interpreter", "smallvec", "target-lexicon", - "thiserror 1.0.69", - "wasmparser 0.221.3", + "thiserror 2.0.18", + "wasmparser 0.236.1", "wasmtime-environ", - "wasmtime-versioned-export-macros", -] - -[[package]] -name = "wasmtime-environ" -version = "29.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdadc1af7097347aa276a4f008929810f726b5b46946971c660b6d421e9994ad" -dependencies = [ - "anyhow", - "cpp_demangle", - "cranelift-bitset", - "cranelift-entity", - "gimli 0.31.1", - "indexmap 2.13.0", - "log", - "object 0.36.7", - "postcard", - "rustc-demangle", - "semver", - "serde", - "serde_derive", - "smallvec", - "target-lexicon", - "wasm-encoder 0.221.3", - "wasmparser 0.221.3", - "wasmprinter", - "wasmtime-component-util", + "wasmtime-internal-math", + "wasmtime-internal-versioned-export-macros", ] [[package]] -name = "wasmtime-fiber" -version = "29.0.1" +name = "wasmtime-internal-fiber" +version = "36.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccba90d4119f081bca91190485650730a617be1fff5228f8c4757ce133d21117" +checksum = "7b238e4c20bddb900ec0cb380252d63e8d0644fd94de001119574f5921e895d9" dependencies = [ "anyhow", "cc", "cfg-if", - "rustix 0.38.44", - "wasmtime-asm-macros", - "wasmtime-versioned-export-macros", - "windows-sys 0.59.0", + "libc", + "rustix 1.1.4", + "wasmtime-internal-asm-macros", + "wasmtime-internal-versioned-export-macros", + "windows-sys 0.60.2", ] [[package]] -name = "wasmtime-jit-debug" -version = "29.0.1" +name = "wasmtime-internal-jit-debug" +version = "36.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e7b61488a5ee00c35c8c22de707c36c0aecacf419a3be803a6a2ba5e860f56a" +checksum = "8f259b13685ad51e3dcf58cb69031279ed0d79c25bc3ccc8b50e7160ed04fbfe" dependencies = [ - "object 0.36.7", - "rustix 0.38.44", - "wasmtime-versioned-export-macros", + "cc", + "object", + "rustix 1.1.4", + "wasmtime-internal-versioned-export-macros", ] [[package]] -name = "wasmtime-jit-icache-coherence" -version = "29.0.1" +name = "wasmtime-internal-jit-icache-coherence" +version = "36.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec5e8552e01692e6c2e5293171704fed8abdec79d1a6995a0870ab190e5747d1" +checksum = "41fed85537936b16460bac352ad149052c025db50467c7bc539dd47b31439374" dependencies = [ "anyhow", "cfg-if", "libc", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] -name = "wasmtime-math" -version = "29.0.1" +name = "wasmtime-internal-math" +version = "36.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29210ec2aa25e00f4d54605cedaf080f39ec01a872c5bd520ad04c67af1dde17" +checksum = "82fff10da41d0d15d90ebba70946a0aa16ed0957ae7b77e0b6d2a46e8221e555" dependencies = [ "libm", ] [[package]] -name = "wasmtime-slab" -version = "29.0.1" +name = "wasmtime-internal-slab" +version = "36.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcb5821a96fa04ac14bc7b158bb3d5cd7729a053db5a74dad396cd513a5e5ccf" +checksum = "e44a8c097bab08d349d57dce1ab818859fefbe261ab3632b38fe127b1b551108" [[package]] -name = "wasmtime-versioned-export-macros" -version = "29.0.1" +name = "wasmtime-internal-unwinder" +version = "36.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86ff86db216dc0240462de40c8290887a613dddf9685508eb39479037ba97b5b" +checksum = "7f40a57d5e7c221ce56391d7dca0a918ba17ea00185462c7facbf534d7745184" +dependencies = [ + "anyhow", + "cfg-if", + "cranelift-codegen", + "log", + "object", +] + +[[package]] +name = "wasmtime-internal-versioned-export-macros" +version = "36.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e085bfce1cb2089dbeef6e280a5d598666923d3dcd308712fe429fe43c9d19f5" dependencies = [ "proc-macro2", "quote", "syn", ] +[[package]] +name = "wasmtime-internal-winch" +version = "36.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4916cd526e1ce294984cc5b70264cfc0ca103b41ca665b58728bde250b6b82f" +dependencies = [ + "anyhow", + "cranelift-codegen", + "gimli", + "object", + "target-lexicon", + "wasmparser 0.236.1", + "wasmtime-environ", + "wasmtime-internal-cranelift", + "winch-codegen", +] + +[[package]] +name = "wasmtime-internal-wit-bindgen" +version = "36.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ad2f9d9c3baa70ee4d157b55b4c08e5dc00f1d80aad3692b1e0806393914ea" +dependencies = [ + "anyhow", + "bitflags 2.11.0", + "heck", + "indexmap 2.13.0", + "wit-parser 0.236.1", +] + [[package]] name = "wasmtime-wasi" -version = "29.0.1" +version = "36.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d1be69bfcab1bdac74daa7a1f9695ab992b9c8e21b9b061e7d66434097e0ca4" +checksum = "841c11707aaeaf09895677757614d83a23c45d2b2689df908d0005aa191a933b" dependencies = [ "anyhow", "async-trait", @@ -8273,45 +8567,29 @@ dependencies = [ "futures", "io-extras", "io-lifetimes", - "rustix 0.38.44", + "rustix 1.1.4", "system-interface", - "thiserror 1.0.69", + "thiserror 2.0.18", "tokio", "tracing", - "trait-variant", "url", "wasmtime", + "wasmtime-wasi-io", "wiggle", - "windows-sys 0.59.0", -] - -[[package]] -name = "wasmtime-winch" -version = "29.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdbabfb8f20502d5e1d81092b9ead3682ae59988487aafcd7567387b7a43cf8f" -dependencies = [ - "anyhow", - "cranelift-codegen", - "gimli 0.31.1", - "object 0.36.7", - "target-lexicon", - "wasmparser 0.221.3", - "wasmtime-cranelift", - "wasmtime-environ", - "winch-codegen", + "windows-sys 0.60.2", ] [[package]] -name = "wasmtime-wit-bindgen" -version = "29.0.1" +name = "wasmtime-wasi-io" +version = "36.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8358319c2dd1e4db79e3c1c5d3a5af84956615343f9f89f4e4996a36816e06e6" +checksum = "8b76b03de2cba3c036f81074c17bb1c7b77662578424887ebbc090b4c23ba33e" dependencies = [ "anyhow", - "heck", - "indexmap 2.13.0", - "wit-parser 0.221.3", + "async-trait", + "bytes", + "futures", + "wasmtime", ] [[package]] @@ -8420,14 +8698,14 @@ dependencies = [ [[package]] name = "wiggle" -version = "29.0.1" +version = "36.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b9af35bc9629c52c261465320a9a07959164928b4241980ba1cf923b9e6751d" +checksum = "a333010f4b8b770a500e181e0abda564dde49a51db21aed4307fde800746ff97" dependencies = [ "anyhow", "async-trait", "bitflags 2.11.0", - "thiserror 1.0.69", + "thiserror 2.0.18", "tracing", "wasmtime", "wiggle-macro", @@ -8435,24 +8713,23 @@ dependencies = [ [[package]] name = "wiggle-generate" -version = "29.0.1" +version = "36.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cf267dd05673912c8138f4b54acabe6bd53407d9d1536f0fadb6520dd16e101" +checksum = "1e909aa247f90d2ba77b6860b36288cfeeb559fe01b140165f077b250cb1ba56" dependencies = [ "anyhow", "heck", "proc-macro2", "quote", - "shellexpand", "syn", "witx", ] [[package]] name = "wiggle-macro" -version = "29.0.1" +version = "36.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08c5c473d4198e6c2d377f3809f713ff0c110cab88a0805ae099a82119ee250c" +checksum = "a84de50b5bf6530a15bbb228043561d525c8e21c8929693c3078f7cad2051ad3" dependencies = [ "proc-macro2", "quote", @@ -8493,20 +8770,22 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "winch-codegen" -version = "29.0.1" +version = "36.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f849ef2c5f46cb0a20af4b4487aaa239846e52e2c03f13fa3c784684552859c" +checksum = "e826c012c68403725e77adf6b904c2ea809e5d464aaf25aa6eda14559300b3df" dependencies = [ "anyhow", + "cranelift-assembler-x64", "cranelift-codegen", - "gimli 0.31.1", + "gimli", "regalloc2", "smallvec", "target-lexicon", - "thiserror 1.0.69", - "wasmparser 0.221.3", - "wasmtime-cranelift", + "thiserror 2.0.18", + "wasmparser 0.236.1", "wasmtime-environ", + "wasmtime-internal-cranelift", + "wasmtime-internal-math", ] [[package]] @@ -8924,9 +9203,9 @@ dependencies = [ [[package]] name = "wit-parser" -version = "0.221.3" +version = "0.236.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "896112579ed56b4a538b07a3d16e562d101ff6265c46b515ce0c701eef16b2ac" +checksum = "16e4833a20cd6e85d6abfea0e63a399472d6f88c6262957c17f546879a80ba15" dependencies = [ "anyhow", "id-arena", @@ -8937,7 +9216,7 @@ dependencies = [ "serde_derive", "serde_json", "unicode-xid", - "wasmparser 0.221.3", + "wasmparser 0.236.1", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 91416d5f4..fe0f4ab6e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -102,7 +102,7 @@ tracing-opentelemetry = "0.29" dotenvy = "0.15" # XML (CSDL parsing) -quick-xml = { version = "0.37", features = ["serialize"] } +quick-xml = { version = "0.41", features = ["serialize"] } # CLI clap = { version = "4", features = ["derive"] } @@ -161,9 +161,11 @@ temper-transport = { path = "crates/temper-transport" } temper-actor-runtime = { path = "crates/temper-actor-runtime" } temper-agents = { path = "crates/temper-agents" } -# WASM runtime -wasmtime = { version = "29", features = ["component-model", "profiling"] } -wasmtime-wasi = "29" +# WASM runtime. Floor is 36.0.13: 36.0.7 fixes RUSTSEC-2026-0096 (the ARN-169 +# objective), and 36.0.13 additionally fixes RUSTSEC-2026-0222, so the lower bound +# must not admit an earlier 36.x patch. +wasmtime = { version = "36.0.13", features = ["component-model", "profiling"] } +wasmtime-wasi = "36.0.13" sha2 = "0.10" hostname = "0.4" aes-gcm = "0.10" diff --git a/crates/temper-authz/src/context.rs b/crates/temper-authz/src/context.rs index 36eb26f5d..052cf8a20 100644 --- a/crates/temper-authz/src/context.rs +++ b/crates/temper-authz/src/context.rs @@ -3,6 +3,7 @@ use std::collections::HashMap; use serde::{Deserialize, Serialize}; +use temper_runtime::tenant::TenantId; /// The kind of principal making the request. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -34,8 +35,18 @@ pub struct Principal { pub attributes: HashMap, } +/// Cedar context keys that only a resolved `SecurityContext` may populate. +pub fn is_cedar_authority_context_key(name: &str) -> bool { + matches!( + name, + "sessionId" | "agentId" | "agentType" | "agentTypeVerified" | "role" | "actingFor" + ) +} + /// Security context carried with every actor message dispatch. -/// Constructed from HTTP request headers at the server boundary. +/// +/// Protected HTTP requests receive this from credential resolution. The legacy +/// header constructor intentionally produces only anonymous authority. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct SecurityContext { /// The principal making the request. @@ -46,94 +57,139 @@ pub struct SecurityContext { pub correlation_id: String, } +/// Credential-authenticated authority bound to one tenant. +/// +/// Authentication middleware constructs this once after resolving a credential. +/// Downstream handlers receive the value as an axum extension or a direct +/// in-process argument. Private fields prevent handlers from replacing only the +/// tenant or only the principal and accidentally creating a mixed context. +#[derive(Debug, Clone)] +pub struct AuthenticatedRequestContext { + tenant: TenantId, + security_context: SecurityContext, + /// Caller-declared intent for this request, for telemetry only. + /// + /// Deliberately NOT part of `security_context.context_attrs`: those are + /// Cedar inputs, and this value is caller-supplied. A denial record without + /// the intent behind it says what was blocked but not what the caller was + /// trying to do — the half that drives policy proposals — so it is carried + /// here, where authorization cannot read it. + intent: Option, + /// Caller-declared session id for this request, for telemetry only. + /// + /// Held here for the same reason as `intent`, and the reason is sharper: + /// Cedar policies condition on `context.sessionId` (session-scoped permits + /// generated from an approved decision). Routing a caller-supplied header + /// into `context_attrs` would let any caller satisfy the session scope that + /// made such an approval narrow, replaying it indefinitely. + /// + /// The asserted header becomes a Cedar input only through the validated + /// path: the bearer edge checks it against the server-side grant record (an + /// approved decision binding that session to this principal) and only then + /// passes it into the resolved `SecurityContext`. This field always carries + /// the raw assertion for telemetry, validated or not. + session_id: Option, +} + +impl AuthenticatedRequestContext { + /// Bind an already-resolved security context to its authenticated tenant. + pub fn new(tenant: TenantId, security_context: SecurityContext) -> Self { + Self { + tenant, + security_context, + intent: None, + session_id: None, + } + } + + /// Attach the caller-declared intent from the correlation headers. + #[must_use] + pub fn with_intent(mut self, intent: Option) -> Self { + self.intent = intent; + self + } + + /// Caller-declared intent, for denial telemetry. Never an authorization input. + pub fn intent(&self) -> Option<&str> { + self.intent.as_deref() + } + + /// Attach the caller-declared session id from the correlation headers. + #[must_use] + pub fn with_session_id(mut self, session_id: Option) -> Self { + self.session_id = session_id; + self + } + + /// Caller-declared session id, for telemetry. Never an authorization input. + pub fn session_id(&self) -> Option<&str> { + self.session_id.as_deref() + } + + /// Tenant selected during credential resolution. + pub fn tenant(&self) -> &TenantId { + &self.tenant + } + + /// Exact security context produced by credential resolution. + pub fn security_context(&self) -> &SecurityContext { + &self.security_context + } +} + impl SecurityContext { - /// Create a security context from HTTP request headers. - /// In production, this would validate JWT tokens, API keys, etc. - /// For now, extracts from X-Temper-* headers. + /// The anonymous principal used for routes declared public. + /// + /// Explicitly constructed rather than "derived from no headers", so no + /// production path expresses identity as a function of request headers. + pub fn anonymous() -> Self { + SecurityContext { + principal: Principal { + id: "anonymous".to_string(), + kind: PrincipalKind::Customer, + role: None, + acting_for: None, + agent_type: None, + attributes: HashMap::new(), + }, + context_attrs: HashMap::new(), + correlation_id: uuid::Uuid::now_v7().to_string(), + } + } + + /// Build a context from request headers. + /// + /// Test-only since ADR-0157: identity comes from a resolved credential, and + /// keeping this out of the production build makes header-derived identity + /// impossible to reintroduce by accident rather than merely unused. + #[cfg(test)] pub fn from_headers(headers: &[(String, String)]) -> Self { - let mut principal_id = "anonymous".to_string(); - // Default to Customer (most restrictive). System bypass is only via - // SecurityContext::system() or explicit "system" header value. - let mut kind = PrincipalKind::Customer; - let mut role = None; - let mut acting_for = None; - let mut agent_type = None; - let mut attributes = HashMap::new(); - let mut context_attrs = HashMap::new(); let mut correlation_id = uuid::Uuid::now_v7().to_string(); for (key, value) in headers { - match key.to_lowercase().as_str() { - "x-temper-principal-id" => principal_id = value.clone(), - "x-temper-principal-kind" => { - kind = match value.as_str() { - "customer" => PrincipalKind::Customer, - "agent" => PrincipalKind::Agent, - "admin" => PrincipalKind::Admin, - // "system" is NOT accepted from headers to prevent - // privilege escalation via header spoofing. Use - // SecurityContext::system() for trusted internal paths. - _ => PrincipalKind::Customer, - }; - } - "x-temper-agent-role" => role = Some(value.clone()), - "x-temper-acting-for" => acting_for = Some(value.clone()), - "x-temper-agent-type" => agent_type = Some(value.clone()), - "x-temper-principal-scopes" => { - let scopes: Vec = value - .split(|c: char| c == ',' || c == ';' || c.is_whitespace()) - .filter(|s| !s.is_empty()) - .map(|s| serde_json::Value::String(s.to_string())) - .collect(); - attributes.insert("scopes".to_string(), serde_json::Value::Array(scopes)); - } - "x-temper-action-context" => { - attributes.insert( - "action_context".to_string(), - serde_json::Value::String(value.clone()), - ); - } - "x-temper-correlation-id" => correlation_id = value.clone(), - k if k.starts_with("x-temper-attr-") => { - let attr_name = k.strip_prefix("x-temper-attr-").unwrap(); // ci-ok: guarded by starts_with - attributes.insert( - attr_name.to_string(), - serde_json::Value::String(value.clone()), - ); - } - k if k.starts_with("x-temper-ctx-") => { - let raw_ctx_name = k.strip_prefix("x-temper-ctx-").unwrap(); // ci-ok: guarded by starts_with - let ctx_name = match raw_ctx_name { - "agentid" => "agentId", - "agenttype" => "agentType", - "agenttypeverified" => "agentTypeVerified", - "sessionid" => "sessionId", - other => other, - }; - context_attrs.insert( - ctx_name.to_string(), - serde_json::Value::String(value.clone()), - ); - } - _ => {} + if key.eq_ignore_ascii_case("x-temper-correlation-id") { + correlation_id = value.clone(); } } SecurityContext { principal: Principal { - id: principal_id, - kind, - role, - acting_for, - agent_type, - attributes, + id: "anonymous".to_string(), + kind: PrincipalKind::Customer, + role: None, + acting_for: None, + agent_type: None, + attributes: HashMap::new(), }, - context_attrs, + context_attrs: HashMap::new(), correlation_id, } } - /// Create a system-level security context (bypasses all checks). + /// Create a system-level security context for explicit in-process work. + /// + /// System requests still pass Cedar evaluation through the built-in system + /// policy; this constructor is never reachable from HTTP headers. pub fn system() -> Self { SecurityContext { principal: Principal { @@ -200,11 +256,11 @@ impl SecurityContext { } } - /// Enrich security context with agent identity from self-declared headers. + /// Enrich an internal context with an explicitly unverified agent identity. /// - /// **Deprecated**: Use `from_resolved_identity()` for credential-based identity. - /// This method is retained only for the global API key path (admin/operator access) - /// where no agent credential exists. + /// **Deprecated**: Use `from_resolved_identity()` for credential-based + /// identity. This compatibility helper is limited to legacy internal + /// trigger/service construction; callers must never pass raw HTTP values. pub fn with_agent_context( mut self, agent_id: Option<&str>, @@ -268,7 +324,7 @@ mod tests { use super::*; #[test] - fn test_context_from_headers_customer() { + fn customer_headers_are_non_authoritative() { let headers = vec![ ("X-Temper-Principal-Id".to_string(), "cust-123".to_string()), ( @@ -278,13 +334,25 @@ mod tests { ]; let ctx = SecurityContext::from_headers(&headers); - assert_eq!(ctx.principal.id, "cust-123"); + assert_eq!(ctx.principal.id, "anonymous"); assert_eq!(ctx.principal.kind, PrincipalKind::Customer); assert!(ctx.principal.role.is_none()); } #[test] - fn test_context_from_headers_agent() { + fn correlation_header_does_not_restore_authority() { + let ctx = SecurityContext::from_headers(&[ + ("x-temper-correlation-id".to_string(), "trace-1".to_string()), + ("x-temper-principal-kind".to_string(), "admin".to_string()), + ]); + + assert_eq!(ctx.correlation_id, "trace-1"); + assert_eq!(ctx.principal.id, "anonymous"); + assert_eq!(ctx.principal.kind, PrincipalKind::Customer); + } + + #[test] + fn agent_role_and_delegation_headers_are_non_authoritative() { let headers = vec![ ("X-Temper-Principal-Id".to_string(), "agent-1".to_string()), ("X-Temper-Principal-Kind".to_string(), "agent".to_string()), @@ -296,9 +364,10 @@ mod tests { ]; let ctx = SecurityContext::from_headers(&headers); - assert_eq!(ctx.principal.kind, PrincipalKind::Agent); - assert_eq!(ctx.principal.role, Some("customer_agent".to_string())); - assert_eq!(ctx.principal.acting_for, Some("cust-456".to_string())); + assert_eq!(ctx.principal.id, "anonymous"); + assert_eq!(ctx.principal.kind, PrincipalKind::Customer); + assert!(ctx.principal.role.is_none()); + assert!(ctx.principal.acting_for.is_none()); } #[test] @@ -317,13 +386,84 @@ mod tests { ]; let ctx = SecurityContext::from_headers(&headers); - assert_eq!(ctx.principal.kind, PrincipalKind::Admin); - assert!(ctx.principal.attributes.contains_key("approvallimit")); - assert!(ctx.context_attrs.contains_key("ratelimitexceeded")); + assert_eq!(ctx.principal.kind, PrincipalKind::Customer); + assert!(ctx.principal.attributes.is_empty()); + assert!(ctx.context_attrs.is_empty()); + } + + #[test] + fn no_header_marker_can_mint_admin() { + let headers = vec![ + ("X-Temper-Principal-Id".to_string(), "attacker".to_string()), + ("X-Temper-Principal-Kind".to_string(), "admin".to_string()), + ( + "x-temper-internal-trusted-principal".to_string(), + "1".to_string(), + ), + ]; + let ctx = SecurityContext::from_headers(&headers); + assert_eq!(ctx.principal.kind, PrincipalKind::Customer); + assert_eq!(ctx.principal.id, "anonymous"); + } + + #[test] + fn caller_declared_correlation_never_becomes_a_cedar_input() { + // Cedar's context is built from `SecurityContext::context_attrs`. Session + // id and intent are caller-supplied headers, and policies condition on + // `context.sessionId` (session-scoped permits minted from an approved + // decision), so putting either in `context_attrs` would let any caller + // satisfy the scope that made such an approval narrow. + let authenticated = AuthenticatedRequestContext::new( + TenantId::default(), + SecurityContext::from_resolved_identity("agent-1", "operator", None), + ) + .with_session_id(Some("sess-approved".to_string())) + .with_intent(Some("delete everything".to_string())); + + assert_eq!(authenticated.session_id(), Some("sess-approved")); + assert_eq!(authenticated.intent(), Some("delete everything")); + + let attrs = &authenticated.security_context().context_attrs; + assert!( + attrs.get("sessionId").is_none(), + "a caller-supplied session id must not reach the Cedar context" + ); + assert!( + attrs.get("intent").is_none(), + "caller-supplied intent must not reach the Cedar context" + ); + } + + #[test] + fn system_principal_never_derivable_from_headers() { + let headers = vec![ + ("X-Temper-Principal-Id".to_string(), "svc".to_string()), + ("X-Temper-Principal-Kind".to_string(), "system".to_string()), + ]; + let ctx = SecurityContext::from_headers(&headers); + assert_eq!(ctx.principal.kind, PrincipalKind::Customer); + } + + #[test] + fn authenticated_context_binds_exact_tenant_and_security_context() { + let security_context = + SecurityContext::from_resolved_identity("agent-1", "operator", Some("session-1")); + let authenticated = + AuthenticatedRequestContext::new(TenantId::new("tenant-a"), security_context.clone()); + + assert_eq!(authenticated.tenant().as_str(), "tenant-a"); + assert_eq!( + authenticated.security_context().principal.id, + security_context.principal.id + ); + assert_eq!( + authenticated.security_context().context_attrs, + security_context.context_attrs + ); } #[test] - fn test_context_from_headers_normalizes_session_id() { + fn context_attribute_headers_are_non_authoritative() { let headers = vec![ ("X-Temper-Principal-Id".to_string(), "agent-1".to_string()), ("X-Temper-Principal-Kind".to_string(), "agent".to_string()), @@ -334,10 +474,7 @@ mod tests { ]; let ctx = SecurityContext::from_headers(&headers); - assert_eq!( - ctx.context_attrs.get("sessionId"), - Some(&serde_json::Value::String("session-1".to_string())) - ); + assert!(ctx.context_attrs.is_empty()); } #[test] @@ -369,7 +506,7 @@ mod tests { } #[test] - fn test_with_agent_context_preserves_explicit_principal() { + fn explicit_agent_context_promotes_header_anonymous_context() { let headers = vec![ ("X-Temper-Principal-Id".to_string(), "cust-123".to_string()), ( @@ -380,16 +517,23 @@ mod tests { let ctx = SecurityContext::from_headers(&headers).with_agent_context(Some("agent-1"), None, None); - // Should NOT overwrite explicit customer principal - assert_eq!(ctx.principal.id, "cust-123"); - assert_eq!(ctx.principal.kind, PrincipalKind::Customer); - // But agentId should be in context attrs + assert_eq!(ctx.principal.id, "agent-1"); + assert_eq!(ctx.principal.kind, PrincipalKind::Agent); assert_eq!( ctx.context_attrs.get("agentId"), Some(&serde_json::Value::String("agent-1".to_string())) ); } + #[test] + fn cedar_authority_context_keys_are_reserved() { + assert!(is_cedar_authority_context_key("sessionId")); + assert!(is_cedar_authority_context_key("agentTypeVerified")); + assert!(is_cedar_authority_context_key("role")); + assert!(!is_cedar_authority_context_key("status")); + assert!(!is_cedar_authority_context_key("Customer")); + } + #[test] fn system_principal_cannot_be_spoofed_via_headers() { let headers = vec![ @@ -399,7 +543,7 @@ mod tests { let ctx = SecurityContext::from_headers(&headers); // Must NOT be System — falls back to Customer. assert_eq!(ctx.principal.kind, PrincipalKind::Customer); - assert_eq!(ctx.principal.id, "attacker"); + assert_eq!(ctx.principal.id, "anonymous"); } #[test] @@ -418,7 +562,7 @@ mod tests { ("X-Temper-Agent-Type".to_string(), "claude-code".to_string()), ]; let ctx = SecurityContext::from_headers(&headers); - assert_eq!(ctx.principal.agent_type, Some("claude-code".to_string())); + assert!(ctx.principal.agent_type.is_none()); } #[test] @@ -435,25 +579,17 @@ mod tests { ), ]; let ctx = SecurityContext::from_headers(&headers); - assert_eq!( - ctx.principal.attributes.get("scopes"), - Some(&serde_json::json!(["repo:read", "repo:write", "force"])) - ); + assert!(!ctx.principal.attributes.contains_key("scopes")); } #[test] - fn action_context_is_principal_attribute() { + fn action_context_header_is_ignored_but_explicit_context_is_preserved() { let ctx = SecurityContext::from_headers(&[( "x-temper-action-context".to_string(), "composite:Apps.Fork".to_string(), )]); - assert_eq!( - ctx.principal.attributes.get("action_context"), - Some(&serde_json::Value::String( - "composite:Apps.Fork".to_string() - )) - ); + assert!(!ctx.principal.attributes.contains_key("action_context")); let ctx = SecurityContext::from_headers(&[]).with_action_context("composite:Repo.Write"); assert_eq!( diff --git a/crates/temper-authz/src/engine/mod.rs b/crates/temper-authz/src/engine/mod.rs index f2caf62cb..f2957b4e3 100644 --- a/crates/temper-authz/src/engine/mod.rs +++ b/crates/temper-authz/src/engine/mod.rs @@ -14,7 +14,7 @@ use cedar_policy::{ Request, Response as CedarResponse, }; -use crate::context::{PrincipalKind, SecurityContext}; +use crate::context::{PrincipalKind, SecurityContext, is_cedar_authority_context_key}; use crate::error::{AuthzDenial, AuthzError}; use crate::metrics::{CedarDecisionMetric, CedarPhaseOutcome}; @@ -88,6 +88,10 @@ pub struct AuthzEngine { /// Fallback global policy set for callers that don't specify a tenant. /// Deprecated: callers should migrate to `authorize_for_tenant`. fallback_policy_set: RwLock, + /// Immutable platform policy used when a tenant has no app policy set. + /// Only System principals are evaluated against this set; all externally + /// resolvable principal kinds still fail closed for an unloaded tenant. + platform_policy_set: CompiledPolicies, authorizer: Authorizer, } @@ -106,6 +110,7 @@ impl AuthzEngine { Ok(Self { tenant_policies: RwLock::new(BTreeMap::new()), fallback_policy_set: RwLock::new(CompiledPolicies::new(policy_set)), + platform_policy_set: compiled_system_platform_policies(), authorizer: Authorizer::new(), }) } @@ -123,6 +128,7 @@ impl AuthzEngine { Self { tenant_policies: RwLock::new(BTreeMap::new()), fallback_policy_set: RwLock::new(CompiledPolicies::new(policy_set)), + platform_policy_set: compiled_system_platform_policies(), authorizer: Authorizer::new(), } } @@ -138,10 +144,19 @@ impl AuthzEngine { Self { tenant_policies: RwLock::new(BTreeMap::new()), fallback_policy_set: RwLock::new(CompiledPolicies::new(policy_set)), + platform_policy_set: compiled_system_platform_policies(), authorizer: Authorizer::new(), } } + /// Parse and validate policy text without changing the active policy set. + pub fn validate_tenant_policies(&self, policy_text: &str) -> Result<(), AuthzError> { + policy_text + .parse::() + .map(|_| ()) + .map_err(|error| AuthzError::PolicyParse(error.to_string())) + } + /// Hot-reload Cedar policies for a specific tenant. Parses and validates /// the new policy text, then atomically swaps the tenant's policy set. /// If parsing fails, existing policies remain in effect. @@ -318,8 +333,12 @@ impl AuthzEngine { /// Evaluate an authorization request against a specific tenant's policy set. /// - /// If the tenant has no policies loaded, falls back to Cedar default-deny - /// (returns `NoMatchingPermit`). + /// If the tenant has no app policy set loaded, externally resolvable + /// principals fail closed. Kernel System authority is evaluated only + /// against the immutable built-in platform policy, never the process-wide + /// compatibility policy. This preserves internal bootstrap/recovery while + /// preventing a missing tenant policy from inheriting another scope + /// (ARN-230). pub fn authorize_for_tenant( &self, tenant: &str, @@ -338,18 +357,26 @@ impl AuthzEngine { }; if let Some(tp) = tenants.get(tenant) { - self.evaluate_request( + return self.evaluate_request( security_ctx, action, resource_type, resource_attrs, &tp.policies, - ) - } else { - // No per-tenant policies loaded — fall back to global. - drop(tenants); - self.authorize(security_ctx, action, resource_type, resource_attrs) + ); + } + + if Self::is_system(security_ctx) { + return self.evaluate_request( + security_ctx, + action, + resource_type, + resource_attrs, + &self.platform_policy_set, + ); } + + AuthzDecision::Deny(AuthzDenial::NoMatchingPermit) } /// Core Cedar evaluation logic shared by both `authorize` and @@ -430,9 +457,12 @@ impl AuthzEngine { insert_json_as_cedar(&mut ctx_map, key.clone(), value); } - // Inject resource attributes into context (enables Cedar policies to - // reference entity state and cross-entity context via `context.key`). + // Resource fields may appear as context.key for domain policies. + // Authority keys (sessionId, agentType, role, …) stay on SecurityContext. for (key, value) in resource_attrs { + if is_cedar_authority_context_key(key) || ctx_map.contains_key(key) { + continue; + } insert_json_as_cedar(&mut ctx_map, key.clone(), value); } crate::metrics::record_cedar_request_attribute_count("context", ctx_map.len()); @@ -706,6 +736,12 @@ fn merge_system_platform_policy(combined: &mut PolicySet) { } } +fn compiled_system_platform_policies() -> CompiledPolicies { + let mut policy_set = PolicySet::new(); + merge_system_platform_policy(&mut policy_set); + CompiledPolicies::new(policy_set) +} + /// Count user-authored policies in a [`PolicySet`], excluding the built-in /// `system-platform` policies (ADR-0046). Tenants should reason about their /// own policy surface without the platform's internals polluting the count. diff --git a/crates/temper-authz/src/engine/tests.rs b/crates/temper-authz/src/engine/tests.rs index 5d167451e..337cc94bf 100644 --- a/crates/temper-authz/src/engine/tests.rs +++ b/crates/temper-authz/src/engine/tests.rs @@ -6,20 +6,37 @@ const PM_ISSUE_POLICY: &str = include_str!("../../../../os-apps/project-management/specs/policies/issue.cedar"); fn admin_context() -> SecurityContext { - SecurityContext::from_headers(&[ - ("X-Temper-Principal-Id".to_string(), "admin-1".to_string()), - ("X-Temper-Principal-Kind".to_string(), "admin".to_string()), - ]) + SecurityContext { + principal: crate::context::Principal { + id: "admin-1".to_string(), + kind: crate::context::PrincipalKind::Admin, + role: None, + acting_for: None, + agent_type: None, + attributes: HashMap::new(), + }, + context_attrs: HashMap::new(), + correlation_id: "test-admin".to_string(), + } } fn customer_context(id: &str) -> SecurityContext { - SecurityContext::from_headers(&[ - ("X-Temper-Principal-Id".to_string(), id.to_string()), - ( - "X-Temper-Principal-Kind".to_string(), - "customer".to_string(), - ), - ]) + SecurityContext { + principal: crate::context::Principal { + id: id.to_string(), + kind: crate::context::PrincipalKind::Customer, + role: None, + acting_for: None, + agent_type: None, + attributes: HashMap::new(), + }, + context_attrs: HashMap::new(), + correlation_id: "test-customer".to_string(), + } +} + +fn agent_context(id: &str, agent_type: &str) -> SecurityContext { + SecurityContext::from_resolved_identity(id, agent_type, None) } #[test] @@ -63,13 +80,7 @@ fn system_authorized_via_system_platform_policy_not_bypass() { // Non-system principal against empty engine is denied (Cedar default-deny). // This would have been silently bypassed if we still used is_system. - let customer = SecurityContext::from_headers(&[ - ("X-Temper-Principal-Id".to_string(), "alice".to_string()), - ( - "X-Temper-Principal-Kind".to_string(), - "customer".to_string(), - ), - ]); + let customer = customer_context("alice"); let decision = engine.authorize(&customer, "AnyAction", "AnyResource", &attrs); assert!( !decision.is_allowed(), @@ -130,17 +141,11 @@ fn test_scoped_customer_principal_exposes_account_id() { "#; let engine = AuthzEngine::new(policy).unwrap(); - let ctx = SecurityContext::from_headers(&[ - ("X-Temper-Principal-Id".to_string(), "acct-1".to_string()), - ( - "X-Temper-Principal-Kind".to_string(), - "customer".to_string(), - ), - ( - "X-Temper-Principal-Scopes".to_string(), - "repo:read,repo:write".to_string(), - ), - ]); + let mut ctx = customer_context("acct-1"); + ctx.principal.attributes.insert( + "scopes".to_string(), + serde_json::json!(["repo:read", "repo:write"]), + ); let attrs = HashMap::from([ ( "Id".to_string(), @@ -281,22 +286,14 @@ fn test_agent_type_in_cedar_context() { ) .unwrap(); // With matching agentType -> Allow - let ctx = SecurityContext::from_headers(&[ - ("X-Temper-Principal-Id".to_string(), "bot-1".to_string()), - ("X-Temper-Principal-Kind".to_string(), "agent".to_string()), - ]) - .with_agent_context(Some("bot-1"), None, Some("claude-code")); + let ctx = agent_context("bot-1", "claude-code"); let mut attrs = HashMap::new(); attrs.insert("id".to_string(), serde_json::json!("doc-1")); let result = engine.authorize(&ctx, "read", "Doc", &attrs); assert!(result.is_allowed(), "should allow claude-code agent"); // Without matching agentType -> Deny - let ctx2 = SecurityContext::from_headers(&[ - ("X-Temper-Principal-Id".to_string(), "bot-2".to_string()), - ("X-Temper-Principal-Kind".to_string(), "agent".to_string()), - ]) - .with_agent_context(Some("bot-2"), None, Some("openclaw")); + let ctx2 = agent_context("bot-2", "openclaw"); let mut attrs2 = HashMap::new(); attrs2.insert("id".to_string(), serde_json::json!("doc-2")); let result2 = engine.authorize(&ctx2, "read", "Doc", &attrs2); @@ -310,10 +307,7 @@ fn test_exact_agent_principal_match() { let policy = r#"permit(principal == Agent::"bot-1", action == Action::"Assign", resource is Issue);"#; let engine = AuthzEngine::new(policy).unwrap(); - let ctx = SecurityContext::from_headers(&[ - ("X-Temper-Principal-Id".to_string(), "bot-1".to_string()), - ("X-Temper-Principal-Kind".to_string(), "agent".to_string()), - ]); + let ctx = agent_context("bot-1", "test-agent"); let mut attrs = HashMap::new(); attrs.insert("id".to_string(), serde_json::json!("issue-1")); let decision = engine.authorize(&ctx, "Assign", "Issue", &attrs); @@ -328,10 +322,7 @@ fn test_exact_principal_match_wrong_id_denied() { let policy = r#"permit(principal == Agent::"bot-1", action == Action::"Assign", resource is Issue);"#; let engine = AuthzEngine::new(policy).unwrap(); - let ctx = SecurityContext::from_headers(&[ - ("X-Temper-Principal-Id".to_string(), "bot-2".to_string()), - ("X-Temper-Principal-Kind".to_string(), "agent".to_string()), - ]); + let ctx = agent_context("bot-2", "test-agent"); let mut attrs = HashMap::new(); attrs.insert("id".to_string(), serde_json::json!("issue-1")); let decision = engine.authorize(&ctx, "Assign", "Issue", &attrs); @@ -356,11 +347,7 @@ fn test_principal_attribute_access_in_policy() { let engine = AuthzEngine::new(policy).unwrap(); // With matching agent_type → Allow - let ctx = SecurityContext::from_headers(&[ - ("X-Temper-Principal-Id".to_string(), "bot-1".to_string()), - ("X-Temper-Principal-Kind".to_string(), "agent".to_string()), - ("X-Temper-Agent-Type".to_string(), "supervisor".to_string()), - ]); + let ctx = agent_context("bot-1", "supervisor"); let mut attrs = HashMap::new(); attrs.insert("id".to_string(), serde_json::json!("issue-1")); let decision = engine.authorize(&ctx, "Triage", "Issue", &attrs); @@ -370,11 +357,7 @@ fn test_principal_attribute_access_in_policy() { ); // Without matching agent_type → Deny - let ctx2 = SecurityContext::from_headers(&[ - ("X-Temper-Principal-Id".to_string(), "bot-2".to_string()), - ("X-Temper-Principal-Kind".to_string(), "agent".to_string()), - ("X-Temper-Agent-Type".to_string(), "worker".to_string()), - ]); + let ctx2 = agent_context("bot-2", "worker"); let decision2 = engine.authorize(&ctx2, "Triage", "Issue", &attrs); assert!( !decision2.is_allowed(), @@ -399,14 +382,7 @@ fn test_resource_attribute_access_in_policy() { "#; let engine = AuthzEngine::new(policy).unwrap(); - let worker_ctx = SecurityContext::from_headers(&[ - ( - "X-Temper-Principal-Id".to_string(), - "local-codex-worker".to_string(), - ), - ("X-Temper-Principal-Kind".to_string(), "agent".to_string()), - ("X-Temper-Agent-Type".to_string(), "worker".to_string()), - ]); + let worker_ctx = agent_context("local-codex-worker", "worker"); let mut attrs = HashMap::new(); attrs.insert("id".to_string(), serde_json::json!("wr-1")); attrs.insert( @@ -420,14 +396,7 @@ fn test_resource_attribute_access_in_policy() { "claimed worker should be allowed through resource.worker_id: {decision:?}" ); - let other_ctx = SecurityContext::from_headers(&[ - ( - "X-Temper-Principal-Id".to_string(), - "other-worker".to_string(), - ), - ("X-Temper-Principal-Kind".to_string(), "agent".to_string()), - ("X-Temper-Agent-Type".to_string(), "worker".to_string()), - ]); + let other_ctx = agent_context("other-worker", "worker"); let decision = engine.authorize(&other_ctx, "StartLocal", "WorkerRun", &attrs); assert!( !decision.is_allowed(), @@ -451,28 +420,14 @@ fn test_principal_agent_type_set_membership_filtering() { let mut attrs = HashMap::new(); attrs.insert("id".to_string(), serde_json::json!("issue-1")); - let supervisor_ctx = SecurityContext::from_headers(&[ - ( - "X-Temper-Principal-Id".to_string(), - "bot-supervisor".to_string(), - ), - ("X-Temper-Principal-Kind".to_string(), "agent".to_string()), - ("X-Temper-Agent-Type".to_string(), "supervisor".to_string()), - ]); + let supervisor_ctx = agent_context("bot-supervisor", "supervisor"); let supervisor_decision = engine.authorize(&supervisor_ctx, "Assign", "Issue", &attrs); assert!( supervisor_decision.is_allowed(), "set membership should allow supervisor agent_type: {supervisor_decision:?}" ); - let worker_ctx = SecurityContext::from_headers(&[ - ( - "X-Temper-Principal-Id".to_string(), - "bot-worker".to_string(), - ), - ("X-Temper-Principal-Kind".to_string(), "agent".to_string()), - ("X-Temper-Agent-Type".to_string(), "worker".to_string()), - ]); + let worker_ctx = agent_context("bot-worker", "worker"); let worker_decision = engine.authorize(&worker_ctx, "Assign", "Issue", &attrs); assert!( !worker_decision.is_allowed(), @@ -495,10 +450,7 @@ fn test_context_entity_status_in_cedar_context() { let engine = AuthzEngine::new(policy).unwrap(); - let ctx = SecurityContext::from_headers(&[ - ("x-temper-principal-id".to_string(), "agent-1".to_string()), - ("x-temper-principal-kind".to_string(), "agent".to_string()), - ]); + let ctx = agent_context("agent-1", "test-agent"); // Without context entity status: should deny let mut attrs = HashMap::new(); @@ -613,6 +565,91 @@ fn test_per_tenant_isolation() { ); } +#[test] +fn missing_tenant_never_inherits_permissive_fallback() { + let engine = AuthzEngine::permissive(); + let ctx = customer_context("anonymous"); + let attrs = HashMap::from([("id".to_string(), serde_json::json!("doc-1"))]); + + assert!( + engine.authorize(&ctx, "read", "Doc", &attrs).is_allowed(), + "negative control: the explicit global compatibility policy is permissive" + ); + let decision = engine.authorize_for_tenant("unloaded", &ctx, "read", "Doc", &attrs); + assert_eq!( + decision, + AuthzDecision::Deny(AuthzDenial::NoMatchingPermit), + "tenant-scoped authorization must fail closed when its policy is absent" + ); +} + +#[test] +fn missing_tenant_system_uses_only_the_builtin_platform_policy() { + let engine = AuthzEngine::permissive(); + let attrs = HashMap::from([("id".to_string(), serde_json::json!("doc-1"))]); + + assert!( + engine + .authorize_for_tenant( + "unloaded", + &SecurityContext::system(), + "bootstrap", + "Doc", + &attrs, + ) + .is_allowed(), + "kernel System work must remain authorized by the explicit built-in policy" + ); + assert_eq!( + engine.authorize_for_tenant( + "unloaded", + &customer_context("anonymous"), + "read", + "Doc", + &attrs, + ), + AuthzDecision::Deny(AuthzDenial::NoMatchingPermit), + "an unloaded tenant must not inherit the permissive compatibility policy" + ); +} + +#[test] +fn failed_tenant_reload_preserves_last_known_good_without_global_fallback() { + let engine = AuthzEngine::permissive(); + engine + .reload_tenant_policies( + "tenant-a", + r#"permit(principal is Customer, action == Action::"read", resource is Doc);"#, + ) + .unwrap(); + assert!( + engine + .reload_tenant_policies("tenant-a", "not cedar") + .is_err() + ); + + let ctx = customer_context("customer-1"); + let attrs = HashMap::from([("id".to_string(), serde_json::json!("doc-1"))]); + assert!( + engine + .authorize_for_tenant("tenant-a", &ctx, "read", "Doc", &attrs) + .is_allowed(), + "a failed reload must preserve the exact tenant's last-known-good policy" + ); + assert!( + !engine + .authorize_for_tenant("tenant-a", &ctx, "write", "Doc", &attrs) + .is_allowed(), + "a failed reload must not widen to the permissive global policy" + ); + assert!( + !engine + .authorize_for_tenant("tenant-b", &ctx, "read", "Doc", &attrs) + .is_allowed(), + "another unloaded tenant must remain denied" + ); +} + #[test] fn tenant_policy_reloads_keep_system_platform_policy() { let engine = AuthzEngine::empty(); @@ -760,3 +797,124 @@ fn candidate_filter_preserves_named_forbid_policy_ids() { "candidate filtering must preserve named policy diagnostics, got: {policy_ids:?}" ); } + +/// Resource attributes must not satisfy a session-scoped permit. The only +/// legitimate `context.sessionId` is the grant-checked value on SecurityContext. +#[test] +fn resource_session_id_cannot_satisfy_session_scoped_permit() { + let policy = r#" + permit( + principal is Agent, + action == Action::"create", + resource is Order + ) when { context.sessionId == "approved-session" }; + "#; + let engine = AuthzEngine::new(policy).unwrap(); + let ctx = SecurityContext::from_resolved_identity("agent-1", "operator", None); + let mut attrs = HashMap::new(); + attrs.insert("id".to_string(), serde_json::json!("order-1")); + attrs.insert( + "sessionId".to_string(), + serde_json::json!("approved-session"), + ); + + let decision = engine.authorize(&ctx, "create", "Order", &attrs); + assert!( + !decision.is_allowed(), + "caller-supplied resource.sessionId must not satisfy context.sessionId, got: {decision:?}" + ); +} + +/// The grant-checked session id on SecurityContext still authorizes. +#[test] +fn grant_checked_session_id_satisfies_session_scoped_permit() { + let policy = r#" + permit( + principal is Agent, + action == Action::"create", + resource is Order + ) when { context.sessionId == "approved-session" }; + "#; + let engine = AuthzEngine::new(policy).unwrap(); + let ctx = + SecurityContext::from_resolved_identity("agent-1", "operator", Some("approved-session")); + let mut attrs = HashMap::new(); + attrs.insert("id".to_string(), serde_json::json!("order-1")); + + let decision = engine.authorize(&ctx, "create", "Order", &attrs); + assert!( + decision.is_allowed(), + "grant-checked context.sessionId must still match, got: {decision:?}" + ); +} + +/// A resource body must not overwrite a different grant-checked session id. +#[test] +fn resource_session_id_cannot_overwrite_grant_checked_session_id() { + let policy = r#" + permit( + principal is Agent, + action == Action::"create", + resource is Order + ) when { context.sessionId == "forged-session" }; + "#; + let engine = AuthzEngine::new(policy).unwrap(); + let ctx = SecurityContext::from_resolved_identity("agent-1", "operator", Some("real-session")); + let mut attrs = HashMap::new(); + attrs.insert("id".to_string(), serde_json::json!("order-1")); + attrs.insert("sessionId".to_string(), serde_json::json!("forged-session")); + + let decision = engine.authorize(&ctx, "create", "Order", &attrs); + assert!( + !decision.is_allowed(), + "resource.sessionId must not overwrite the grant-checked session, got: {decision:?}" + ); +} + +/// Resource attrs must not mint `agentType` / `agentTypeVerified` that +/// `from_resolved_identity` is supposed to be the only source of. +#[test] +fn resource_attrs_cannot_forge_agent_type_verified() { + let policy = r#" + permit( + principal is Agent, + action == Action::"create", + resource is Order + ) when { context.agentType == "supervisor" && context.agentTypeVerified == true }; + "#; + let engine = AuthzEngine::new(policy).unwrap(); + let ctx = SecurityContext::from_resolved_identity("agent-1", "operator", None); + let mut attrs = HashMap::new(); + attrs.insert("id".to_string(), serde_json::json!("order-1")); + attrs.insert("agentType".to_string(), serde_json::json!("supervisor")); + attrs.insert("agentTypeVerified".to_string(), serde_json::json!(true)); + + let decision = engine.authorize(&ctx, "create", "Order", &attrs); + assert!( + !decision.is_allowed(), + "resource agentType/agentTypeVerified must not overwrite Cedar context, got: {decision:?}" + ); +} + +/// Resource `role` must not become `context.role`. +#[test] +fn resource_role_cannot_satisfy_role_scoped_permit() { + let policy = r#" + permit( + principal is Agent, + action == Action::"create", + resource is Order + ) when { context.role == "admin" }; + "#; + let engine = AuthzEngine::new(policy).unwrap(); + let ctx = SecurityContext::from_resolved_identity("agent-1", "operator", None); + let mut attrs = HashMap::new(); + attrs.insert("id".to_string(), serde_json::json!("order-1")); + attrs.insert("role".to_string(), serde_json::json!("admin")); + + let decision = engine.authorize(&ctx, "create", "Order", &attrs); + assert!( + !decision.is_allowed(), + "resource.role must not satisfy context.role, got: {decision:?}" + ); +} diff --git a/crates/temper-authz/src/lib.rs b/crates/temper-authz/src/lib.rs index 3fc2fb33f..dbb07dd1d 100644 --- a/crates/temper-authz/src/lib.rs +++ b/crates/temper-authz/src/lib.rs @@ -10,7 +10,10 @@ mod error; mod metrics; mod policy_gen; -pub use context::{Principal, PrincipalKind, SecurityContext}; +pub use context::{ + AuthenticatedRequestContext, Principal, PrincipalKind, SecurityContext, + is_cedar_authority_context_key, +}; pub use engine::{AuthzDecision, AuthzEngine}; pub use error::{AuthzDenial, AuthzError}; pub use metrics::init_metrics; diff --git a/crates/temper-cli/src/decide/mod.rs b/crates/temper-cli/src/decide/mod.rs index c527b2954..65c8ad23d 100644 --- a/crates/temper-cli/src/decide/mod.rs +++ b/crates/temper-cli/src/decide/mod.rs @@ -7,6 +7,7 @@ use std::io::{self, Write}; use anyhow::{Context, Result}; +use reqwest::header::{AUTHORIZATION, HeaderMap, HeaderValue}; use temper_authz::{ActionScope, DurationScope, PolicyScopeMatrix, PrincipalScope, ResourceScope}; /// Run the `temper decide` interactive loop. @@ -14,7 +15,14 @@ use temper_authz::{ActionScope, DurationScope, PolicyScopeMatrix, PrincipalScope /// Polls the server for pending decisions and prompts the human to /// approve/deny each one with scope selection. pub async fn run(port: u16, tenant: &str) -> Result<()> { - let client = reqwest::Client::new(); + let api_key = std::env::var("TEMPER_API_KEY") + .context("TEMPER_API_KEY is required to use `temper decide`")?; + let mut default_headers = HeaderMap::new(); + default_headers.insert(AUTHORIZATION, authorization_header(&api_key)?); + let client = reqwest::Client::builder() + .default_headers(default_headers) + .build() + .context("Failed to configure the authenticated Temper client")?; let base_url = format!("http://127.0.0.1:{port}"); println!("Temper Decide — Governance Terminal"); @@ -101,6 +109,11 @@ pub async fn run(port: u16, tenant: &str) -> Result<()> { } } +fn authorization_header(api_key: &str) -> Result { + HeaderValue::from_str(&format!("Bearer {api_key}")) + .context("TEMPER_API_KEY is not valid as an HTTP bearer credential") +} + async fn fetch_pending_decisions( client: &reqwest::Client, base_url: &str, @@ -183,3 +196,21 @@ async fn deny_decision( Ok(()) } + +#[cfg(test)] +mod tests { + use super::authorization_header; + + #[test] + fn authorization_header_uses_bearer_scheme() { + assert_eq!( + authorization_header("tenant-key").unwrap(), + "Bearer tenant-key" + ); + } + + #[test] + fn authorization_header_rejects_header_injection() { + assert!(authorization_header("key\r\nx-forged: true").is_err()); + } +} diff --git a/crates/temper-cli/src/main.rs b/crates/temper-cli/src/main.rs index 4c052480d..48117877f 100644 --- a/crates/temper-cli/src/main.rs +++ b/crates/temper-cli/src/main.rs @@ -100,7 +100,7 @@ enum Commands { #[arg(long, default_value = "pinned")] follow_policy: String, }, - /// Approve or deny pending governance decisions from the terminal + /// Approve or deny pending governance decisions using `TEMPER_API_KEY` Decide { /// Port where Temper HTTP server is running #[arg(short, long, default_value = "3000")] diff --git a/crates/temper-cli/src/migrate_turso_to_postgres.rs b/crates/temper-cli/src/migrate_turso_to_postgres.rs index cf616944d..aad1ae81d 100644 --- a/crates/temper-cli/src/migrate_turso_to_postgres.rs +++ b/crates/temper-cli/src/migrate_turso_to_postgres.rs @@ -388,6 +388,8 @@ async fn migrate_tenant_platform_tables( migrate_secrets(source, pool, tenant, dry_run, builder).await?; migrate_policy_denial_patterns(source, pool, tenant, dry_run, builder).await?; migrate_query_projections(source, target, tenant, dry_run, builder).await?; + migrate_feature_requests(source, pool, tenant, dry_run, builder).await?; + migrate_evolution_records(source, pool, tenant, dry_run, builder).await?; Ok(()) } @@ -397,8 +399,6 @@ async fn migrate_global_platform_tables( dry_run: bool, builder: &mut ManifestBuilder, ) -> Result<()> { - migrate_feature_requests(source, pool, dry_run, builder).await?; - migrate_evolution_records(source, pool, dry_run, builder).await?; migrate_blobs(source, pool, dry_run, builder).await?; Ok(()) } @@ -677,12 +677,8 @@ async fn migrate_trajectories( dry_run: bool, builder: &mut ManifestBuilder, ) -> Result<()> { - let all_rows = source.load_recent_trajectories(MAX_ROWS).await?; - ensure_row_bound("trajectories", all_rows.len())?; - let rows = all_rows - .into_iter() - .filter(|row| row.tenant == tenant) - .collect::>(); + let rows = source.load_recent_trajectories(tenant, MAX_ROWS).await?; + ensure_row_bound("trajectories", rows.len())?; builder.record_source( tenant, "trajectories", @@ -707,8 +703,8 @@ async fn migrate_trajectories( sqlx::query( "INSERT INTO trajectories \ (tenant, entity_type, entity_id, action, success, from_status, to_status, error, agent_id, session_id, \ - authz_denied, denied_resource, denied_module, source, spec_governed, created_at, request_body, intent, matched_policy_ids) \ - SELECT $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19 \ + authz_denied, denied_resource, denied_module, source, spec_governed, created_at, request_body, intent, matched_policy_ids, capture_seq) \ + SELECT $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20 \ WHERE NOT EXISTS ( \ SELECT 1 FROM trajectories \ WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 AND action = $4 \ @@ -735,6 +731,10 @@ async fn migrate_trajectories( .bind(request_body) .bind(row.intent) .bind(matched_policy_ids) + // Carried, not regenerated: capture order is what the session read + // replays, and dropping it would leave migrated rows ordered by the + // order the migration happened to insert them. + .bind(row.capture_seq) .execute(pool) .await?; } @@ -966,8 +966,9 @@ async fn migrate_ots_trajectories( let mut rows_with_data = Vec::new(); for row in rows { let data = source - .get_ots_trajectory(&row.trajectory_id) + .get_ots_trajectory(&row.tenant, &row.trajectory_id) .await? + .map(|document| document.data) .unwrap_or_else(|| "{}".to_string()); values.push(json!({ "trajectory_id": row.trajectory_id, @@ -1154,12 +1155,13 @@ async fn migrate_query_projections( async fn migrate_feature_requests( source: &TursoEventStore, pool: &PgPool, + tenant: &str, dry_run: bool, builder: &mut ManifestBuilder, ) -> Result<()> { - let rows = source.list_feature_requests(None).await?; + let rows = source.list_feature_requests(tenant, None).await?; builder.record_source( - GLOBAL_TENANT, + tenant, "feature_requests", rows.iter() .map(feature_request_value) @@ -1169,15 +1171,17 @@ async fn migrate_feature_requests( return Ok(()); } for row in rows { - sqlx::query( + let result = sqlx::query( "INSERT INTO feature_requests \ (id, tenant, category, description, frequency, trajectory_refs, disposition, developer_notes, created_at, updated_at) \ - VALUES ($1, 'default', $2, $3, $4, $5, $6, $7, $8, $9) \ + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) \ ON CONFLICT (id) DO UPDATE SET category = EXCLUDED.category, description = EXCLUDED.description, \ frequency = EXCLUDED.frequency, trajectory_refs = EXCLUDED.trajectory_refs, disposition = EXCLUDED.disposition, \ - developer_notes = EXCLUDED.developer_notes, created_at = EXCLUDED.created_at, updated_at = EXCLUDED.updated_at", + developer_notes = EXCLUDED.developer_notes, created_at = EXCLUDED.created_at, updated_at = EXCLUDED.updated_at \ + WHERE feature_requests.tenant = EXCLUDED.tenant", ) .bind(row.id) + .bind(row.tenant) .bind(row.category) .bind(row.description) .bind(row.frequency) @@ -1188,6 +1192,11 @@ async fn migrate_feature_requests( .bind(parse_source_timestamp(&row.updated_at)?) .execute(pool) .await?; + if result.rows_affected() != 1 { + return Err(anyhow!( + "feature request ID collision across tenants during migration" + )); + } } Ok(()) } @@ -1195,18 +1204,19 @@ async fn migrate_feature_requests( async fn migrate_evolution_records( source: &TursoEventStore, pool: &PgPool, + tenant: &str, dry_run: bool, builder: &mut ManifestBuilder, ) -> Result<()> { - let rows = source.list_evolution_records(None, None).await?; + let rows = source.list_evolution_records(tenant, None, None).await?; builder.record_source( - GLOBAL_TENANT, + tenant, "evolution_records", rows.iter() .map(|row| { json!({ "id": row.id, - "tenant": "default", + "tenant": row.tenant, "record_type": row.record_type, "status": row.status, "created_by": row.created_by, @@ -1221,14 +1231,16 @@ async fn migrate_evolution_records( return Ok(()); } for row in rows { - sqlx::query( + let result = sqlx::query( "INSERT INTO evolution_records (id, tenant, record_type, status, created_by, derived_from, payload, timestamp) \ - VALUES ($1, 'default', $2, $3, $4, $5, $6, $7) \ + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) \ ON CONFLICT (id) DO UPDATE SET tenant = EXCLUDED.tenant, record_type = EXCLUDED.record_type, \ status = EXCLUDED.status, created_by = EXCLUDED.created_by, derived_from = EXCLUDED.derived_from, \ - payload = EXCLUDED.payload, timestamp = EXCLUDED.timestamp", + payload = EXCLUDED.payload, timestamp = EXCLUDED.timestamp \ + WHERE evolution_records.tenant = EXCLUDED.tenant", ) .bind(row.id) + .bind(row.tenant) .bind(row.record_type) .bind(row.status) .bind(row.created_by) @@ -1237,6 +1249,11 @@ async fn migrate_evolution_records( .bind(parse_source_timestamp(&row.timestamp)?) .execute(pool) .await?; + if result.rows_affected() != 1 { + return Err(anyhow!( + "evolution record ID collision across tenants during migration" + )); + } } Ok(()) } @@ -1477,7 +1494,7 @@ async fn target_installed_apps(pool: &PgPool, tenant: &str) -> Result async fn target_trajectories(pool: &PgPool, tenant: &str) -> Result> { let rows = sqlx::query( "SELECT tenant, entity_type, entity_id, action, success, from_status, to_status, error, agent_id, session_id, \ - authz_denied, denied_resource, denied_module, source, spec_governed, created_at, request_body, intent, matched_policy_ids \ + authz_denied, denied_resource, denied_module, source, spec_governed, created_at, request_body, intent, matched_policy_ids, capture_seq \ FROM trajectories WHERE tenant = $1 ORDER BY created_at, entity_type, entity_id, action", ) .bind(tenant) @@ -1846,6 +1863,7 @@ fn trajectory_value(row: &temper_store_turso::TursoTrajectoryRow) -> Result Value { "request_body": row.get::, _>("request_body"), "intent": row.get::, _>("intent"), "matched_policy_ids": row.get::, _>("matched_policy_ids"), + "capture_seq": row.get::, _>("capture_seq"), }) } diff --git a/crates/temper-cli/src/serve/bootstrap.rs b/crates/temper-cli/src/serve/bootstrap.rs index 2e206e453..a093ea7ea 100644 --- a/crates/temper-cli/src/serve/bootstrap.rs +++ b/crates/temper-cli/src/serve/bootstrap.rs @@ -480,9 +480,8 @@ pub(super) async fn bootstrap_tenants(state: &PlatformState, apps: &[(String, St } } - // Auto-register operator credential for the global API key (ADR-0033). - // This ensures the bearer auth middleware resolves the global key as a - // verified "operator" identity instead of falling through as anonymous. + // Register the bootstrap key as a normal operator credential in the default + // tenant. It grants no implicit authority in other tenants (ADR-0157). if let Some(ref api_key) = state.api_token { temper_platform::bootstrap_operator_credential(state, api_key, "default").await; } diff --git a/crates/temper-cli/src/serve/loader.rs b/crates/temper-cli/src/serve/loader.rs index c554422a3..f5b573e7c 100644 --- a/crates/temper-cli/src/serve/loader.rs +++ b/crates/temper-cli/src/serve/loader.rs @@ -436,13 +436,10 @@ effect = "set phantom true" assert!(combined.contains("resource is Issue")); let engine = temper_authz::AuthzEngine::new(&combined).expect("policy parses"); - let customer_ctx = temper_authz::SecurityContext::from_headers(&[ - ("x-temper-principal-id".to_string(), "cust-1".to_string()), - ( - "x-temper-principal-kind".to_string(), - "customer".to_string(), - ), - ]); + // A Customer principal. The principal headers this used to pass are + // stripped at the edge (ADR-0157) and no longer influence the context, + // so the anonymous Customer is what that construction actually produced. + let customer_ctx = temper_authz::SecurityContext::anonymous(); let issue = engine.authorize( &customer_ctx, diff --git a/crates/temper-evolution/src/lib.rs b/crates/temper-evolution/src/lib.rs index b02da12c2..174dae747 100644 --- a/crates/temper-evolution/src/lib.rs +++ b/crates/temper-evolution/src/lib.rs @@ -20,7 +20,9 @@ pub mod store; // Re-export primary types at crate root. pub use chain::{ChainValidation, validate_chain}; pub use insight::{classify_insight, compute_priority_score, generate_digest}; -pub use pg_store::{GenericEvolutionRow, PgRecordStoreError, PostgresRecordStore}; +pub use pg_store::{ + GenericEvolutionRecordInsert, GenericEvolutionRow, PgRecordStoreError, PostgresRecordStore, +}; pub use records::{ AnalysisRecord, Complexity, Decision, DecisionRecord, FeatureRequestDisposition, FeatureRequestRecord, InsightCategory, InsightRecord, InsightSignal, ObservationClass, diff --git a/crates/temper-evolution/src/pg_store.rs b/crates/temper-evolution/src/pg_store.rs index f610514fe..21bf6f3f3 100644 --- a/crates/temper-evolution/src/pg_store.rs +++ b/crates/temper-evolution/src/pg_store.rs @@ -25,6 +25,7 @@ pub enum PgRecordStoreError { #[allow(dead_code)] struct EvolutionRow { id: String, + tenant: String, record_type: String, status: String, created_by: String, @@ -41,6 +42,8 @@ struct EvolutionRow { pub struct GenericEvolutionRow { /// Record ID. pub id: String, + /// Owning tenant. + pub tenant: String, /// Record type: Observation, Problem, Analysis, Decision, Insight. pub record_type: String, /// Status: Open, Resolved, Superseded, Rejected. @@ -55,10 +58,30 @@ pub struct GenericEvolutionRow { pub timestamp: String, } +/// Backend-neutral values for one generic evolution-record insert. +#[derive(Clone, Copy, Debug)] +pub struct GenericEvolutionRecordInsert<'a> { + /// Tenant that owns the record. + pub tenant: &'a str, + /// Stable evolution record identifier. + pub id: &'a str, + /// Evolution record kind. + pub record_type: &'a str, + /// Current record status. + pub status: &'a str, + /// Principal that created the record. + pub created_by: &'a str, + /// Optional predecessor record identifier. + pub derived_from: Option<&'a str>, + /// Serialized record payload. + pub data_json: &'a str, +} + impl From for GenericEvolutionRow { fn from(row: EvolutionRow) -> Self { Self { id: row.id, + tenant: row.tenant, record_type: row.record_type, status: row.status, created_by: row.created_by, @@ -90,6 +113,7 @@ impl PostgresRecordStore { r#" CREATE TABLE IF NOT EXISTS evolution_records ( id TEXT PRIMARY KEY, + tenant TEXT NOT NULL DEFAULT 'default', record_type TEXT NOT NULL, status TEXT NOT NULL, created_by TEXT NOT NULL, @@ -102,6 +126,12 @@ impl PostgresRecordStore { .execute(&self.pool) .await?; + sqlx::query( + "ALTER TABLE evolution_records ADD COLUMN IF NOT EXISTS tenant TEXT NOT NULL DEFAULT 'default'", + ) + .execute(&self.pool) + .await?; + sqlx::query( r#" CREATE INDEX IF NOT EXISTS idx_evolution_records_type_status @@ -202,7 +232,7 @@ impl PostgresRecordStore { /// Get all open observations (status = 'Open'). pub async fn open_observations(&self) -> Result, PgRecordStoreError> { let rows: Vec = sqlx::query_as( - "SELECT id, record_type, status, created_by, derived_from, timestamp, payload \ + "SELECT id, tenant, record_type, status, created_by, derived_from, timestamp, payload \ FROM evolution_records WHERE record_type = 'Observation' AND status = 'Open' \ ORDER BY timestamp DESC", ) @@ -220,7 +250,7 @@ impl PostgresRecordStore { /// Get all open insights sorted by priority score (highest first). pub async fn ranked_insights(&self) -> Result, PgRecordStoreError> { let rows: Vec = sqlx::query_as( - "SELECT id, record_type, status, created_by, derived_from, timestamp, payload \ + "SELECT id, tenant, record_type, status, created_by, derived_from, timestamp, payload \ FROM evolution_records WHERE record_type = 'Insight' AND status = 'Open' \ ORDER BY (payload->>'priority_score')::float8 DESC", ) @@ -301,38 +331,43 @@ impl PostgresRecordStore { /// data_json, timestamp_rfc3339)` tuples for backend-neutral consumption. pub async fn list_records_generic( &self, + tenant: &str, record_type: Option<&str>, status: Option<&str>, ) -> Result, PgRecordStoreError> { let rows: Vec = match (record_type, status) { (Some(rt), Some(st)) => sqlx::query_as( - "SELECT id, record_type, status, created_by, derived_from, timestamp, payload \ - FROM evolution_records WHERE record_type = $1 AND status = $2 \ + "SELECT id, tenant, record_type, status, created_by, derived_from, timestamp, payload \ + FROM evolution_records WHERE tenant = $1 AND record_type = $2 AND status = $3 \ ORDER BY timestamp DESC", ) + .bind(tenant) .bind(rt) .bind(st) .fetch_all(&self.pool) .await?, (Some(rt), None) => sqlx::query_as( - "SELECT id, record_type, status, created_by, derived_from, timestamp, payload \ - FROM evolution_records WHERE record_type = $1 ORDER BY timestamp DESC", + "SELECT id, tenant, record_type, status, created_by, derived_from, timestamp, payload \ + FROM evolution_records WHERE tenant = $1 AND record_type = $2 ORDER BY timestamp DESC", ) + .bind(tenant) .bind(rt) .fetch_all(&self.pool) .await?, (None, Some(st)) => sqlx::query_as( - "SELECT id, record_type, status, created_by, derived_from, timestamp, payload \ - FROM evolution_records WHERE status = $1 ORDER BY timestamp DESC", + "SELECT id, tenant, record_type, status, created_by, derived_from, timestamp, payload \ + FROM evolution_records WHERE tenant = $1 AND status = $2 ORDER BY timestamp DESC", ) + .bind(tenant) .bind(st) .fetch_all(&self.pool) .await?, (None, None) => sqlx::query_as( - "SELECT id, record_type, status, created_by, derived_from, timestamp, payload \ - FROM evolution_records ORDER BY timestamp DESC", + "SELECT id, tenant, record_type, status, created_by, derived_from, timestamp, payload \ + FROM evolution_records WHERE tenant = $1 ORDER BY timestamp DESC", ) + .bind(tenant) .fetch_all(&self.pool) .await?, }; @@ -343,12 +378,14 @@ impl PostgresRecordStore { /// Get a single evolution record by ID. pub async fn get_record_generic( &self, + tenant: &str, id: &str, ) -> Result, PgRecordStoreError> { let row: Option = sqlx::query_as( - "SELECT id, record_type, status, created_by, derived_from, timestamp, payload \ - FROM evolution_records WHERE id = $1", + "SELECT id, tenant, record_type, status, created_by, derived_from, timestamp, payload \ + FROM evolution_records WHERE tenant = $1 AND id = $2", ) + .bind(tenant) .bind(id) .fetch_optional(&self.pool) .await?; @@ -359,12 +396,14 @@ impl PostgresRecordStore { /// List ranked insights as generic rows, sorted by priority_score descending. pub async fn list_ranked_insights_generic( &self, + tenant: &str, ) -> Result, PgRecordStoreError> { let rows: Vec = sqlx::query_as( - "SELECT id, record_type, status, created_by, derived_from, timestamp, payload \ - FROM evolution_records WHERE record_type = 'Insight' AND status = 'Open' \ + "SELECT id, tenant, record_type, status, created_by, derived_from, timestamp, payload \ + FROM evolution_records WHERE tenant = $1 AND record_type = 'Insight' AND status = 'Open' \ ORDER BY (payload->>'priority_score')::float8 DESC", ) + .bind(tenant) .fetch_all(&self.pool) .await?; @@ -374,22 +413,27 @@ impl PostgresRecordStore { /// Insert a generic evolution record (for backend-neutral writes). pub async fn insert_record_generic( &self, - id: &str, - record_type: &str, - status: &str, - created_by: &str, - derived_from: Option<&str>, - data_json: &str, + record: GenericEvolutionRecordInsert<'_>, ) -> Result<(), PgRecordStoreError> { + let GenericEvolutionRecordInsert { + tenant, + id, + record_type, + status, + created_by, + derived_from, + data_json, + } = record; let payload: serde_json::Value = serde_json::from_str(data_json) .map_err(|e| PgRecordStoreError::Serialization(e.to_string()))?; let now = chrono::Utc::now(); // determinism-ok: pg_store is I/O-bound, not sim-visible sqlx::query( "INSERT INTO evolution_records \ - (id, record_type, status, created_by, derived_from, timestamp, payload) \ - VALUES ($1, $2, $3, $4, $5, $6, $7)", + (id, tenant, record_type, status, created_by, derived_from, timestamp, payload) \ + VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", ) .bind(id) + .bind(tenant) .bind(record_type) .bind(status) .bind(created_by) @@ -436,7 +480,7 @@ impl PostgresRecordStore { expected_type: &str, ) -> Result, PgRecordStoreError> { let row: Option = sqlx::query_as( - "SELECT id, record_type, status, created_by, derived_from, timestamp, payload \ + "SELECT id, tenant, record_type, status, created_by, derived_from, timestamp, payload \ FROM evolution_records WHERE id = $1 AND record_type = $2", ) .bind(id) diff --git a/crates/temper-mcp/Cargo.toml b/crates/temper-mcp/Cargo.toml index 6ae8204e9..05c1d35b0 100644 --- a/crates/temper-mcp/Cargo.toml +++ b/crates/temper-mcp/Cargo.toml @@ -24,6 +24,7 @@ monty = { git = "https://github.com/pydantic/monty.git", package = "monty", rev [dev-dependencies] axum = { workspace = true } +temper-platform = { workspace = true } temper-server = { workspace = true, features = ["observe"] } temper-runtime = { workspace = true } temper-spec = { workspace = true } diff --git a/crates/temper-mcp/src/lib.rs b/crates/temper-mcp/src/lib.rs index cdba1a3ef..992a8c908 100644 --- a/crates/temper-mcp/src/lib.rs +++ b/crates/temper-mcp/src/lib.rs @@ -2,6 +2,7 @@ mod protocol; mod runtime; +mod trajectory_bounds; pub mod repl; pub use runtime::run_stdio_server; diff --git a/crates/temper-mcp/src/lib_tests.rs b/crates/temper-mcp/src/lib_tests.rs index 0d0e0e56b..547defec0 100644 --- a/crates/temper-mcp/src/lib_tests.rs +++ b/crates/temper-mcp/src/lib_tests.rs @@ -5,13 +5,65 @@ use std::collections::BTreeMap; use axum::Router; use serde_json::{Value, json}; -use temper_runtime::ActorSystem; -use temper_server::{ServerState, StorageStack}; -use temper_spec::parse_csdl; +use temper_platform::bootstrap::{bootstrap_operator_credential, bootstrap_system_tenant}; +use temper_platform::router::build_platform_router; +use temper_platform::state::PlatformState; +use temper_runtime::tenant::TenantId; +use temper_server::StorageStack; +use temper_server::registry::{ + EntityLevelSummary, EntityVerificationResult, SpecRegistry, VerificationStatus, +}; +use temper_spec::csdl::parse_csdl; use temper_store_turso::TursoEventStore; use tokio::net::TcpListener; use tokio::sync::oneshot; +const TEST_TENANT: &str = "demo"; +const TEST_OPERATOR_KEY: &str = "temper-mcp-test-operator-key"; +const ORDER_CSDL_XML: &str = include_str!("../../../test-fixtures/specs/model.csdl.xml"); +const ORDER_IOA: &str = include_str!("../../../test-fixtures/specs/order.ioa.toml"); +const AGENT_TYPE_IOA: &str = include_str!("../../temper-platform/src/specs/agent_type.ioa.toml"); +const AGENT_CREDENTIAL_IOA: &str = + include_str!("../../temper-platform/src/specs/agent_credential.ioa.toml"); + +const CREDENTIAL_CSDL_XML: &str = r#" + + + + + + + + + + + + + + + + + + + + + + + + + + + +"#; + +const TEST_OPERATOR_POLICY: &str = r#" +permit(principal == Agent::"operator", action == Action::"create", resource is Order); +permit(principal == Agent::"operator", action == Action::"read", resource is Order); +permit(principal == Agent::"operator", action == Action::"list", resource is Order); +permit(principal == Agent::"operator", action == Action::"CancelOrder", resource is Order); +permit(principal == Agent::"operator", action == Action::"read_specs", resource is Spec); +"#; + /// Build a RuntimeContext pointing at a local port. fn ctx_for_port(port: u16) -> RuntimeContext { RuntimeContext::from_config(&McpConfig { @@ -25,6 +77,19 @@ fn ctx_for_port(port: u16) -> RuntimeContext { .expect("ctx") } +/// Build an authenticated RuntimeContext pointing at a local test server. +fn authenticated_ctx_for_port(port: u16) -> RuntimeContext { + RuntimeContext::from_config(&McpConfig { + temper_port: Some(port), + temper_url: None, + agent_id: None, + agent_type: None, + session_id: Some("temper-mcp-test-session".to_string()), + api_key: Some(TEST_OPERATOR_KEY.to_string()), + }) + .expect("ctx") +} + /// Build a RuntimeContext pointing at a URL. fn ctx_for_url(url: &str) -> RuntimeContext { RuntimeContext::from_config(&McpConfig { @@ -38,6 +103,19 @@ fn ctx_for_url(url: &str) -> RuntimeContext { .expect("ctx") } +/// Build an authenticated RuntimeContext pointing at a URL. +fn authenticated_ctx_for_url(url: &str) -> RuntimeContext { + RuntimeContext::from_config(&McpConfig { + temper_port: None, + temper_url: Some(url.to_string()), + agent_id: None, + agent_type: None, + session_id: Some("temper-mcp-test-session".to_string()), + api_key: Some(TEST_OPERATOR_KEY.to_string()), + }) + .expect("ctx") +} + async fn rpc(ctx: &mut RuntimeContext, request: Value) -> Value { dispatch_json_value(ctx, request) .await @@ -70,6 +148,68 @@ fn tool_text(response: &Value) -> (&str, bool) { (text, is_error) } +fn build_test_registry() -> SpecRegistry { + let csdl = parse_csdl(ORDER_CSDL_XML).expect("parse order CSDL"); + let mut registry = SpecRegistry::new(); + registry.register_tenant( + TEST_TENANT, + csdl, + ORDER_CSDL_XML.to_string(), + &[("Order", ORDER_IOA)], + ); + mark_specs_verified(&mut registry, TEST_TENANT, &[("Order", ORDER_IOA)]); + registry +} + +fn mark_specs_verified(registry: &mut SpecRegistry, tenant: &str, specs: &[(&str, &str)]) { + let tenant_id = TenantId::new(tenant); + for (entity_type, _) in specs { + registry.set_verification_status( + &tenant_id, + entity_type, + VerificationStatus::Completed(EntityVerificationResult { + all_passed: true, + levels: vec![EntityLevelSummary { + level: "MCP fixture".to_string(), + passed: true, + summary: "Pre-verified for authenticated MCP E2E tests".to_string(), + details: None, + }], + verified_at: "2026-07-11T00:00:00Z".to_string(), + }), + ); + } +} + +async fn register_test_operator(state: &PlatformState) { + let credential_csdl = parse_csdl(CREDENTIAL_CSDL_XML).expect("parse credential CSDL"); + let credential_specs = [ + ("AgentType", AGENT_TYPE_IOA), + ("AgentCredential", AGENT_CREDENTIAL_IOA), + ]; + { + let mut registry = state.registry.write().expect("registry lock"); + registry + .try_register_tenant_with_reactions_and_constraints( + TenantId::new(TEST_TENANT), + credential_csdl, + CREDENTIAL_CSDL_XML.to_string(), + &credential_specs, + Vec::new(), + None, + true, + ) + .expect("merge credential specs"); + mark_specs_verified(&mut registry, TEST_TENANT, &credential_specs); + } + bootstrap_operator_credential(state, TEST_OPERATOR_KEY, TEST_TENANT).await; + state + .server + .authz + .reload_tenant_policies(TEST_TENANT, TEST_OPERATOR_POLICY) + .expect("load test operator policy"); +} + async fn start_test_temper_server() -> (u16, oneshot::Sender<()>) { static COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); let id = COUNTER.fetch_add(1, std::sync::atomic::Ordering::SeqCst); @@ -79,25 +219,14 @@ async fn start_test_temper_server() -> (u16, oneshot::Sender<()>) { .await .expect("create local turso db"); - let csdl_xml = include_str!("../../../test-fixtures/specs/model.csdl.xml"); - let csdl = parse_csdl(csdl_xml).expect("parse csdl"); - - let mut ioa_sources = BTreeMap::new(); - ioa_sources.insert( - "Order".to_string(), - include_str!("../../../test-fixtures/specs/order.ioa.toml").to_string(), - ); - - let mut state = ServerState::with_specs( - ActorSystem::new("temper-mcp-tests"), - csdl, - csdl_xml.to_string(), - ioa_sources, - ) - .unwrap(); - state.set_storage_stack(StorageStack::from_turso(turso)); + let mut state = PlatformState::with_registry(build_test_registry(), None); + state + .server + .set_storage_stack(StorageStack::from_turso(turso)); + bootstrap_system_tenant(&state, &BTreeMap::new()); + register_test_operator(&state).await; - let router: Router = temper_server::build_router(state); + let router: Router = build_platform_router(state); let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); let port = listener.local_addr().expect("addr").port(); @@ -253,7 +382,7 @@ fn from_config_port_mode() { #[tokio::test] async fn execute_url_mode_works() { let (port, shutdown) = start_test_temper_server().await; - let mut ctx = ctx_for_url(&format!("http://127.0.0.1:{port}")); + let mut ctx = authenticated_ctx_for_url(&format!("http://127.0.0.1:{port}")); let response = rpc( &mut ctx, @@ -279,7 +408,7 @@ async fn execute_url_mode_works() { #[tokio::test] async fn execute_creates_entity_and_reads_it_back() { let (port, shutdown) = start_test_temper_server().await; - let mut ctx = ctx_for_port(port); + let mut ctx = authenticated_ctx_for_port(port); let response = rpc( &mut ctx, @@ -306,7 +435,7 @@ return fetched['fields']['customer'] #[tokio::test] async fn execute_invalid_action_returns_409_cleanly() { let (port, shutdown) = start_test_temper_server().await; - let mut ctx = ctx_for_port(port); + let mut ctx = authenticated_ctx_for_port(port); let response = rpc( &mut ctx, @@ -315,7 +444,8 @@ async fn execute_invalid_action_returns_409_cleanly() { "execute", r#" await temper.create('demo', 'Orders', {'id': 'mcp-bad-action-1'}) -await temper.action('demo', 'Orders', 'mcp-bad-action-1', 'ShipOrder', {'Reason': 'invalid from draft'}) +await temper.action('demo', 'Orders', 'mcp-bad-action-1', 'CancelOrder', {'Reason': 'first cancel'}) +await temper.action('demo', 'Orders', 'mcp-bad-action-1', 'CancelOrder', {'Reason': 'cancelled is final'}) return 'unreachable' "#, ), @@ -335,7 +465,7 @@ return 'unreachable' #[tokio::test] async fn execute_supports_compound_operation() { let (port, shutdown) = start_test_temper_server().await; - let mut ctx = ctx_for_port(port); + let mut ctx = authenticated_ctx_for_port(port); let response = rpc( &mut ctx, @@ -363,11 +493,10 @@ return fetched['status'] #[tokio::test] async fn execute_specs_returns_data() { let (port, shutdown) = start_test_temper_server().await; - let mut ctx = ctx_for_port(port); + let mut ctx = authenticated_ctx_for_port(port); - // The test server uses with_specs() which doesn't populate the spec registry. - // The /observe/specs endpoint reads from the registry, so it returns an empty list - // (not a 404). This verifies the specs() method dispatches correctly. + // The authenticated fixture uses the platform router so this call verifies + // specs() dispatches with the same bearer-to-typed-context path as runtime. let response = rpc( &mut ctx, call_tool_request(10, "execute", "return await temper.specs('demo')"), diff --git a/crates/temper-mcp/src/runtime.rs b/crates/temper-mcp/src/runtime.rs index 72d2986ee..e89e147e7 100644 --- a/crates/temper-mcp/src/runtime.rs +++ b/crates/temper-mcp/src/runtime.rs @@ -10,10 +10,16 @@ use temper_ots::{ OTSMessageContent, OTSMetadata, OutcomeType, TrajectoryBuilder, }; use temper_runtime::scheduler::sim_now; -use tokio::io::{self, AsyncBufReadExt, AsyncWriteExt, BufReader}; +use tokio::io::{self, AsyncWriteExt, BufReader}; use super::McpConfig; use super::protocol::dispatch_json_line; +use crate::trajectory_bounds::{ + MAX_STDIO_LINE_BYTES, MAX_TRAJECTORY_TOTAL_BYTES, MAX_TRAJECTORY_TURNS, StdioFrame, + TRAJECTORY_TURN_ENVELOPE_BYTES, bounded_trajectory_actions, bump_seen, floor_char_boundary, + json_string_cost, json_value_cost, read_stdio_frame, trajectory_storage_tenant, + truncate_trajectory_text, +}; const OTS_UPLOAD_MAX_ATTEMPTS: u32 = 3; const OTS_UPLOAD_RETRY_DELAY_MS: u64 = 100; @@ -56,10 +62,18 @@ pub(crate) struct RuntimeContext { sandbox: temper_sandbox::runner::PersistentSandbox, /// OTS trajectory builder for capturing agent execution traces. pub(crate) trajectory: Option, - /// Tenants observed in executed calls during this session. + /// Tenants observed in executed calls during this session (observability + /// signal only; never used to route trajectory storage). tenants_seen: BTreeMap, - /// Entity types observed in executed calls during this session. - entity_types_seen: BTreeMap, + /// Number of turns recorded into the current trajectory (bounds its size). + turns_recorded: usize, + /// Estimated serialized bytes recorded so far. Bounds the whole upload to + /// under the server's ingest limit so a within-per-turn-cap session can't + /// still produce a trajectory the server rejects (ARN-222). + trajectory_bytes: usize, + /// Whether the turn/byte cap has already been reported for this trajectory, + /// so hitting the cap warns once rather than on every subsequent turn. + capped_warned: bool, } impl RuntimeContext { @@ -89,7 +103,9 @@ impl RuntimeContext { sandbox: temper_sandbox::runner::PersistentSandbox::new(&[("temper", "Temper", 1)]), trajectory: None, tenants_seen: BTreeMap::new(), - entity_types_seen: BTreeMap::new(), + turns_recorded: 0, + trajectory_bytes: 0, + capped_warned: false, }) } @@ -158,6 +174,10 @@ impl RuntimeContext { } /// Initialize OTS trajectory capture after the MCP handshake completes. + /// + /// Resets the per-trajectory budgets so a client that re-sends `initialize` + /// gets a fresh trajectory with a fresh turn/byte budget rather than + /// inheriting a consumed one. pub(crate) fn init_trajectory(&mut self) { let now = sim_now(); // determinism-ok: sim_now is DST-safe let agent_id = self.agent_id.as_deref().unwrap_or("unknown"); @@ -166,11 +186,73 @@ impl RuntimeContext { let context = OTSContext::new(); self.trajectory = Some(TrajectoryBuilder::new(metadata, context)); + self.turns_recorded = 0; + self.trajectory_bytes = 0; + self.capped_warned = false; + } + + /// Warn once per trajectory that a capture cap was hit, so a long agent loop + /// that keeps executing past the cap does not emit an identical warning every + /// turn. + fn warn_capped_once(&mut self, cap: &str, limit: usize) { + if !self.capped_warned { + self.capped_warned = true; + tracing::warn!( + cap, + limit, + "mcp trajectory cap reached; dropping further turns" + ); + } } /// Record an execute tool call as an OTS turn with a decision. pub(crate) fn record_execute_turn(&mut self, code: &str, result: &Result) { + if self.trajectory.is_none() { + return; + } + // Bound the trajectory: drop further turns once the cap is reached (ARN-222). + if self.turns_recorded >= MAX_TRAJECTORY_TURNS { + self.warn_capped_once("turn count", MAX_TRAJECTORY_TURNS); + return; + } + let extracted_actions = extract_trajectory_actions_from_code(code); + // Bound recorded content: a large code blob or result can't grow the + // trajectory without limit (ARN-222). + let code_text = truncate_trajectory_text(code); + let result_text = match result { + Ok(text) => truncate_trajectory_text(text), + Err(e) => truncate_trajectory_text(&e.to_string()), + }; + let is_failure = result.is_err(); + let action_arguments = (!extracted_actions.is_empty()).then(|| { + serde_json::json!({ + "trajectory_actions": bounded_trajectory_actions(extracted_actions), + }) + }); + + // Total-size budget: meter the turn's *serialized* contribution (escaped + // text, the embedded actions, the duplicated error field, and a per-turn + // envelope) — not raw string bytes, which undercount the wire size by 2x + // or more. Stop recording once the cumulative serialized estimate would + // exceed the budget, so the whole upload stays under the server's ingest + // limit and is never rejected (and silently dropped) as too large, which + // would suppress the audit trail (ARN-222). + let turn_bytes = TRAJECTORY_TURN_ENVELOPE_BYTES + + json_string_cost(&code_text) + + json_string_cost(&result_text) + + if is_failure { + json_string_cost(&result_text) + } else { + 0 + } + + action_arguments.as_ref().map(json_value_cost).unwrap_or(0); + if self.trajectory_bytes + turn_bytes > MAX_TRAJECTORY_TOTAL_BYTES { + self.warn_capped_once("serialized byte budget", MAX_TRAJECTORY_TOTAL_BYTES); + return; + } + self.turns_recorded += 1; + self.trajectory_bytes += turn_bytes; let Some(ref mut builder) = self.trajectory else { return; @@ -182,39 +264,39 @@ impl RuntimeContext { // User message: the Python code submitted builder.add_message(OTSMessage::new( MessageRole::User, - OTSMessageContent::text(code), + OTSMessageContent::text(code_text), now, )); // Decision: the execution outcome let (outcome_str, consequence) = match result { - Ok(text) => { + Ok(_) => { // Assistant message: the execution result builder.add_message(OTSMessage::new( MessageRole::Assistant, - OTSMessageContent::text(text), + OTSMessageContent::text(result_text), now, )); ("success", OTSConsequence::success()) } - Err(e) => { + Err(_) => { builder.add_message(OTSMessage::new( MessageRole::Assistant, - OTSMessageContent::text(e.to_string()), + OTSMessageContent::text(result_text.clone()), now, )); + // Bound the error_type too — it is the same (already truncated) text. ( "failure", - OTSConsequence::failure().with_error_type(e.to_string()), + OTSConsequence::failure().with_error_type(result_text), ) } }; - let mut choice = OTSChoice::new(format!("execute: {}", &code[..code.len().min(100)])); - if !extracted_actions.is_empty() { - choice = choice.with_arguments(serde_json::json!({ - "trajectory_actions": extracted_actions, - })); + let label_end = floor_char_boundary(code, 100); + let mut choice = OTSChoice::new(format!("execute: {}", &code[..label_end])); + if let Some(arguments) = action_arguments { + choice = choice.with_arguments(arguments); } let decision = OTSDecision::new(DecisionType::ToolSelection, choice, consequence); @@ -225,17 +307,30 @@ impl RuntimeContext { tracing::debug!(outcome = outcome_str, "ots.trajectory.turn_recorded"); for meta in extract_temper_call_metadata(code) { + // Cap the distinct-key growth of these observability maps so a single + // large code blob can't insert unbounded unique keys (ARN-222). if let Some(tenant) = meta.tenant { - self.tenants_seen - .entry(tenant) - .and_modify(|count| *count += 1) - .or_insert(1); - } - if let Some(entity_type) = meta.entity_type { - self.entity_types_seen - .entry(entity_type) - .and_modify(|count| *count += 1) - .or_insert(1); + // Emit the cross-tenant signal once per unique foreign tenant, the + // turn it is first tracked — never on every turn. `bump_seen` + // returns true only when it actually inserts a new key, so once + // `tenants_seen` saturates (or the key is oversized) it stops + // returning true and the warn does not spam. Storage is unaffected. + let foreign = if tenant != self.identity_tenant { + Some(tenant.clone()) + } else { + None + }; + let newly_tracked = bump_seen(&mut self.tenants_seen, tenant); + if let Some(referenced) = foreign + && newly_tracked + { + tracing::warn!( + identity_tenant = %self.identity_tenant, + referenced_tenant = %referenced, + "mcp session code referenced a tenant other than its identity; \ + trajectory stored under the identity tenant" + ); + } } } } @@ -325,11 +420,17 @@ impl RuntimeContext { .post(&url) .body(json) .header("Content-Type", "application/json") - .header("X-Tenant-Id", self.primary_tenant()); - - if let Some(primary_entity_type) = self.primary_entity_type() { - request = request.header("X-Entity-Type", primary_entity_type); - } + .header( + "X-Tenant-Id", + trajectory_storage_tenant(&self.identity_tenant), + ); + + // No code-derived value is placed in a request header: a `\n` or other + // illegal byte in an attacker-controlled entity type would make the HTTP + // client reject the whole upload, silently losing the trajectory. The + // previous `X-Entity-Type` header was code-derived and had no server-side + // reader, so it is dropped entirely (ARN-222). Agent/session ids below are + // startup config, not code-derived. if let Some(ref agent_id) = self.agent_id { request = request.header("X-Agent-Id", agent_id); } @@ -360,28 +461,10 @@ impl RuntimeContext { } } - /// Most-used tenant for this session, falling back to configured identity tenant. - fn primary_tenant(&self) -> &str { - self.tenants_seen - .iter() - .max_by_key(|(_, count)| *count) - .map(|(tenant, _)| tenant.as_str()) - .unwrap_or(self.identity_tenant.as_str()) - } - - /// Most-used entity type for this session. - fn primary_entity_type(&self) -> Option<&str> { - self.entity_types_seen - .iter() - .max_by_key(|(_, count)| *count) - .map(|(entity_type, _)| entity_type.as_str()) - } - pub(crate) async fn run_execute(&mut self, code: &str) -> Result { let http = self.http.clone(); let base_url = self.base_url.clone(); let agent_id = self.agent_id.clone(); - let agent_type = self.agent_type.clone(); let session_id = self.session_id.clone(); let api_key = self.api_key.clone(); @@ -394,7 +477,6 @@ impl RuntimeContext { let http = http.clone(); let base_url = base_url.clone(); let agent_id = agent_id.clone(); - let agent_type = agent_type.clone(); let session_id = session_id.clone(); let api_key = api_key.clone(); async move { @@ -425,14 +507,15 @@ impl RuntimeContext { base_url: &base_url, tenant: &tenant, agent_id: agent_id.as_deref(), - agent_type: agent_type.as_deref(), session_id: session_id.as_deref(), - principal_id: None, - principal_kind: None, - agent_role: None, entity_set_resolver: None, binary_path: None, api_key: api_key.as_deref(), + internal_credential_issuer: None, + // Local stdio MCP: the host process is the + // developer's own machine, so upload_wasm/compile_wasm + // are legitimate developer ops (ARN-166). + allow_host_ops: true, }; temper_sandbox::dispatch::dispatch_temper_method( &ctx, @@ -507,8 +590,10 @@ fn extract_trajectory_actions_from_code(code: &str) -> Vec { #[derive(Debug, Clone, Default)] struct TemperCallMetadata { + /// Tenant referenced by the call, when the call uses the tenant-first + /// signature. Used only as a cross-tenant observability signal — never to + /// route trajectory storage (ARN-222). tenant: Option, - entity_type: Option, } fn extract_temper_call_metadata(code: &str) -> Vec { @@ -520,52 +605,23 @@ fn extract_temper_call_metadata(code: &str) -> Vec { fn extract_temper_action_metadata(code: &str) -> Vec { extract_call_metadata(code, "temper.action", |args| { - // New signature: temper.action(tenant, entity_type, id, action, params) - if args.len() >= 5 - && let (Some(tenant), Some(entity_type), Some(_action)) = ( - parse_python_string_literal(args[0]), - parse_python_string_literal(args[1]), - parse_python_string_literal(args[3]), - ) - { - return TemperCallMetadata { - tenant: Some(tenant), - entity_type: Some(entity_type), - }; - } - - // Legacy signature: temper.action(entity_type, id, action, params) - TemperCallMetadata { - tenant: None, - entity_type: args - .first() - .and_then(|raw| parse_python_string_literal(raw)), - } + // New signature: temper.action(tenant, entity_type, id, action, params). + // Only the tenant is retained; the legacy signature carries no tenant. + let tenant = (args.len() >= 5) + .then(|| parse_python_string_literal(args[0])) + .flatten(); + TemperCallMetadata { tenant } }) } fn extract_temper_create_metadata(code: &str) -> Vec { extract_call_metadata(code, "temper.create", |args| { - // New signature: temper.create(tenant, entity_type, fields) - if args.len() >= 3 - && let (Some(tenant), Some(entity_type)) = ( - parse_python_string_literal(args[0]), - parse_python_string_literal(args[1]), - ) - { - return TemperCallMetadata { - tenant: Some(tenant), - entity_type: Some(entity_type), - }; - } - - // Legacy signature: temper.create(entity_type, fields) - TemperCallMetadata { - tenant: None, - entity_type: args - .first() - .and_then(|raw| parse_python_string_literal(raw)), - } + // New signature: temper.create(tenant, entity_type, fields). Only the + // tenant is retained; the legacy signature carries no tenant. + let tenant = (args.len() >= 3) + .then(|| parse_python_string_literal(args[0])) + .flatten(); + TemperCallMetadata { tenant } }) } @@ -844,14 +900,34 @@ fn normalize_pythonish_json(input: &str) -> String { } /// Run the MCP server on stdio with JSON-RPC over newline-delimited JSON. +/// +/// Frames are read through [`read_stdio_frame`], which bounds each frame to +/// `MAX_STDIO_LINE_BYTES`; oversized frames are dropped and invalid UTF-8 frames +/// are skipped rather than aborting the session. pub async fn run_stdio_server(config: McpConfig) -> Result<()> { let mut ctx = RuntimeContext::from_config(&config)?; - let stdin = BufReader::new(io::stdin()); - let mut lines = stdin.lines(); + let mut stdin = BufReader::new(io::stdin()); let mut stdout = io::stdout(); - while let Some(line) = lines.next_line().await? { - let line = line.trim(); + loop { + let buf = match read_stdio_frame(&mut stdin).await? { + StdioFrame::Eof => break, + StdioFrame::TooLarge => { + tracing::warn!( + limit = MAX_STDIO_LINE_BYTES, + "mcp.stdio.frame_too_large: dropped oversized frame" + ); + continue; + } + StdioFrame::Line(buf) => buf, + }; + let line = match std::str::from_utf8(&buf) { + Ok(text) => text.trim(), + Err(_) => { + tracing::warn!("mcp.stdio.invalid_utf8: dropped frame"); + continue; + } + }; if line.is_empty() { continue; } @@ -871,108 +947,5 @@ pub async fn run_stdio_server(config: McpConfig) -> Result<()> { } #[cfg(test)] -mod tests { - use super::*; - use axum::{Router, extract::State, http::StatusCode, routing::post}; - use std::sync::{ - Arc, - atomic::{AtomicUsize, Ordering}, - }; - - #[test] - fn extract_trajectory_actions_from_temper_action_calls() { - let code = r#" -result = temper.action("Issue", "issue-1", "PromoteToCritical", {"Reason": "prod incident"}) -other = temper.action('Issue', 'issue-1', 'Assign', {'AgentId': 'agent-2'}) -tenant = temper.action("gepa-tenant", "Issues", "11111111-1111-1111-1111-111111111111", "Reassign", {"NewAssigneeId": "agent-3"}) -"#; - - let actions = extract_trajectory_actions_from_code(code); - assert_eq!(actions.len(), 3); - assert_eq!( - actions[0].get("action").and_then(Value::as_str), - Some("PromoteToCritical") - ); - assert_eq!( - actions[2] - .get("params") - .and_then(Value::as_object) - .and_then(|m| m.get("NewAssigneeId")) - .and_then(Value::as_str), - Some("agent-3") - ); - } - - #[test] - fn normalize_python_literals_to_json() { - let value = parse_python_json_value("{'enabled': True, 'reason': None, 'count': 2}") - .expect("python dict should parse"); - assert_eq!(value["enabled"], serde_json::json!(true)); - assert_eq!(value["reason"], serde_json::Value::Null); - assert_eq!(value["count"], serde_json::json!(2)); - } - - #[test] - fn extract_temper_call_metadata_tracks_tenant_and_entity() { - let code = r#" -await temper.action("tenant-a", "Issue", "i-1", "Assign", {"AgentId": "agent-1"}) -await temper.create("tenant-b", "Task", {"Title": "x"}) -"#; - let metadata = extract_temper_call_metadata(code); - assert!( - metadata.iter().any(|m| { - m.tenant.as_deref() == Some("tenant-a") && m.entity_type.as_deref() == Some("Issue") - }), - "expected tenant-a/Issue metadata" - ); - assert!( - metadata.iter().any(|m| { - m.tenant.as_deref() == Some("tenant-b") && m.entity_type.as_deref() == Some("Task") - }), - "expected tenant-b/Task metadata" - ); - } - - #[tokio::test] - async fn finalize_trajectory_retries_retryable_ots_upload_failure() { - async fn handler(State(attempts): State>) -> StatusCode { - let attempt = attempts.fetch_add(1, Ordering::SeqCst); - if attempt == 0 { - StatusCode::SERVICE_UNAVAILABLE - } else { - StatusCode::ACCEPTED - } - } - - let attempts = Arc::new(AtomicUsize::new(0)); - let app = Router::new() - .route("/api/ots/trajectories", post(handler)) - .with_state(attempts.clone()); - let listener = tokio::net::TcpListener::bind("127.0.0.1:0") - .await - .expect("bind test server"); - let addr = listener.local_addr().expect("test server addr"); - tokio::spawn(async move { - axum::serve(listener, app).await.expect("serve test server"); - }); - - let mut ctx = RuntimeContext { - base_url: format!("http://{addr}"), - http: reqwest::Client::new(), - agent_id: Some("agent".to_string()), - agent_type: Some("test-agent".to_string()), - session_id: Some("session".to_string()), - api_key: None, - identity_tenant: "tenant".to_string(), - sandbox: temper_sandbox::runner::PersistentSandbox::new(&[("temper", "Temper", 1)]), - trajectory: None, - tenants_seen: BTreeMap::new(), - entity_types_seen: BTreeMap::new(), - }; - ctx.init_trajectory(); - - ctx.finalize_trajectory().await; - - assert_eq!(attempts.load(Ordering::SeqCst), 2); - } -} +#[path = "runtime_test.rs"] +mod tests; diff --git a/crates/temper-mcp/src/runtime_test.rs b/crates/temper-mcp/src/runtime_test.rs new file mode 100644 index 000000000..fbbc8c7cb --- /dev/null +++ b/crates/temper-mcp/src/runtime_test.rs @@ -0,0 +1,329 @@ +//! Unit tests for the MCP runtime (`runtime.rs`). Split out so `runtime.rs` +//! stays under the file-size budget; excluded from the readability ratchet as a +//! `*_test.rs` file. + +use super::*; +use crate::trajectory_bounds::{MAX_TRAJECTORY_ACTIONS, MAX_TRAJECTORY_TEXT_BYTES}; +use axum::{Router, extract::State, http::HeaderMap, http::StatusCode, routing::post}; +use std::sync::Mutex; + +#[test] +fn bounded_trajectory_actions_caps_count_and_size() { + // ARN-222: trajectory_actions come from the full untruncated code and must + // be bounded in both count and serialized size. + let many: Vec = (0..MAX_TRAJECTORY_ACTIONS + 50) + .map(|i| serde_json::json!({ "a": i })) + .collect(); + assert_eq!( + bounded_trajectory_actions(many).as_array().map(|a| a.len()), + Some(MAX_TRAJECTORY_ACTIONS), + "action count must be capped" + ); + // One action with a huge params blob collapses to a summary. + let huge = vec![serde_json::json!({ "params": "x".repeat(MAX_TRAJECTORY_TEXT_BYTES * 2) })]; + assert_eq!( + bounded_trajectory_actions(huge).get("trajectory_actions_truncated"), + Some(&serde_json::json!(true)), + "oversized actions must collapse to a bounded summary" + ); +} + +#[tokio::test] +async fn trajectory_upload_uses_identity_tenant_header() { + // ARN-222 (wire-level): the trajectory must be filed under the authenticated + // identity even when the code referenced other tenants — a call-site guard. + async fn handler( + State(captured): State>>>, + headers: HeaderMap, + ) -> StatusCode { + let tenant = headers + .get("X-Tenant-Id") + .and_then(|v| v.to_str().ok()) + .map(str::to_string); + *captured.lock().expect("lock") = tenant; + StatusCode::ACCEPTED + } + + let captured = Arc::new(Mutex::new(None)); + let app = Router::new() + .route("/api/ots/trajectories", post(handler)) + .with_state(captured.clone()); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind"); + let addr = listener.local_addr().expect("addr"); + tokio::spawn(async move { + axum::serve(listener, app).await.expect("serve"); + }); + + let mut ctx = trajectory_test_ctx("my-identity"); + ctx.base_url = format!("http://{addr}"); + // Code referenced a foreign tenant far more than the identity. + ctx.tenants_seen.insert("attacker-tenant".to_string(), 99); + ctx.init_trajectory(); + ctx.finalize_trajectory().await; + + assert_eq!( + captured.lock().expect("lock").as_deref(), + Some("my-identity"), + "trajectory must be filed under the identity tenant, not a code-referenced one" + ); +} + +#[test] +fn trajectory_storage_tenant_uses_identity_not_code() { + // ARN-222: the trajectory must be stored under the authenticated session + // identity, never a tenant derived from the (attacker-controlled) code. + assert_eq!( + trajectory_storage_tenant("my-identity"), + "my-identity", + "code-referenced tenants must not move the trajectory off the identity" + ); +} + +#[test] +fn truncate_trajectory_text_bounds_large_content() { + // ARN-222: recorded code/result text must be bounded. + let big = "x".repeat(MAX_TRAJECTORY_TEXT_BYTES * 4); + let truncated = truncate_trajectory_text(&big); + assert!( + truncated.len() <= MAX_TRAJECTORY_TEXT_BYTES + 128, + "recorded text must be bounded, got {} bytes", + truncated.len() + ); + assert!( + truncated.contains("truncated"), + "truncation must be marked in the recorded text" + ); + // Text within the cap is recorded verbatim. + assert_eq!(truncate_trajectory_text("hello"), "hello"); + // A multibyte char straddling the cap must not panic and must stay valid UTF-8. + let multibyte = "é".repeat(MAX_TRAJECTORY_TEXT_BYTES); // 2 bytes each + let t = truncate_trajectory_text(&multibyte); + assert!(t.len() <= MAX_TRAJECTORY_TEXT_BYTES + 128); + assert!(t.contains("truncated")); +} + +fn trajectory_test_ctx(identity: &str) -> RuntimeContext { + RuntimeContext { + base_url: "http://127.0.0.1:1".to_string(), + http: reqwest::Client::new(), + agent_id: None, + agent_type: None, + session_id: None, + api_key: None, + identity_tenant: identity.to_string(), + sandbox: temper_sandbox::runner::PersistentSandbox::new(&[("temper", "Temper", 1)]), + trajectory: None, + tenants_seen: BTreeMap::new(), + turns_recorded: 0, + trajectory_bytes: 0, + capped_warned: false, + } +} + +#[test] +fn turn_cap_bounds_recorded_turns() { + // ARN-222: the trajectory records up to MAX_TRAJECTORY_TURNS turns, then + // drops further turns instead of growing without bound. + let mut ctx = trajectory_test_ctx("t"); + ctx.init_trajectory(); + let ok: Result = Ok("result".to_string()); + + ctx.record_execute_turn("code", &ok); + assert_eq!(ctx.turns_recorded, 1, "a turn under the cap is recorded"); + + ctx.turns_recorded = MAX_TRAJECTORY_TURNS; + ctx.record_execute_turn("code", &ok); + assert_eq!( + ctx.turns_recorded, MAX_TRAJECTORY_TURNS, + "a turn at the cap is dropped, not recorded" + ); +} + +#[test] +fn total_byte_budget_stops_recording() { + // ARN-222: even within the per-turn and turn-count caps, the total recorded + // text is bounded so the upload can't exceed the server ingest limit and be + // silently dropped. Once the byte budget is (near) exhausted, further turns + // are not recorded. + use crate::trajectory_bounds::MAX_TRAJECTORY_TOTAL_BYTES; + let mut ctx = trajectory_test_ctx("t"); + ctx.init_trajectory(); + let ok: Result = Ok("r".to_string()); + + // Pretend the budget is already spent. + ctx.trajectory_bytes = MAX_TRAJECTORY_TOTAL_BYTES; + let before = ctx.turns_recorded; + ctx.record_execute_turn("more code", &ok); + assert_eq!( + ctx.turns_recorded, before, + "a turn over the byte budget is dropped, not recorded" + ); +} + +#[test] +fn worst_case_trajectory_serializes_under_server_ingest_limit() { + // ARN-222 (the real audit-suppression guarantee): a session that records up to + // every cap must still serialize to under the server's 2 MiB ingest limit, so + // it is never 413'd and silently dropped. Uses control-character text (each + // byte escapes to `\u00XX`, 6x) plus embedded actions and a failing result so + // error_type is duplicated — every budgeted contributor at once, not just raw + // byte length. + const SERVER_INGEST_LIMIT: usize = 2 * 1024 * 1024; + let mut ctx = trajectory_test_ctx("t"); + ctx.init_trajectory(); + + let filler = "\u{0001}".repeat(MAX_TRAJECTORY_TEXT_BYTES); + let code = format!("temper.action(\"t\", \"E\", \"id\", \"Act\", {{\"x\": \"{filler}\"}})"); + let err: Result = Err(anyhow::anyhow!( + "{}", + "\u{0001}".repeat(MAX_TRAJECTORY_TEXT_BYTES) + )); + + for _ in 0..MAX_TRAJECTORY_TURNS { + ctx.record_execute_turn(&code, &err); + } + + let snapshot = ctx.trajectory.as_ref().expect("trajectory").snapshot(); + let serialized = serde_json::to_string(&snapshot).expect("serialize"); + assert!( + serialized.len() < SERVER_INGEST_LIMIT, + "worst-case trajectory serialized to {} bytes; must stay under the {} server limit", + serialized.len(), + SERVER_INGEST_LIMIT + ); +} + +#[test] +fn init_trajectory_resets_budgets() { + // Fable follow-up: re-sending `initialize` must reset the turn and byte + // budgets, not inherit a consumed one. + let mut ctx = trajectory_test_ctx("t"); + ctx.init_trajectory(); + ctx.turns_recorded = 123; + ctx.trajectory_bytes = 456; + ctx.capped_warned = true; + ctx.init_trajectory(); + assert_eq!(ctx.turns_recorded, 0); + assert_eq!(ctx.trajectory_bytes, 0); + assert!( + !ctx.capped_warned, + "the cap-warned flag must reset on re-initialize" + ); +} + +#[test] +fn choice_label_does_not_panic_on_multibyte_code() { + // ARN-222 follow-up: the choice label slices the first 100 bytes of `code`; + // a multibyte char at the boundary must not crash the client. + let mut ctx = trajectory_test_ctx("t"); + ctx.init_trajectory(); + let code = "é".repeat(80); // 160 bytes; byte 100 is mid-char + let ok: Result = Ok("ok".to_string()); + ctx.record_execute_turn(&code, &ok); // must not panic + assert_eq!(ctx.turns_recorded, 1); +} +use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, +}; + +#[test] +fn extract_trajectory_actions_from_temper_action_calls() { + let code = r#" +result = temper.action("Issue", "issue-1", "PromoteToCritical", {"Reason": "prod incident"}) +other = temper.action('Issue', 'issue-1', 'Assign', {'AgentId': 'agent-2'}) +tenant = temper.action("gepa-tenant", "Issues", "11111111-1111-1111-1111-111111111111", "Reassign", {"NewAssigneeId": "agent-3"}) +"#; + + let actions = extract_trajectory_actions_from_code(code); + assert_eq!(actions.len(), 3); + assert_eq!( + actions[0].get("action").and_then(Value::as_str), + Some("PromoteToCritical") + ); + assert_eq!( + actions[2] + .get("params") + .and_then(Value::as_object) + .and_then(|m| m.get("NewAssigneeId")) + .and_then(Value::as_str), + Some("agent-3") + ); +} + +#[test] +fn normalize_python_literals_to_json() { + let value = parse_python_json_value("{'enabled': True, 'reason': None, 'count': 2}") + .expect("python dict should parse"); + assert_eq!(value["enabled"], serde_json::json!(true)); + assert_eq!(value["reason"], serde_json::Value::Null); + assert_eq!(value["count"], serde_json::json!(2)); +} + +#[test] +fn extract_temper_call_metadata_tracks_tenant() { + let code = r#" +await temper.action("tenant-a", "Issue", "i-1", "Assign", {"AgentId": "agent-1"}) +await temper.create("tenant-b", "Task", {"Title": "x"}) +"#; + let metadata = extract_temper_call_metadata(code); + assert!( + metadata + .iter() + .any(|m| m.tenant.as_deref() == Some("tenant-a")), + "expected tenant-a metadata" + ); + assert!( + metadata + .iter() + .any(|m| m.tenant.as_deref() == Some("tenant-b")), + "expected tenant-b metadata" + ); +} + +#[tokio::test] +async fn finalize_trajectory_retries_retryable_ots_upload_failure() { + async fn handler(State(attempts): State>) -> StatusCode { + let attempt = attempts.fetch_add(1, Ordering::SeqCst); + if attempt == 0 { + StatusCode::SERVICE_UNAVAILABLE + } else { + StatusCode::ACCEPTED + } + } + + let attempts = Arc::new(AtomicUsize::new(0)); + let app = Router::new() + .route("/api/ots/trajectories", post(handler)) + .with_state(attempts.clone()); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind test server"); + let addr = listener.local_addr().expect("test server addr"); + tokio::spawn(async move { + axum::serve(listener, app).await.expect("serve test server"); + }); + + let mut ctx = RuntimeContext { + base_url: format!("http://{addr}"), + http: reqwest::Client::new(), + agent_id: Some("agent".to_string()), + agent_type: Some("test-agent".to_string()), + session_id: Some("session".to_string()), + api_key: None, + identity_tenant: "tenant".to_string(), + sandbox: temper_sandbox::runner::PersistentSandbox::new(&[("temper", "Temper", 1)]), + trajectory: None, + tenants_seen: BTreeMap::new(), + turns_recorded: 0, + trajectory_bytes: 0, + capped_warned: false, + }; + ctx.init_trajectory(); + + ctx.finalize_trajectory().await; + + assert_eq!(attempts.load(Ordering::SeqCst), 2); +} diff --git a/crates/temper-mcp/src/trajectory_bounds.rs b/crates/temper-mcp/src/trajectory_bounds.rs new file mode 100644 index 000000000..defe0f007 --- /dev/null +++ b/crates/temper-mcp/src/trajectory_bounds.rs @@ -0,0 +1,281 @@ +//! Capture bounds and stdio framing for MCP trajectory recording (ARN-222). +//! +//! These are the pure, side-effect-free guards that keep an attacker-controlled +//! MCP session from growing the recorded trajectory or the stdio read buffer +//! without bound. They are split out of `runtime.rs` so the bounds (and their +//! tests) live in one auditable place. + +use serde_json::Value; +use std::collections::BTreeMap; +use tokio::io::{self, AsyncBufRead, AsyncBufReadExt, AsyncReadExt}; + +/// Per-message cap on recorded trajectory text (submitted code / execution result). +pub(crate) const MAX_TRAJECTORY_TEXT_BYTES: usize = 16 * 1024; +/// Cap on the number of turns recorded in a single trajectory. +pub(crate) const MAX_TRAJECTORY_TURNS: usize = 500; +/// Cap on the number of extracted actions embedded in a single turn's decision. +pub(crate) const MAX_TRAJECTORY_ACTIONS: usize = 64; +/// Cap on the number of distinct tenants / entity types tracked per session. +pub(crate) const MAX_SEEN_KEYS: usize = 256; +/// Cap on the byte length of a single tracked observability key (tenant / entity +/// type). Distinct-key *count* alone does not bound memory: 256 near-1-MiB keys +/// would still retain hundreds of MiB, so an oversized key is dropped rather than +/// stored (ARN-222). +pub(crate) const MAX_SEEN_KEY_BYTES: usize = 256; +/// Cap on the total *serialized* JSON size of the trajectory, kept under the +/// server's 2 MiB ingest limit (`POST /api/ots/trajectories` uses Axum's default +/// `DEFAULT_LIMIT` of 2_097_152) so a session that is within the per-turn and +/// per-count caps cannot still produce a trajectory the server rejects with 413 — +/// which the client treats as non-retryable and would silently drop, suppressing +/// the audit trail (ARN-222). +/// +/// The budget is metered against a conservative estimate of each turn's serialized +/// contribution (JSON-escaped text, embedded actions, a duplicated error field, +/// and a fixed per-turn envelope), not raw string bytes — raw bytes undercount the +/// wire size by 2x or more once JSON escaping and the OTS envelope are included. +pub(crate) const MAX_TRAJECTORY_TOTAL_BYTES: usize = 1_800_000; + +/// Conservative fixed overhead charged to each recorded turn's budget: the OTS +/// turn/message/decision envelope (ids, timestamps, roles, choice label, decision +/// wrapper). Deliberately larger than the real ~600–800 B so metering never +/// undercounts the serialized turn. +pub(crate) const TRAJECTORY_TURN_ENVELOPE_BYTES: usize = 2048; + +/// The number of bytes `text` occupies once serialized as a JSON string value, +/// including surrounding quotes and escaping. Used to meter a turn's real wire +/// cost against the ingest budget rather than its raw byte length. +pub(crate) fn json_string_cost(text: &str) -> usize { + serde_json::to_string(text) + .map(|s| s.len()) + .unwrap_or(text.len() + 2) +} + +/// The number of bytes `value` occupies once serialized as JSON. +pub(crate) fn json_value_cost(value: &Value) -> usize { + serde_json::to_string(value).map(|s| s.len()).unwrap_or(0) +} +/// Maximum bytes accepted for a single stdio JSON-RPC frame. A peer that never +/// sends a newline would otherwise make a line reader buffer the whole stream +/// into one allocation. +pub(crate) const MAX_STDIO_LINE_BYTES: usize = 1_048_576; + +/// Bound the `trajectory_actions` embedded in a turn's decision. The actions are +/// parsed from the full (untruncated) code, so both their count and total +/// serialized size are capped, replacing the array with a summary when it would +/// exceed the text budget — otherwise a large `temper.action(params=...)` dict +/// bypasses the per-message cap every turn (ARN-222). +pub(crate) fn bounded_trajectory_actions(mut actions: Vec) -> Value { + actions.truncate(MAX_TRAJECTORY_ACTIONS); + let value = Value::Array(actions); + let too_large = serde_json::to_string(&value) + .map(|s| s.len() > MAX_TRAJECTORY_TEXT_BYTES) + .unwrap_or(true); + if too_large { + return serde_json::json!({ + "trajectory_actions_truncated": true, + "note": "actions omitted: serialized size exceeded the trajectory cap", + }); + } + value +} + +/// The tenant a captured trajectory is stored under. It must be the session's +/// authenticated `identity` — never a tenant derived from the executed code, +/// which is attacker-controlled and would let a session file its trajectory under +/// another tenant (ARN-222). +pub(crate) fn trajectory_storage_tenant(identity: &str) -> &str { + identity +} + +/// Increment `key`'s count in a per-session observability map, bounding both the +/// number of distinct keys (`MAX_SEEN_KEYS`) and each key's byte length +/// (`MAX_SEEN_KEY_BYTES`). Returns `true` only when a new key was inserted, so a +/// caller can emit a once-per-key signal without re-firing after the map +/// saturates. +pub(crate) fn bump_seen(map: &mut BTreeMap, key: String) -> bool { + if let Some(count) = map.get_mut(&key) { + *count += 1; + false + } else if key.len() <= MAX_SEEN_KEY_BYTES && map.len() < MAX_SEEN_KEYS { + map.insert(key, 1); + true + } else { + false + } +} + +/// Largest byte index `<= max` that is a UTF-8 char boundary of `s`, so +/// `&s[..floor_char_boundary(s, max)]` never panics on a multibyte char. +pub(crate) fn floor_char_boundary(s: &str, max: usize) -> usize { + let mut end = max.min(s.len()); + while end > 0 && !s.is_char_boundary(end) { + end -= 1; + } + end +} + +/// Truncate recorded trajectory text to `MAX_TRAJECTORY_TEXT_BYTES`, appending an +/// explicit marker, so a large submitted code blob or result can't grow the +/// trajectory without bound (ARN-222). +pub(crate) fn truncate_trajectory_text(text: &str) -> String { + if text.len() <= MAX_TRAJECTORY_TEXT_BYTES { + return text.to_string(); + } + let end = floor_char_boundary(text, MAX_TRAJECTORY_TEXT_BYTES); + format!("{}…[truncated {} bytes]", &text[..end], text.len() - end) +} + +/// Outcome of reading one newline-delimited frame from stdin. +pub(crate) enum StdioFrame { + /// A complete frame within budget (may include the trailing newline). + Line(Vec), + /// The frame exceeded `MAX_STDIO_LINE_BYTES`; it was drained and dropped. + TooLarge, + /// End of input. + Eof, +} + +/// Read one JSON-RPC frame, holding at most `MAX_STDIO_LINE_BYTES + 1` bytes at a +/// time. +/// +/// `#365` flagged unbounded stdio reads but checked the line length only *after* +/// `next_line()` had already buffered the whole line, so it did not bound memory. +/// This reads through a `take`-limited reader so no single read allocates past the +/// budget, and drains an oversized frame to the next newline (reusing the same +/// buffer, not a second one) so peak memory stays near one budget rather than two. +pub(crate) async fn read_stdio_frame( + reader: &mut R, +) -> io::Result { + let mut buf = Vec::new(); + let read = reader + .take((MAX_STDIO_LINE_BYTES as u64) + 1) + .read_until(b'\n', &mut buf) + .await?; + if read == 0 { + return Ok(StdioFrame::Eof); + } + // A complete frame ends with '\n'. Missing a trailing newline is oversized + // only when we also consumed the full budget (`buf.len() > cap`); a shorter + // no-newline read is the final EOF-terminated frame and is kept. This keeps a + // frame of exactly `MAX_STDIO_LINE_BYTES` content bytes plus its newline + // (buf.len == cap + 1, ends in '\n') as a valid line rather than rejecting it. + if buf.last() != Some(&b'\n') && buf.len() > MAX_STDIO_LINE_BYTES { + // Drain the rest of this oversized frame (bounded per chunk, reusing the + // buffer) before rejecting it, so the loop resynchronizes on the next + // frame without holding a second budget's worth of memory. + loop { + buf.clear(); + let n = reader + .take((MAX_STDIO_LINE_BYTES as u64) + 1) + .read_until(b'\n', &mut buf) + .await?; + if n == 0 || buf.last() == Some(&b'\n') { + break; + } + } + return Ok(StdioFrame::TooLarge); + } + Ok(StdioFrame::Line(buf)) +} + +#[cfg(test)] +mod tests { + use super::*; + use tokio::io::BufReader; + + fn frame_label(frame: &StdioFrame) -> &'static str { + match frame { + StdioFrame::Line(_) => "Line", + StdioFrame::TooLarge => "TooLarge", + StdioFrame::Eof => "Eof", + } + } + + #[tokio::test] + async fn stdio_frame_reads_normal_line() { + let data = b"{\"jsonrpc\":\"2.0\"}\n".to_vec(); + let mut reader = BufReader::new(&data[..]); + match read_stdio_frame(&mut reader).await.unwrap() { + StdioFrame::Line(buf) => assert_eq!(buf, data), + other => panic!("expected a line, got {}", frame_label(&other)), + } + assert!(matches!( + read_stdio_frame(&mut reader).await.unwrap(), + StdioFrame::Eof + )); + } + + #[tokio::test] + async fn stdio_frame_rejects_oversized_and_resyncs() { + let mut data = vec![b'x'; MAX_STDIO_LINE_BYTES + 10]; + data.push(b'\n'); + data.extend_from_slice(b"{\"ok\":true}\n"); + let mut reader = BufReader::new(&data[..]); + + assert!( + matches!( + read_stdio_frame(&mut reader).await.unwrap(), + StdioFrame::TooLarge + ), + "oversized frame must be rejected" + ); + match read_stdio_frame(&mut reader).await.unwrap() { + StdioFrame::Line(buf) => assert_eq!(buf, b"{\"ok\":true}\n"), + other => panic!("expected the following frame, got {}", frame_label(&other)), + } + } + + #[tokio::test] + async fn stdio_frame_accepts_exact_budget_with_newline() { + let mut data = vec![b'a'; MAX_STDIO_LINE_BYTES]; + data.push(b'\n'); + let mut reader = BufReader::new(&data[..]); + match read_stdio_frame(&mut reader).await.unwrap() { + StdioFrame::Line(buf) => assert_eq!(buf.len(), MAX_STDIO_LINE_BYTES + 1), + other => panic!( + "exact-budget frame must be a line, got {}", + frame_label(&other) + ), + } + } + + #[tokio::test] + async fn stdio_frame_keeps_final_line_without_newline() { + let data = b"{\"final\":true}".to_vec(); + let mut reader = BufReader::new(&data[..]); + match read_stdio_frame(&mut reader).await.unwrap() { + StdioFrame::Line(buf) => assert_eq!(buf, data), + other => panic!( + "EOF-terminated frame must be a line, got {}", + frame_label(&other) + ), + } + } + + #[test] + fn bump_seen_bounds_count_and_key_length() { + let mut map = BTreeMap::new(); + assert!(bump_seen(&mut map, "a".to_string()), "first insert is new"); + assert!(!bump_seen(&mut map, "a".to_string()), "repeat is not new"); + assert_eq!(map.get("a"), Some(&2)); + + // Oversized key is dropped, not stored. + let huge = "x".repeat(MAX_SEEN_KEY_BYTES + 1); + assert!(!bump_seen(&mut map, huge.clone())); + assert!(!map.contains_key(&huge)); + + // Distinct-key count is capped. + for i in 0..MAX_SEEN_KEYS + 10 { + bump_seen(&mut map, format!("k{i}")); + } + assert!(map.len() <= MAX_SEEN_KEYS); + } + + #[test] + fn truncate_trajectory_text_is_multibyte_safe() { + let s = "日本語".repeat(20_000); // well over the byte budget + let out = truncate_trajectory_text(&s); + assert!(std::str::from_utf8(out.as_bytes()).is_ok()); + assert!(out.contains("[truncated")); + } +} diff --git a/crates/temper-observe/src/wide_event/runtime.rs b/crates/temper-observe/src/wide_event/runtime.rs index f6e7dad79..a764bf84b 100644 --- a/crates/temper-observe/src/wide_event/runtime.rs +++ b/crates/temper-observe/src/wide_event/runtime.rs @@ -74,6 +74,11 @@ pub struct AuthzDecisionInput<'a> { pub action: &'a str, /// Resource type being authorized. pub resource_type: &'a str, + /// Identifier of the resource instance being authorized. + /// + /// Empty when the decision is not scoped to a single instance (for example + /// a `list` over a whole entity set). + pub entity_id: &'a str, /// Kind of principal (user, admin, system). pub principal_kind: &'a str, /// Decision outcome ("Allow" or "Deny"). @@ -82,9 +87,17 @@ pub struct AuthzDecisionInput<'a> { pub duration_ns: u64, /// Tenant identifier. pub tenant: &'a str, + /// Trace ID for correlation with the request that triggered the check. + /// + /// Empty when no sampled trace context is active. + pub trace_id: &'a str, } /// Build a WideEvent from a Cedar authorization decision. +/// +/// `entity_id` and `trace_id` come from the caller: without them an authz +/// decision cannot be joined to the request or the entity it governed, which +/// leaves every denial unattributable in the trace view. pub fn from_authz_decision(input: AuthzDecisionInput<'_>) -> WideEvent { let span_id = new_span_id(); let mut tags = BTreeMap::new(); @@ -98,6 +111,9 @@ pub fn from_authz_decision(input: AuthzDecisionInput<'_>) -> WideEvent { serde_json::json!(input.principal_kind), ); attributes.insert("tenant".into(), serde_json::json!(input.tenant)); + // entity_id stays an attribute, never a tag: it is high-cardinality and + // would multiply the metric series (the cost decoupling in this module). + attributes.insert("entity_id".into(), serde_json::json!(input.entity_id)); let mut measurements = BTreeMap::new(); measurements.insert("decision_count".into(), 1.0); @@ -106,14 +122,14 @@ pub fn from_authz_decision(input: AuthzDecisionInput<'_>) -> WideEvent { WideEvent { event_kind: EventKind::AuthzDecision, entity_type: input.resource_type.into(), - entity_id: String::new(), + entity_id: input.entity_id.into(), operation: input.action.into(), from_status: String::new(), to_status: String::new(), success: input.decision == "Allow", duration_ns: input.duration_ns, timestamp: event_timestamp(), - trace_id: String::new(), + trace_id: input.trace_id.into(), span_id, tags, attributes, diff --git a/crates/temper-observe/src/wide_event/tests/mod.rs b/crates/temper-observe/src/wide_event/tests/mod.rs index 26a7889de..462474cda 100644 --- a/crates/temper-observe/src/wide_event/tests/mod.rs +++ b/crates/temper-observe/src/wide_event/tests/mod.rs @@ -116,10 +116,12 @@ fn test_authz_decision_event() { let event = from_authz_decision(AuthzDecisionInput { action: "SubmitOrder", resource_type: "Order", + entity_id: "order-123", principal_kind: "user", decision: "Allow", duration_ns: 500_000, tenant: "tenant-b", + trace_id: "4bf92f3577b34da6a3ce929d0e0e4736", }); assert_eq!(event.event_kind, EventKind::AuthzDecision); assert_eq!(event.tags["decision"], "Allow"); @@ -128,15 +130,57 @@ fn test_authz_decision_event() { assert_eq!(event.attributes["principal_kind"], "user"); } +#[test] +fn test_authz_decision_carries_entity_and_trace_correlation() { + let event = from_authz_decision(AuthzDecisionInput { + action: "SubmitOrder", + resource_type: "Order", + entity_id: "order-123", + principal_kind: "user", + decision: "Allow", + duration_ns: 500_000, + tenant: "tenant-b", + trace_id: "4bf92f3577b34da6a3ce929d0e0e4736", + }); + + assert_eq!(event.entity_id, "order-123"); + assert_eq!(event.trace_id, "4bf92f3577b34da6a3ce929d0e0e4736"); + assert_eq!(event.attributes["entity_id"], "order-123"); + // entity_id must stay out of metric tags — it is high-cardinality. + assert!(!event.tags.contains_key("entity_id")); +} + +#[test] +fn test_authz_decision_without_instance_or_trace_context() { + // Set-wide checks (e.g. `list`) and unsampled requests have no instance id + // or trace id; the event must still build rather than fabricate values. + let event = from_authz_decision(AuthzDecisionInput { + action: "list", + resource_type: "Order", + entity_id: "", + principal_kind: "user", + decision: "Allow", + duration_ns: 100_000, + tenant: "tenant-b", + trace_id: "", + }); + + assert!(event.entity_id.is_empty()); + assert!(event.trace_id.is_empty()); + assert_eq!(event.attributes["entity_id"], ""); +} + #[test] fn test_authz_deny_decision() { let event = from_authz_decision(AuthzDecisionInput { action: "DeleteOrder", resource_type: "Order", + entity_id: "order-123", principal_kind: "user", decision: "Deny", duration_ns: 800_000, tenant: "tenant-b", + trace_id: "4bf92f3577b34da6a3ce929d0e0e4736", }); assert!(!event.success); } @@ -309,10 +353,12 @@ fn test_emit_span_all_event_kinds() { from_authz_decision(AuthzDecisionInput { action: "a", resource_type: "T", + entity_id: "id", principal_kind: "user", decision: "Allow", duration_ns: 0, tenant: "t", + trace_id: "", }), from_invariant_check(InvariantCheckInput { invariant_name: "inv", diff --git a/crates/temper-odata/src/query/filter.rs b/crates/temper-odata/src/query/filter.rs index d379fca1f..375ff03d6 100644 --- a/crates/temper-odata/src/query/filter.rs +++ b/crates/temper-odata/src/query/filter.rs @@ -5,10 +5,20 @@ use super::types::{BinaryOperator, FilterExpr, ODataValue, UnaryOperator}; use crate::error::ODataError; +mod budget; + +use budget::FilterBudget; +#[cfg(test)] +use budget::{ + FILTER_ARGUMENT_BUDGET, FILTER_INPUT_BYTE_BUDGET, FILTER_LITERAL_BYTE_BUDGET, + FILTER_NODE_BUDGET, FILTER_OPERATOR_BUDGET, FILTER_TOKEN_BUDGET, +}; + /// Parse a `$filter` expression string into a [`FilterExpr`] AST. pub fn parse_filter(input: &str) -> Result { - let tokens = tokenize_filter(input)?; - let mut parser = FilterParser::new(&tokens); + let mut budget = FilterBudget::new(input)?; + let tokens = tokenize_filter(input, &mut budget)?; + let mut parser = FilterParser::new(&tokens, budget); let expr = parser.parse_or()?; // Make sure we consumed everything @@ -33,7 +43,7 @@ struct Token { offset: usize, } -fn tokenize_filter(input: &str) -> Result, ODataError> { +fn tokenize_filter(input: &str, budget: &mut FilterBudget) -> Result, ODataError> { let mut tokens = Vec::new(); let chars: Vec = input.chars().collect(); let mut i = 0; @@ -51,6 +61,7 @@ fn tokenize_filter(input: &str) -> Result, ODataError> { if chars[i] == '\'' { i += 1; let mut s = String::new(); + let mut closed = false; while i < chars.len() { if chars[i] == '\'' { // Check for escaped quote '' @@ -59,6 +70,7 @@ fn tokenize_filter(input: &str) -> Result, ODataError> { i += 2; } else { i += 1; + closed = true; break; } } else { @@ -66,19 +78,22 @@ fn tokenize_filter(input: &str) -> Result, ODataError> { i += 1; } } - tokens.push(Token { - text: format!("'{s}'"), - offset, - }); + // An unterminated literal (no closing quote) must be rejected rather + // than silently accepted as if it were closed. + if !closed { + return Err(ODataError::InvalidFilter { + message: "unterminated string literal".into(), + position: offset, + }); + } + budget.consume_literal_bytes(s.len(), offset)?; + push_token(&mut tokens, budget, format!("'{s}'"), offset)?; continue; } // Parentheses and comma if chars[i] == '(' || chars[i] == ')' || chars[i] == ',' { - tokens.push(Token { - text: chars[i].to_string(), - offset, - }); + push_token(&mut tokens, budget, chars[i].to_string(), offset)?; i += 1; continue; } @@ -96,12 +111,18 @@ fn tokenize_filter(input: &str) -> Result, ODataError> { num.push(chars[i]); i += 1; } - tokens.push(Token { text: num, offset }); + push_token(&mut tokens, budget, num, offset)?; continue; } - // Identifiers and keywords (including dotted names like 'guid', property paths) - if chars[i].is_ascii_alphabetic() || chars[i] == '_' || chars[i] == '$' { + // Identifiers and keywords (dotted names like 'guid', property paths). + // + // '$' is deliberately NOT an identifier-start character. It is not valid + // in a property path, and accepting it here while the loop below does not + // consume it previously spun forever without advancing `i` — a + // denial-of-service on any `$filter` value containing a bare '$'. A stray + // '$' now falls through to the "unexpected character" error below (400). + if chars[i].is_ascii_alphabetic() || chars[i] == '_' { let mut word = String::new(); while i < chars.len() && (chars[i].is_ascii_alphanumeric() @@ -110,14 +131,14 @@ fn tokenize_filter(input: &str) -> Result, ODataError> { || chars[i] == '/' || chars[i] == '-') { - // Stop at '.' if it looks like it's followed by whitespace or end - // (to not consume e.g. "Name eq" as "Name.eq"). Actually, dots - // are used in property paths like Address/City and in GUIDs. - // Let's keep consuming alphanumeric, underscore, dot, slash, and hyphen. word.push(chars[i]); i += 1; } - tokens.push(Token { text: word, offset }); + // Invariant: the identifier-start set (alphabetic | '_') is a subset + // of the continue set above, so the loop always consumes at least one + // character. This guarantees the tokenizer makes forward progress. + debug_assert!(i > offset, "tokenizer failed to advance on identifier"); + push_token(&mut tokens, budget, word, offset)?; continue; } @@ -130,16 +151,73 @@ fn tokenize_filter(input: &str) -> Result, ODataError> { Ok(tokens) } +fn push_token( + tokens: &mut Vec, + budget: &mut FilterBudget, + text: String, + offset: usize, +) -> Result<(), ODataError> { + budget.consume_token(offset)?; + tokens.push(Token { text, offset }); + Ok(()) +} + // -- Recursive descent parser ------------------------------------------------ +/// Nesting-depth budget for a `$filter` expression. +/// +/// The recursive-descent parser descends one level per parenthesized +/// sub-expression, `not` operand, and function-call argument. Without a bound, a +/// crafted filter such as `((((…))))` or `not not not …` would recurse until the +/// request thread's stack overflows and the process aborts — a denial-of-service +/// reachable from the public OData query surface. A well-formed query never +/// nests anywhere near this deep. +// Keep ample headroom below the request worker's stack: filter canaries +// exercise the accepted boundary on 512 KiB, rather than merely fitting a +// default 2 MiB worker before the surrounding request frames are considered. +const FILTER_DEPTH_BUDGET: usize = 32; + struct FilterParser<'a> { tokens: &'a [Token], pos: usize, + /// Current recursion depth, bounded by [`FILTER_DEPTH_BUDGET`]. + depth: usize, + budget: FilterBudget, } impl<'a> FilterParser<'a> { - fn new(tokens: &'a [Token]) -> Self { - Self { tokens, pos: 0 } + fn new(tokens: &'a [Token], budget: FilterBudget) -> Self { + Self { + tokens, + pos: 0, + depth: 0, + budget, + } + } + + /// Enter one nesting level, rejecting filters that nest past + /// [`FILTER_DEPTH_BUDGET`] before the recursion can overflow the stack. + /// + /// Every recursive descent (parentheses, `not`, function arguments) must be + /// wrapped in a matching [`descend`](Self::descend)/[`ascend`](Self::ascend) + /// pair so that width (many siblings) does not count as depth. + fn descend(&mut self) -> Result<(), ODataError> { + if self.depth == FILTER_DEPTH_BUDGET { + return Err(ODataError::InvalidFilter { + message: format!( + "filter expression nesting exceeds depth budget of {FILTER_DEPTH_BUDGET}" + ), + position: self.current_offset(), + }); + } + self.depth += 1; + Ok(()) + } + + /// Leave a nesting level previously entered via [`descend`](Self::descend). + fn ascend(&mut self) { + assert!(self.depth > 0, "filter parser depth underflow"); + self.depth -= 1; } fn peek(&self) -> Option<&Token> { @@ -176,39 +254,41 @@ impl<'a> FilterParser<'a> { // or_expr = and_expr ( 'or' and_expr )* fn parse_or(&mut self) -> Result { - let mut left = self.parse_and()?; + let mut operands = vec![self.parse_and()?]; while self.peek_text_is("or") { + let operator_offset = self.current_offset(); + self.budget.consume_operator(operator_offset)?; + self.budget.consume_node(operator_offset)?; self.advance(); - let right = self.parse_and()?; - left = FilterExpr::BinaryOp { - left: Box::new(left), - op: BinaryOperator::Or, - right: Box::new(right), - }; + operands.push(self.parse_and()?); } - Ok(left) + Ok(balance_associative(operands, BinaryOperator::Or)) } // and_expr = not_expr ( 'and' not_expr )* fn parse_and(&mut self) -> Result { - let mut left = self.parse_not()?; + let mut operands = vec![self.parse_not()?]; while self.peek_text_is("and") { + let operator_offset = self.current_offset(); + self.budget.consume_operator(operator_offset)?; + self.budget.consume_node(operator_offset)?; self.advance(); - let right = self.parse_not()?; - left = FilterExpr::BinaryOp { - left: Box::new(left), - op: BinaryOperator::And, - right: Box::new(right), - }; + operands.push(self.parse_not()?); } - Ok(left) + Ok(balance_associative(operands, BinaryOperator::And)) } // not_expr = 'not' not_expr | comparison fn parse_not(&mut self) -> Result { if self.peek_text_is("not") { + let operator_offset = self.current_offset(); + self.budget.consume_operator(operator_offset)?; + self.budget.consume_node(operator_offset)?; self.advance(); - let operand = self.parse_not()?; + self.descend()?; + let operand = self.parse_not(); + self.ascend(); + let operand = operand?; return Ok(FilterExpr::UnaryOp { op: UnaryOperator::Not, operand: Box::new(operand), @@ -221,6 +301,9 @@ impl<'a> FilterParser<'a> { fn parse_comparison(&mut self) -> Result { let left = self.parse_primary()?; if let Some(op) = self.peek_comparison_op() { + let operator_offset = self.current_offset(); + self.budget.consume_operator(operator_offset)?; + self.budget.consume_node(operator_offset)?; self.advance(); let right = self.parse_primary()?; Ok(FilterExpr::BinaryOp { @@ -250,20 +333,25 @@ impl<'a> FilterParser<'a> { // Parenthesized sub-expression if text == "(" { self.advance(); - let expr = self.parse_or()?; + self.descend()?; + let expr = self.parse_or(); + self.ascend(); + let expr = expr?; self.expect_text(")")?; return Ok(expr); } // String literal if text.starts_with('\'') && text.ends_with('\'') && text.len() >= 2 { - let s = text[1..text.len() - 1].to_string(); + self.budget.consume_node(offset)?; self.advance(); + let s = text[1..text.len() - 1].to_string(); return Ok(FilterExpr::Literal(ODataValue::String(s))); } // Numeric literal if text.starts_with(|c: char| c.is_ascii_digit() || c == '-') { + self.budget.consume_node(offset)?; self.advance(); if text.contains('.') { let val: f64 = text.parse().map_err(|_| ODataError::InvalidFilter { @@ -282,14 +370,17 @@ impl<'a> FilterParser<'a> { // Keywords: null, true, false if text == "null" { + self.budget.consume_node(offset)?; self.advance(); return Ok(FilterExpr::Literal(ODataValue::Null)); } if text == "true" { + self.budget.consume_node(offset)?; self.advance(); return Ok(FilterExpr::Literal(ODataValue::Boolean(true))); } if text == "false" { + self.budget.consume_node(offset)?; self.advance(); return Ok(FilterExpr::Literal(ODataValue::Boolean(false))); } @@ -301,6 +392,7 @@ impl<'a> FilterParser<'a> { // Check for function call: name followed by '(' if self.peek_text_is("(") { + self.budget.consume_node(offset)?; self.advance(); // consume '(' let args = self.parse_argument_list()?; self.expect_text(")")?; @@ -308,6 +400,7 @@ impl<'a> FilterParser<'a> { } // Otherwise it's a property reference + self.budget.consume_node(offset)?; return Ok(FilterExpr::Property(name)); } @@ -326,7 +419,11 @@ impl<'a> FilterParser<'a> { } loop { - args.push(self.parse_or()?); + self.budget.consume_argument(self.current_offset())?; + self.descend()?; + let arg = self.parse_or(); + self.ascend(); + args.push(arg?); if self.peek_text_is(",") { self.advance(); } else { @@ -359,180 +456,48 @@ impl<'a> FilterParser<'a> { } } -// --------------------------------------------------------------------------- -// Tests -// --------------------------------------------------------------------------- - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn filter_simple_eq_string() { - let expr = parse_filter("Name eq 'foo'").unwrap(); - assert_eq!( - expr, - FilterExpr::BinaryOp { - left: Box::new(FilterExpr::Property("Name".into())), - op: BinaryOperator::Eq, - right: Box::new(FilterExpr::Literal(ODataValue::String("foo".into()))), - } - ); - } - - #[test] - fn filter_simple_gt_float() { - let expr = parse_filter("Price gt 5.0").unwrap(); - assert_eq!( - expr, - FilterExpr::BinaryOp { - left: Box::new(FilterExpr::Property("Price".into())), - op: BinaryOperator::Gt, - right: Box::new(FilterExpr::Literal(ODataValue::Float(5.0))), - } - ); - } - - #[test] - fn filter_and_or_precedence() { - // `A eq 1 and B eq 2 or C eq 3` should parse as `(A eq 1 and B eq 2) or (C eq 3)` - let expr = parse_filter("A eq 1 and B eq 2 or C eq 3").unwrap(); - match &expr { - FilterExpr::BinaryOp { - op: BinaryOperator::Or, - left, - right, - } => { - // Left should be the 'and' node - match left.as_ref() { - FilterExpr::BinaryOp { - op: BinaryOperator::And, - .. - } => {} - other => panic!("expected And on left, got {other:?}"), - } - // Right should be a comparison - match right.as_ref() { - FilterExpr::BinaryOp { - op: BinaryOperator::Eq, - .. - } => {} - other => panic!("expected Eq on right, got {other:?}"), - } - } - other => panic!("expected Or at top, got {other:?}"), - } - } - - #[test] - fn filter_compound_and() { - let expr = parse_filter("Name eq 'foo' and Price gt 5.0").unwrap(); - assert_eq!( - expr, - FilterExpr::BinaryOp { - left: Box::new(FilterExpr::BinaryOp { - left: Box::new(FilterExpr::Property("Name".into())), - op: BinaryOperator::Eq, - right: Box::new(FilterExpr::Literal(ODataValue::String("foo".into()))), - }), - op: BinaryOperator::And, - right: Box::new(FilterExpr::BinaryOp { - left: Box::new(FilterExpr::Property("Price".into())), - op: BinaryOperator::Gt, - right: Box::new(FilterExpr::Literal(ODataValue::Float(5.0))), - }), +/// Build an order-preserving balanced tree for an associative boolean operator. +/// +/// A left fold makes an accepted wide filter's AST as deep as its width. Every +/// downstream consumer then inherits that attacker-controlled recursion depth, +/// including in-memory evaluation, SQL translation, and `Drop`. Pairing adjacent +/// operands keeps the same left-to-right order and boolean meaning while bounding +/// tree depth logarithmically. +fn balance_associative(mut operands: Vec, op: BinaryOperator) -> FilterExpr { + assert!( + !operands.is_empty(), + "boolean chain must contain an operand" + ); + + while operands.len() > 1 { + let mut next_level = Vec::with_capacity(operands.len().div_ceil(2)); + let mut iter = operands.into_iter(); + while let Some(left) = iter.next() { + if let Some(right) = iter.next() { + next_level.push(FilterExpr::BinaryOp { + left: Box::new(left), + op, + right: Box::new(right), + }); + } else { + next_level.push(left); } - ); - } - - #[test] - fn filter_not_operator() { - let expr = parse_filter("not Active eq true").unwrap(); - match &expr { - FilterExpr::UnaryOp { - op: UnaryOperator::Not, - operand, - } => match operand.as_ref() { - FilterExpr::BinaryOp { - op: BinaryOperator::Eq, - .. - } => {} - other => panic!("expected Eq inside not, got {other:?}"), - }, - other => panic!("expected Not at top, got {other:?}"), - } - } - - #[test] - fn filter_parenthesized_expression() { - let expr = parse_filter("(A eq 1 or B eq 2) and C eq 3").unwrap(); - match &expr { - FilterExpr::BinaryOp { - op: BinaryOperator::And, - left, - .. - } => match left.as_ref() { - FilterExpr::BinaryOp { - op: BinaryOperator::Or, - .. - } => {} - other => panic!("expected Or in parens, got {other:?}"), - }, - other => panic!("expected And at top, got {other:?}"), } + operands = next_level; } - #[test] - fn filter_function_call() { - let expr = parse_filter("contains(Name, 'foo')").unwrap(); - assert_eq!( - expr, - FilterExpr::FunctionCall { - name: "contains".into(), - args: vec![ - FilterExpr::Property("Name".into()), - FilterExpr::Literal(ODataValue::String("foo".into())), - ], - } - ); - } + debug_assert_eq!(operands.len(), 1, "balancing must produce one root"); + operands.remove(0) +} - #[test] - fn filter_null_literal() { - let expr = parse_filter("Name eq null").unwrap(); - assert_eq!( - expr, - FilterExpr::BinaryOp { - left: Box::new(FilterExpr::Property("Name".into())), - op: BinaryOperator::Eq, - right: Box::new(FilterExpr::Literal(ODataValue::Null)), - } - ); - } +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- - #[test] - fn filter_boolean_literal() { - let expr = parse_filter("Active eq true").unwrap(); - assert_eq!( - expr, - FilterExpr::BinaryOp { - left: Box::new(FilterExpr::Property("Active".into())), - op: BinaryOperator::Eq, - right: Box::new(FilterExpr::Literal(ODataValue::Boolean(true))), - } - ); - } +#[cfg(test)] +#[path = "filter/basic_tests.rs"] +mod basic_tests; - #[test] - fn filter_negative_number() { - let expr = parse_filter("Amount gt -10").unwrap(); - assert_eq!( - expr, - FilterExpr::BinaryOp { - left: Box::new(FilterExpr::Property("Amount".into())), - op: BinaryOperator::Gt, - right: Box::new(FilterExpr::Literal(ODataValue::Int(-10))), - } - ); - } -} +#[cfg(test)] +#[path = "filter/security_tests.rs"] +mod security_tests; diff --git a/crates/temper-odata/src/query/filter/basic_tests.rs b/crates/temper-odata/src/query/filter/basic_tests.rs new file mode 100644 index 000000000..4762355cb --- /dev/null +++ b/crates/temper-odata/src/query/filter/basic_tests.rs @@ -0,0 +1,170 @@ +use super::*; + +#[test] +fn filter_simple_eq_string() { + let expr = parse_filter("Name eq 'foo'").unwrap(); + assert_eq!( + expr, + FilterExpr::BinaryOp { + left: Box::new(FilterExpr::Property("Name".into())), + op: BinaryOperator::Eq, + right: Box::new(FilterExpr::Literal(ODataValue::String("foo".into()))), + } + ); +} + +#[test] +fn filter_simple_gt_float() { + let expr = parse_filter("Price gt 5.0").unwrap(); + assert_eq!( + expr, + FilterExpr::BinaryOp { + left: Box::new(FilterExpr::Property("Price".into())), + op: BinaryOperator::Gt, + right: Box::new(FilterExpr::Literal(ODataValue::Float(5.0))), + } + ); +} + +#[test] +fn filter_and_or_precedence() { + // `A eq 1 and B eq 2 or C eq 3` should parse as `(A eq 1 and B eq 2) or (C eq 3)` + let expr = parse_filter("A eq 1 and B eq 2 or C eq 3").unwrap(); + match &expr { + FilterExpr::BinaryOp { + op: BinaryOperator::Or, + left, + right, + } => { + // Left should be the 'and' node + match left.as_ref() { + FilterExpr::BinaryOp { + op: BinaryOperator::And, + .. + } => {} + other => panic!("expected And on left, got {other:?}"), + } + // Right should be a comparison + match right.as_ref() { + FilterExpr::BinaryOp { + op: BinaryOperator::Eq, + .. + } => {} + other => panic!("expected Eq on right, got {other:?}"), + } + } + other => panic!("expected Or at top, got {other:?}"), + } +} + +#[test] +fn filter_compound_and() { + let expr = parse_filter("Name eq 'foo' and Price gt 5.0").unwrap(); + assert_eq!( + expr, + FilterExpr::BinaryOp { + left: Box::new(FilterExpr::BinaryOp { + left: Box::new(FilterExpr::Property("Name".into())), + op: BinaryOperator::Eq, + right: Box::new(FilterExpr::Literal(ODataValue::String("foo".into()))), + }), + op: BinaryOperator::And, + right: Box::new(FilterExpr::BinaryOp { + left: Box::new(FilterExpr::Property("Price".into())), + op: BinaryOperator::Gt, + right: Box::new(FilterExpr::Literal(ODataValue::Float(5.0))), + }), + } + ); +} + +#[test] +fn filter_not_operator() { + let expr = parse_filter("not Active eq true").unwrap(); + match &expr { + FilterExpr::UnaryOp { + op: UnaryOperator::Not, + operand, + } => match operand.as_ref() { + FilterExpr::BinaryOp { + op: BinaryOperator::Eq, + .. + } => {} + other => panic!("expected Eq inside not, got {other:?}"), + }, + other => panic!("expected Not at top, got {other:?}"), + } +} + +#[test] +fn filter_parenthesized_expression() { + let expr = parse_filter("(A eq 1 or B eq 2) and C eq 3").unwrap(); + match &expr { + FilterExpr::BinaryOp { + op: BinaryOperator::And, + left, + .. + } => match left.as_ref() { + FilterExpr::BinaryOp { + op: BinaryOperator::Or, + .. + } => {} + other => panic!("expected Or in parens, got {other:?}"), + }, + other => panic!("expected And at top, got {other:?}"), + } +} + +#[test] +fn filter_function_call() { + let expr = parse_filter("contains(Name, 'foo')").unwrap(); + assert_eq!( + expr, + FilterExpr::FunctionCall { + name: "contains".into(), + args: vec![ + FilterExpr::Property("Name".into()), + FilterExpr::Literal(ODataValue::String("foo".into())), + ], + } + ); +} + +#[test] +fn filter_null_literal() { + let expr = parse_filter("Name eq null").unwrap(); + assert_eq!( + expr, + FilterExpr::BinaryOp { + left: Box::new(FilterExpr::Property("Name".into())), + op: BinaryOperator::Eq, + right: Box::new(FilterExpr::Literal(ODataValue::Null)), + } + ); +} + +#[test] +fn filter_boolean_literal() { + let expr = parse_filter("Active eq true").unwrap(); + assert_eq!( + expr, + FilterExpr::BinaryOp { + left: Box::new(FilterExpr::Property("Active".into())), + op: BinaryOperator::Eq, + right: Box::new(FilterExpr::Literal(ODataValue::Boolean(true))), + } + ); +} + +#[test] +fn filter_negative_number() { + let expr = parse_filter("Amount gt -10").unwrap(); + assert_eq!( + expr, + FilterExpr::BinaryOp { + left: Box::new(FilterExpr::Property("Amount".into())), + op: BinaryOperator::Gt, + right: Box::new(FilterExpr::Literal(ODataValue::Int(-10))), + } + ); +} diff --git a/crates/temper-odata/src/query/filter/budget.rs b/crates/temper-odata/src/query/filter/budget.rs new file mode 100644 index 000000000..8768229f6 --- /dev/null +++ b/crates/temper-odata/src/query/filter/budget.rs @@ -0,0 +1,122 @@ +use crate::error::ODataError; + +/// Resource budgets for one `$filter` expression. +/// +/// These budgets bound every input-controlled dimension before the parser can +/// build an AST large enough to exhaust memory or the request thread's stack. +pub(super) const FILTER_INPUT_BYTE_BUDGET: usize = 64 * 1024; +pub(super) const FILTER_TOKEN_BUDGET: usize = 4_096; +pub(super) const FILTER_NODE_BUDGET: usize = 4_096; +pub(super) const FILTER_OPERATOR_BUDGET: usize = 1_024; +/// Total function-call arguments across the whole filter (a cumulative, not +/// per-call, budget). It bounds total argument-parsing work regardless of how the +/// calls are distributed; generous for real queries, which use only a handful of +/// function calls. +pub(super) const FILTER_ARGUMENT_BUDGET: usize = 256; +pub(super) const FILTER_LITERAL_BYTE_BUDGET: usize = 16 * 1024; + +#[derive(Debug)] +pub(super) struct FilterBudget { + tokens_remaining: usize, + nodes_remaining: usize, + operators_remaining: usize, + arguments_remaining: usize, + literal_bytes_remaining: usize, +} + +impl FilterBudget { + pub(super) fn new(input: &str) -> Result { + if input.len() > FILTER_INPUT_BYTE_BUDGET { + // Report the actual over-budget length as the position rather than the + // fixed budget, so the error says how large the rejected input was. + return Err(budget_exceeded( + "input byte", + FILTER_INPUT_BYTE_BUDGET, + input.len(), + )); + } + + Ok(Self { + tokens_remaining: FILTER_TOKEN_BUDGET, + nodes_remaining: FILTER_NODE_BUDGET, + operators_remaining: FILTER_OPERATOR_BUDGET, + arguments_remaining: FILTER_ARGUMENT_BUDGET, + literal_bytes_remaining: FILTER_LITERAL_BYTE_BUDGET, + }) + } + + pub(super) fn consume_token(&mut self, position: usize) -> Result<(), ODataError> { + consume_budget( + &mut self.tokens_remaining, + 1, + "token", + FILTER_TOKEN_BUDGET, + position, + ) + } + + pub(super) fn consume_node(&mut self, position: usize) -> Result<(), ODataError> { + consume_budget( + &mut self.nodes_remaining, + 1, + "AST node", + FILTER_NODE_BUDGET, + position, + ) + } + + pub(super) fn consume_operator(&mut self, position: usize) -> Result<(), ODataError> { + consume_budget( + &mut self.operators_remaining, + 1, + "operator", + FILTER_OPERATOR_BUDGET, + position, + ) + } + + pub(super) fn consume_argument(&mut self, position: usize) -> Result<(), ODataError> { + consume_budget( + &mut self.arguments_remaining, + 1, + "function argument", + FILTER_ARGUMENT_BUDGET, + position, + ) + } + + pub(super) fn consume_literal_bytes( + &mut self, + amount: usize, + position: usize, + ) -> Result<(), ODataError> { + consume_budget( + &mut self.literal_bytes_remaining, + amount, + "string literal byte", + FILTER_LITERAL_BYTE_BUDGET, + position, + ) + } +} + +fn consume_budget( + remaining: &mut usize, + amount: usize, + resource: &str, + allowance: usize, + position: usize, +) -> Result<(), ODataError> { + let Some(next) = remaining.checked_sub(amount) else { + return Err(budget_exceeded(resource, allowance, position)); + }; + *remaining = next; + Ok(()) +} + +fn budget_exceeded(resource: &str, allowance: usize, position: usize) -> ODataError { + ODataError::InvalidFilter { + message: format!("filter {resource} budget of {allowance} exceeded"), + position, + } +} diff --git a/crates/temper-odata/src/query/filter/security_tests.rs b/crates/temper-odata/src/query/filter/security_tests.rs new file mode 100644 index 000000000..940c4007c --- /dev/null +++ b/crates/temper-odata/src/query/filter/security_tests.rs @@ -0,0 +1,336 @@ +use super::*; + +const FILTER_TEST_STACK_BYTES: usize = 512 * 1024; + +fn assert_on_small_stack(name: &str, test: impl FnOnce() + Send + 'static) { + std::thread::Builder::new() + .name(name.into()) + .stack_size(FILTER_TEST_STACK_BYTES) + .spawn(test) + .expect("spawn constrained-stack filter test") + .join() + .expect("filter boundary must fit the constrained request stack"); +} + +// -- Security regression tests (ARN-176) --------------------------------- +// +// Denial-of-service regressions on the public OData `$filter` surface. Each +// hostile input now returns `InvalidFilter` (surfaced as HTTP 400) quickly +// instead of hanging, exhausting memory, or crashing the request thread. + +#[test] +fn filter_dollar_sign_returns_error_not_hang() { + // Pre-fix: the tokenizer accepted '$' as an identifier-start character, + // but the identifier loop never consumed it, so `i` never advanced — an + // infinite loop that also grew the token vec without bound, hanging (and + // eventually OOM-ing) the request thread. The watchdog below turns that + // hang into a test failure instead of blocking the suite forever. On a + // regression the spawned thread is intentionally left detached (it never + // returns); the harness process exits and reaps it after the failure. + use std::sync::mpsc; + use std::time::Duration; + + let (tx, rx) = mpsc::channel(); + // A stray '$' outside a string literal is the trigger. Inside quotes it + // is fine (handled by the string-literal branch). + std::thread::spawn(move || { + let _ = tx.send(parse_filter("Name eq $foo").is_err()); + }); + + match rx.recv_timeout(Duration::from_secs(5)) { + Ok(is_err) => assert!( + is_err, + "'$' in a filter value must return InvalidFilter, not succeed" + ), + Err(_) => { + panic!("parse_filter hung on '$' — tokenizer infinite loop is not fixed") + } + } +} + +#[test] +fn filter_bare_dollar_sign_is_rejected() { + // A lone '$' is an unexpected character, not an identifier. + assert!(parse_filter("$").is_err()); +} + +#[test] +fn filter_deeply_nested_parens_returns_error_not_overflow() { + // Pre-fix: unbounded recursion (parse_primary -> parse_or per '(') + // overflowed the request thread's stack and aborted the process. A 100k-deep + // input is rejected by the input-byte budget before parsing even begins (the + // outermost line of defense); the depth guard itself is exercised at its exact + // boundary by `filter_depth_bound_is_inclusive_at_the_limit`. + let depth = 100_000; + let mut input = String::with_capacity(depth * 2 + 16); + for _ in 0..depth { + input.push('('); + } + input.push_str("Name eq 'x'"); + for _ in 0..depth { + input.push(')'); + } + assert!( + parse_filter(&input).is_err(), + "deeply nested parens must return InvalidFilter, not overflow the stack" + ); +} + +#[test] +fn filter_deeply_nested_not_returns_error_not_overflow() { + // Pre-fix: `not not not …` recursed parse_not without bound and + // overflowed the stack. A 100k-deep input is rejected by the input-byte + // budget; the parse_not depth guard is exercised at its boundary by + // `filter_not_depth_bound_is_inclusive`. + let mut input = String::new(); + for _ in 0..100_000 { + input.push_str("not "); + } + input.push_str("Active eq true"); + assert!( + parse_filter(&input).is_err(), + "deeply nested 'not' must return InvalidFilter, not overflow the stack" + ); +} + +#[test] +fn filter_deeply_nested_functions_returns_error_not_overflow() { + // Pre-fix: `f(f(f(…)))` recursed parse_argument_list -> parse_or without + // bound and overflowed the stack. A 100k-deep input is rejected by the + // input-byte budget; the parse_argument_list depth guard is exercised at its + // boundary by `filter_function_depth_bound_is_inclusive`. + let depth = 100_000; + let mut input = String::with_capacity(depth * 2 + 8); + for _ in 0..depth { + input.push_str("f("); + } + input.push('1'); + for _ in 0..depth { + input.push(')'); + } + assert!( + parse_filter(&input).is_err(), + "deeply nested function calls must return InvalidFilter, not overflow" + ); +} + +#[test] +fn filter_moderately_nested_parens_still_parse() { + // Nesting well within FILTER_DEPTH_BUDGET must still parse — the depth + // bound must not reject legitimate queries. + let depth = FILTER_DEPTH_BUDGET / 2; + let mut input = String::new(); + for _ in 0..depth { + input.push('('); + } + input.push_str("Name eq 'x'"); + for _ in 0..depth { + input.push(')'); + } + assert!( + parse_filter(&input).is_ok(), + "nesting within the depth bound must still parse" + ); + + // A useful wide, shallow filter must fit the total parse budget; width + // is bounded independently from recursive depth. + let wide = (0..500) + .map(|n| format!("Id eq {n}")) + .collect::>() + .join(" or "); + assert!( + parse_filter(&wide).is_ok(), + "moderately wide filters must remain supported" + ); +} + +#[test] +fn filter_input_byte_budget_is_inclusive() { + let at_budget = "a".repeat(FILTER_INPUT_BYTE_BUDGET); + assert!(parse_filter(&at_budget).is_ok()); + + let over_budget = "a".repeat(FILTER_INPUT_BYTE_BUDGET + 1); + assert!(parse_filter(&over_budget).is_err()); +} + +#[test] +fn filter_token_budget_is_inclusive() { + let at_budget = std::iter::repeat_n("a", FILTER_TOKEN_BUDGET) + .collect::>() + .join(" "); + let mut budget = FilterBudget::new(&at_budget).unwrap(); + assert_eq!( + tokenize_filter(&at_budget, &mut budget).unwrap().len(), + FILTER_TOKEN_BUDGET + ); + + let over_budget = format!("{at_budget} a"); + let mut budget = FilterBudget::new(&over_budget).unwrap(); + assert!(tokenize_filter(&over_budget, &mut budget).is_err()); +} + +#[test] +fn filter_node_budget_is_inclusive() { + let mut budget = FilterBudget::new("").unwrap(); + for _ in 0..FILTER_NODE_BUDGET { + assert!(budget.consume_node(0).is_ok()); + } + assert!(budget.consume_node(0).is_err()); +} + +#[test] +fn filter_literal_byte_budget_is_inclusive() { + let at_budget = format!("Name eq '{}'", "x".repeat(FILTER_LITERAL_BYTE_BUDGET)); + assert!(parse_filter(&at_budget).is_ok()); + + let over_budget = format!("Name eq '{}'", "x".repeat(FILTER_LITERAL_BYTE_BUDGET + 1)); + assert!(parse_filter(&over_budget).is_err()); +} + +#[test] +fn filter_function_argument_budget_is_inclusive() { + let at_budget = format!( + "f({})", + std::iter::repeat_n("a", FILTER_ARGUMENT_BUDGET) + .collect::>() + .join(",") + ); + assert!(parse_filter(&at_budget).is_ok()); + + let over_budget = format!( + "f({})", + std::iter::repeat_n("a", FILTER_ARGUMENT_BUDGET + 1) + .collect::>() + .join(",") + ); + assert!(parse_filter(&over_budget).is_err()); +} + +#[test] +fn filter_operator_budget_is_inclusive() { + assert_eq!(FILTER_OPERATOR_BUDGET % 2, 0); + let comparisons = (0..FILTER_OPERATOR_BUDGET / 2) + .map(|n| format!("Id eq {n}")) + .collect::>() + .join(" or "); + assert!(parse_filter(&format!("not {comparisons}")).is_ok()); + assert!(parse_filter(&format!("not not {comparisons}")).is_err()); +} + +#[test] +fn filter_budgeted_wide_ast_parses_and_drops_on_small_stack() { + let input = (0..FILTER_OPERATOR_BUDGET / 2) + .map(|n| format!("Id eq {n}")) + .collect::>() + .join(" or "); + + assert_on_small_stack("filter-wide-boundary", move || { + let expr = parse_filter(&input).expect("expression must fit the budget"); + drop(expr); + }); +} + +#[test] +fn filter_mixed_depth_and_width_fits_small_stack() { + let wide = (0..FILTER_OPERATOR_BUDGET / 2) + .map(|n| format!("Id eq {n}")) + .collect::>() + .join(" or "); + let input = format!( + "{}{}{}", + "(".repeat(FILTER_DEPTH_BUDGET), + wide, + ")".repeat(FILTER_DEPTH_BUDGET) + ); + + assert_on_small_stack("filter-mixed-budget-boundary", move || { + let expr = parse_filter(&input).expect("mixed boundary expression must parse"); + drop(expr); + }); +} + +#[test] +fn filter_wide_ast_over_budget_returns_error() { + let input = (0..40_000) + .map(|n| format!("Id eq {n}")) + .collect::>() + .join(" or "); + assert!(parse_filter(&input).is_err()); +} + +#[test] +fn filter_unterminated_string_returns_error() { + // Pre-fix: an unterminated literal was silently accepted as if closed. + assert!(parse_filter("Name eq 'foo").is_err()); +} + +#[test] +fn filter_depth_bound_is_inclusive_at_the_limit() { + // Exercise the exact boundary so an off-by-one (e.g. flipping `>` to + // `>=`) is caught: FILTER_DEPTH_BUDGET levels of nesting must parse, and one + // level deeper must be rejected. + let nest = |levels: usize| { + let mut s = String::with_capacity(levels * 2 + 16); + for _ in 0..levels { + s.push('('); + } + s.push_str("Name eq 'x'"); + for _ in 0..levels { + s.push(')'); + } + s + }; + assert_on_small_stack("filter-depth-boundary", move || { + assert!( + parse_filter(&nest(FILTER_DEPTH_BUDGET)).is_ok(), + "exactly FILTER_DEPTH_BUDGET levels must be accepted" + ); + assert!( + parse_filter(&nest(FILTER_DEPTH_BUDGET + 1)).is_err(), + "one level past FILTER_DEPTH_BUDGET must be rejected" + ); + }); +} + +#[test] +fn filter_not_depth_bound_is_inclusive() { + // Discriminating boundary test for the `parse_not` recursion guard: these + // inputs are tiny (well under the input-byte budget), so rejection can only + // come from `descend()` in parse_not. Removing that guard would make the + // over-limit case parse (and could overflow the stack) instead of erroring. + let nest_not = |levels: usize| format!("{}Active eq true", "not ".repeat(levels)); + assert!( + parse_filter(&nest_not(FILTER_DEPTH_BUDGET)).is_ok(), + "FILTER_DEPTH_BUDGET `not` levels must parse" + ); + assert!( + parse_filter(&nest_not(FILTER_DEPTH_BUDGET + 1)).is_err(), + "one `not` past the depth budget must be rejected by the depth guard" + ); +} + +#[test] +fn filter_function_depth_bound_is_inclusive() { + // Discriminating boundary test for the `parse_argument_list` recursion guard. + // Nested single-argument function calls recurse through parse_or once per + // level; the input stays tiny, so only `descend()` can reject it. + let nest_fn = |levels: usize| { + let mut s = String::with_capacity(levels * 2 + 4); + for _ in 0..levels { + s.push_str("f("); + } + s.push('1'); + for _ in 0..levels { + s.push(')'); + } + s + }; + assert!( + parse_filter(&nest_fn(FILTER_DEPTH_BUDGET)).is_ok(), + "FILTER_DEPTH_BUDGET nested function levels must parse" + ); + assert!( + parse_filter(&nest_fn(FILTER_DEPTH_BUDGET + 1)).is_err(), + "one nested function past the depth budget must be rejected by the depth guard" + ); +} diff --git a/crates/temper-ots/src/atif/mod.rs b/crates/temper-ots/src/atif/mod.rs new file mode 100644 index 000000000..580491950 --- /dev/null +++ b/crates/temper-ots/src/atif/mod.rs @@ -0,0 +1,497 @@ +//! Export an [`OTSTrajectory`] as an ATIF v1.7 document. +//! +//! ATIF (Agent Trajectory Interchange Format) is the Harbor project's +//! interchange format for agent trajectories, specified in +//! `rfcs/0001-trajectory-format.md` of `github.com/harbor-framework/harbor`. +//! Exporting to it makes Temper trajectories readable by the SFT, RL, and +//! replay tooling built around that format. +//! +//! # Shape of the mapping +//! +//! ATIF is a flat list of steps; OTS is a list of turns, each holding its own +//! messages and decisions. One OTS turn becomes at most one ATIF **agent** +//! step, preceded by one **user** or **system** step per user/system message +//! inside that turn. The trajectory's `system_message` becomes the first step. +//! +//! # Mapping table +//! +//! ## Lossless — carried in ATIF-native fields +//! +//! | OTS | ATIF | +//! | --- | --- | +//! | `trajectory_id` | `trajectory_id` | +//! | `system_message.content` / `.timestamp` | leading step, `source = "system"` | +//! | `turns[].timestamp` | `steps[].timestamp` | +//! | user-role message text | step with `source = "user"` | +//! | system-role message text | step with `source = "system"` | +//! | assistant-role text messages | agent step `message` (joined with newlines) | +//! | assistant `reasoning` | `steps[].reasoning_content` | +//! | invoking `decisions[].choice.action` | `tool_calls[].function_name` | +//! | invoking `decisions[].choice.arguments` | `tool_calls[].arguments` | +//! | invoking `decisions[].cause_id` (else `decision_id`) | `tool_calls[].tool_call_id` and the matching `observation.results[].source_call_id` | +//! | invoking `decisions[].consequence.result_summary` | `observation.results[].content` | +//! | tool-role message content | `observation.results[]` with no `source_call_id` (OTS does not link tool messages to a call id) | +//! | `turns[].prompt_token_ids` / `completion_token_ids` / `logprobs` | `metrics` fields of the same name | +//! | `metadata.harness` (else `framework`, else `agent_id`) | `agent.name` | +//! | `metadata.agent_version` (else [`UNKNOWN_AGENT_VERSION`]) | `agent.version` | +//! +//! "Invoking" means a decision whose type reports +//! [`DecisionType::is_invocation`](crate::models::DecisionType::is_invocation): +//! a tool selection or a parameter choice. A reasoning step or a response +//! formulation names a thought rather than a callable, so it produces no tool +//! call and no observation result — only its `temper.decisions` entry. +//! +//! ## Derived +//! +//! | ATIF | Derivation | +//! | --- | --- | +//! | `schema_version` | the constant [`ATIF_SCHEMA_VERSION`] | +//! | `metrics.prompt_tokens` / `completion_tokens` | length of the matching token-id array; absent when the array is absent, because OTS records no token counts of its own | +//! | `final_metrics.total_prompt_tokens` / `total_completion_tokens` | sum of the per-step derived counts; absent when no turn carried token ids | +//! | `final_metrics.total_steps` | number of emitted steps | +//! | `session_id` | supplied by the caller — OTS holds the session on the storage row, not in the document | +//! +//! ## Carried in `extra` (OTS-only value ATIF has no field for) +//! +//! | Key | Location | Content | +//! | --- | --- | --- | +//! | `temper.ots_version` | root `extra` | the OTS document `version` | +//! | `temper.metadata` | root `extra` | the whole `OTSMetadata` verbatim, including `spec_version`, `harness`, `agent_version`, `outcome`, `feedback_score`, `human_reviewed`, `tags`, `domain`, `environment`, `parent_trajectory_id` | +//! | `temper.context` | root `extra` | the `OTSContext` verbatim, when non-empty | +//! | `temper.final_reward` | root `extra` | `final_reward` | +//! | `temper.agent_id` / `temper.framework` / `temper.harness` / `temper.spec_version` | `agent.extra` | the originals, so the `agent.name` fallback chain stays reversible and the governing spec stays readable without being mistaken for the agent release | +//! | `temper.turn_id` / `temper.span_id` / `temper.parent_span_id` / `temper.duration_ms` / `temper.error` | step `extra` | per-turn identity and outcome | +//! | `temper.decisions` | step `extra` | each decision verbatim: `decision_type`, `state`, `alternatives`, `evaluation`, `credit_assignment`, `embedding`, and the choice `rationale` / `confidence` — ATIF's `ToolCallSchema` carries none of it | +//! | `temper.assistant_content` | step `extra` | assistant message payloads that are not plain text (tool-call, tool-response, widget content) | +//! | `temper.consequence` | `observation.results[].extra` | `success` and `error_type` of the decision that produced the result | +//! | `temper.response_mask` / `temper.turn_reward` | `metrics.extra` | the per-token loss mask (completion-aligned, same length as `completion_token_ids`) and the per-turn reward | +//! +//! ## Cannot round-trip +//! +//! - **Model identity.** ATIF has `agent.model_name` and `steps[].model_name`; +//! OTS records no model anywhere, so both are always absent. +//! - **Cost and cache accounting.** `metrics.cost_usd`, `cached_tokens`, and +//! the `final_metrics` cost totals have no OTS source. +//! - **Subagent embedding.** ATIF nests children under the parent's +//! `subagent_trajectories`; OTS points the other way, from child to parent +//! via `metadata.parent_trajectory_id`. A single OTS document therefore +//! exports with `subagent_trajectories` absent, and the parent link is kept +//! under `temper.metadata`. Reassembling a hierarchy needs the sibling +//! documents, which one export does not have. +//! - **Message identity.** OTS `message_id`, `visibility`, and +//! `context_snapshot` describe individual messages; ATIF steps have no +//! per-message identity to hang them on, so they are not exported. +//! - **Token counts without ids.** A turn that reports token counts but no +//! token ids cannot exist in OTS — counts are derived from ids — so an ATIF +//! document imported into OTS and exported again loses any count whose ids +//! were absent. +//! - **Context compaction.** ATIF links trajectory segments across a +//! summarization boundary with `continued_trajectory_ref` and marks carried +//! steps `is_copied_context`. OTS models neither, so both are always absent; +//! a Temper session that compacts its context records no boundary for the +//! export to point at. + +mod types; + +use std::collections::BTreeMap; + +use chrono::{DateTime, Utc}; +use serde::Serialize; + +pub use types::{ + ATIF_SCHEMA_VERSION, AtifAgent, AtifFinalMetrics, AtifMetrics, AtifObservation, + AtifObservationResult, AtifSource, AtifStep, AtifToolCall, AtifTrajectory, +}; + +use crate::models::{ + ContentType, MessageRole, OTSDecision, OTSMessage, OTSMessageContent, OTSTrajectory, OTSTurn, +}; + +/// `agent.version` for a trajectory whose producer did not report the agent +/// system's release. +/// +/// ATIF requires the field. OTS only knows it when the harness fills +/// `metadata.agent_version`, and no other OTS value answers the question — the +/// spec hash identifies the governing spec and the document `version` +/// identifies the OTS format, so substituting either would tell a consumer +/// that unrelated builds are the same release. +pub const UNKNOWN_AGENT_VERSION: &str = "unknown"; + +/// Why an OTS trajectory could not be exported as ATIF. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AtifExportError { + /// The trajectory produced no steps. + /// + /// ATIF v1.7 models a trajectory as its step list, and a document with an + /// empty one describes no run. It happens for a session finalized before + /// anything ran: no system message, no turn that carried a message, a + /// decision, or token metrics. + NoSteps, +} + +impl std::fmt::Display for AtifExportError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::NoSteps => write!( + f, + "the trajectory produced no ATIF steps; ATIF v1.7 requires at least one, and a \ + run with no system message, no messages, no decisions, and no token metrics has \ + nothing to export" + ), + } + } +} + +impl std::error::Error for AtifExportError {} + +/// Export an OTS trajectory as an ATIF v1.7 document. +/// +/// `session_id` is supplied by the caller because OTS keeps the session on the +/// storage row rather than in the document. Pass `None` when the session is +/// unknown; ATIF v1.7 makes `session_id` optional. +/// +/// Fails with [`AtifExportError::NoSteps`] when the trajectory yields no +/// steps: a stepless document passes Rust's type checker and fails Harbor's +/// validator, so it is refused here rather than handed on as valid ATIF. +/// +/// See the module docs for the full field-by-field mapping. +pub fn to_atif( + trajectory: &OTSTrajectory, + session_id: Option<&str>, +) -> Result { + let mut steps: Vec = Vec::new(); + let mut next_step_id: i64 = 1; + + if let Some(system_message) = &trajectory.system_message { + steps.push(AtifStep { + step_id: next_step_id, + timestamp: Some(format_timestamp(system_message.timestamp)), + source: AtifSource::System, + message: system_message.content.clone(), + reasoning_content: None, + tool_calls: Vec::new(), + observation: None, + metrics: None, + extra: BTreeMap::new(), + }); + next_step_id += 1; + } + + for turn in &trajectory.turns { + for message in &turn.messages { + let source = match message.role { + MessageRole::User => AtifSource::User, + MessageRole::System => AtifSource::System, + MessageRole::Assistant | MessageRole::Tool => continue, + }; + steps.push(AtifStep { + step_id: next_step_id, + timestamp: Some(format_timestamp(message.timestamp)), + source, + message: plain_text(&message.content).unwrap_or_default(), + reasoning_content: None, + tool_calls: Vec::new(), + observation: None, + metrics: None, + extra: turn_identity(turn), + }); + next_step_id += 1; + } + + if let Some(step) = agent_step(turn, next_step_id) { + steps.push(step); + next_step_id += 1; + } + } + + if steps.is_empty() { + return Err(AtifExportError::NoSteps); + } + + let mut extra = BTreeMap::new(); + insert_json(&mut extra, "temper.ots_version", &trajectory.version); + insert_json(&mut extra, "temper.metadata", &trajectory.metadata); + if trajectory.context != crate::models::OTSContext::new() { + insert_json(&mut extra, "temper.context", &trajectory.context); + } + if let Some(final_reward) = trajectory.final_reward { + insert_json(&mut extra, "temper.final_reward", &final_reward); + } + + Ok(AtifTrajectory { + schema_version: ATIF_SCHEMA_VERSION.to_string(), + session_id: session_id.map(str::to_string), + trajectory_id: Some(trajectory.trajectory_id.clone()), + agent: agent_block(trajectory), + final_metrics: final_metrics(&steps), + subagent_trajectories: None, + extra, + steps, + }) +} + +fn agent_block(trajectory: &OTSTrajectory) -> AtifAgent { + let metadata = &trajectory.metadata; + let name = metadata + .harness + .clone() + .or_else(|| metadata.framework.clone()) + .unwrap_or_else(|| metadata.agent_id.clone()); + // ATIF's `agent.version` is the version of the agent system. The spec hash + // and the OTS document version answer different questions, so neither + // stands in for it: a consumer grouping runs by agent release would split + // one build across specs and merge unrelated builds that share one. + let version = metadata + .agent_version + .clone() + .unwrap_or_else(|| UNKNOWN_AGENT_VERSION.to_string()); + + let mut extra = BTreeMap::new(); + insert_json(&mut extra, "temper.agent_id", &metadata.agent_id); + if let Some(framework) = &metadata.framework { + insert_json(&mut extra, "temper.framework", framework); + } + if let Some(harness) = &metadata.harness { + insert_json(&mut extra, "temper.harness", harness); + } + if let Some(spec_version) = &metadata.spec_version { + insert_json(&mut extra, "temper.spec_version", spec_version); + } + + AtifAgent { + name, + version, + model_name: None, + extra, + } +} + +/// Build the agent step for one turn, or `None` when the turn holds nothing an +/// agent step would carry (a turn of user messages only). +fn agent_step(turn: &OTSTurn, step_id: i64) -> Option { + let assistant_messages: Vec<&OTSMessage> = turn + .messages + .iter() + .filter(|m| m.role == MessageRole::Assistant) + .collect(); + let tool_messages: Vec<&OTSMessage> = turn + .messages + .iter() + .filter(|m| m.role == MessageRole::Tool) + .collect(); + + // Only invoking decisions describe an environment interaction; the rest + // name a thought and are carried in `temper.decisions` alone. + let invocations: Vec<&OTSDecision> = turn + .decisions + .iter() + .filter(|decision| decision.decision_type.is_invocation()) + .collect(); + + let metrics = turn_metrics(turn); + let has_agent_content = !assistant_messages.is_empty() + || !tool_messages.is_empty() + || !turn.decisions.is_empty() + || metrics.is_some(); + if !has_agent_content { + return None; + } + + let message = assistant_messages + .iter() + .filter_map(|m| plain_text(&m.content)) + .collect::>() + .join("\n"); + + let reasoning_content = assistant_messages + .iter() + .filter_map(|m| m.reasoning.clone()) + .find(|reasoning| !reasoning.is_empty()); + + let tool_calls: Vec = invocations + .iter() + .map(|decision| AtifToolCall { + tool_call_id: tool_call_id(decision), + function_name: decision.choice.action.clone(), + arguments: object_arguments(decision.choice.arguments.as_ref()), + }) + .collect(); + + let mut results: Vec = invocations + .iter() + .map(|decision| { + let mut extra = BTreeMap::new(); + insert_json( + &mut extra, + "temper.consequence", + &serde_json::json!({ + "success": decision.consequence.success, + "error_type": decision.consequence.error_type, + }), + ); + AtifObservationResult { + source_call_id: Some(tool_call_id(decision)), + content: decision.consequence.result_summary.clone(), + extra, + } + }) + .collect(); + results.extend(tool_messages.iter().map(|message| AtifObservationResult { + // OTS tool messages carry no tool-call linkage, so the result stays + // unattributed rather than guessing a call id by position. + source_call_id: None, + content: any_content(&message.content), + extra: BTreeMap::new(), + })); + + let mut extra = turn_identity(turn); + if !turn.decisions.is_empty() { + insert_json(&mut extra, "temper.decisions", &turn.decisions); + } + let assistant_content: Vec<&OTSMessageContent> = assistant_messages + .iter() + .map(|m| &m.content) + .filter(|content| content.content_type != ContentType::Text) + .collect(); + if !assistant_content.is_empty() { + insert_json(&mut extra, "temper.assistant_content", &assistant_content); + } + + Some(AtifStep { + step_id, + timestamp: Some(format_timestamp(turn.timestamp)), + source: AtifSource::Agent, + message, + reasoning_content, + tool_calls, + observation: (!results.is_empty()).then_some(AtifObservation { results }), + metrics, + extra, + }) +} + +fn turn_metrics(turn: &OTSTurn) -> Option { + let mut extra = BTreeMap::new(); + if let Some(response_mask) = &turn.response_mask { + insert_json(&mut extra, "temper.response_mask", response_mask); + } + if let Some(turn_reward) = turn.turn_reward { + insert_json(&mut extra, "temper.turn_reward", &turn_reward); + } + + let metrics = AtifMetrics { + prompt_tokens: turn.prompt_token_ids.as_ref().map(|ids| ids.len() as u64), + completion_tokens: turn + .completion_token_ids + .as_ref() + .map(|ids| ids.len() as u64), + cost_usd: None, + prompt_token_ids: turn.prompt_token_ids.clone(), + completion_token_ids: turn.completion_token_ids.clone(), + logprobs: turn.logprobs.clone(), + extra, + }; + (!metrics.is_empty()).then_some(metrics) +} + +fn final_metrics(steps: &[AtifStep]) -> Option { + let mut total_prompt_tokens: Option = None; + let mut total_completion_tokens: Option = None; + for metrics in steps.iter().filter_map(|step| step.metrics.as_ref()) { + if let Some(prompt_tokens) = metrics.prompt_tokens { + total_prompt_tokens = Some(total_prompt_tokens.unwrap_or(0) + prompt_tokens); + } + if let Some(completion_tokens) = metrics.completion_tokens { + total_completion_tokens = + Some(total_completion_tokens.unwrap_or(0) + completion_tokens); + } + } + + let final_metrics = AtifFinalMetrics { + total_prompt_tokens, + total_completion_tokens, + total_cost_usd: None, + total_steps: Some(steps.len() as u64), + extra: BTreeMap::new(), + }; + Some(final_metrics) +} + +/// The id linking a decision to the observation it produced. +/// +/// `cause_id` is the provider's tool-call id and is preferred; `decision_id` +/// is the fallback so the correlation still holds for trajectories recorded +/// before causality capture existed. +fn tool_call_id(decision: &OTSDecision) -> String { + decision + .cause_id + .clone() + .unwrap_or_else(|| decision.decision_id.clone()) +} + +/// ATIF requires `arguments` to be a JSON object. +fn object_arguments(arguments: Option<&serde_json::Value>) -> serde_json::Value { + match arguments { + Some(value) if value.is_object() => value.clone(), + // A non-object argument payload is wrapped rather than dropped, so the + // export stays schema-valid without losing what the agent passed. + Some(value) => serde_json::json!({ "temper.value": value }), + None => serde_json::json!({}), + } +} + +/// Text of a plain-text message content, or `None` for structured content. +fn plain_text(content: &OTSMessageContent) -> Option { + match content.content_type { + ContentType::Text => content.text.clone(), + _ => None, + } +} + +/// Text of any message content: the text field when present, else the +/// structured payload serialized as JSON. +fn any_content(content: &OTSMessageContent) -> Option { + content + .text + .clone() + .or_else(|| content.data.as_ref().map(|data| data.to_string())) +} + +fn turn_identity(turn: &OTSTurn) -> BTreeMap { + let mut extra = BTreeMap::new(); + insert_json(&mut extra, "temper.turn_id", &turn.turn_id); + insert_json(&mut extra, "temper.span_id", &turn.span_id); + if let Some(parent_span_id) = &turn.parent_span_id { + insert_json(&mut extra, "temper.parent_span_id", parent_span_id); + } + if let Some(duration_ms) = turn.duration_ms { + insert_json(&mut extra, "temper.duration_ms", &duration_ms); + } + if turn.error { + insert_json(&mut extra, "temper.error", &true); + } + extra +} + +/// Insert a serializable value into an `extra` map. +/// +/// Serialization of OTS types cannot fail — every field is a plain data type +/// with no map keys other than strings — so a failure is dropped rather than +/// propagated as an error the caller could not act on. +fn insert_json( + extra: &mut BTreeMap, + key: &str, + value: &T, +) { + if let Ok(value) = serde_json::to_value(value) { + extra.insert(key.to_string(), value); + } +} + +fn format_timestamp(timestamp: DateTime) -> String { + timestamp.to_rfc3339() +} + +#[cfg(test)] +#[path = "mod_test.rs"] +mod tests; diff --git a/crates/temper-ots/src/atif/mod_test.rs b/crates/temper-ots/src/atif/mod_test.rs new file mode 100644 index 000000000..2a4e022de --- /dev/null +++ b/crates/temper-ots/src/atif/mod_test.rs @@ -0,0 +1,134 @@ +use super::*; +use crate::models::{ + DecisionType, OTSChoice, OTSConsequence, OTSMetadata, OTSSystemMessage, OutcomeType, +}; +use temper_runtime::scheduler::sim_now; + +#[test] +fn non_object_arguments_are_wrapped_not_dropped() { + let wrapped = object_arguments(Some(&serde_json::json!("plain string"))); + assert_eq!(wrapped, serde_json::json!({"temper.value": "plain string"})); + assert_eq!(object_arguments(None), serde_json::json!({})); +} + +#[test] +fn decision_without_cause_id_falls_back_to_decision_id() { + let decision = OTSDecision::new( + DecisionType::ToolSelection, + OTSChoice::new("Ship"), + OTSConsequence::success(), + ) + .with_decision_id("decision-1"); + assert_eq!(tool_call_id(&decision), "decision-1"); + assert_eq!( + tool_call_id(&decision.with_cause_id("call-9")), + "call-9", + "the provider's tool-call id wins over the synthetic decision id" + ); +} + +#[test] +fn user_only_turn_emits_no_agent_step() { + let now = sim_now(); + let metadata = OTSMetadata::new("task", "agent-1", OutcomeType::Success, now); + let turn = OTSTurn::new(1, now).with_message(OTSMessage::new( + MessageRole::User, + OTSMessageContent::text("hello"), + now, + )); + let trajectory = OTSTrajectory::new(metadata) + .with_system_message(OTSSystemMessage::new("be helpful", now)) + .with_turn(turn); + + let atif = to_atif(&trajectory, None).expect("the system message alone is a step"); + let sources: Vec = atif.steps.iter().map(|step| step.source).collect(); + assert_eq!(sources, vec![AtifSource::System, AtifSource::User]); + assert_eq!(atif.steps[1].step_id, 2); +} + +#[test] +fn a_trajectory_with_nothing_in_it_is_not_exportable() { + let now = sim_now(); + let metadata = OTSMetadata::new("task", "agent-1", OutcomeType::Failure, now); + let trajectory = OTSTrajectory::new(metadata); + assert_eq!( + to_atif(&trajectory, None), + Err(AtifExportError::NoSteps), + "ATIF v1.7 requires at least one step; a stepless document is not valid ATIF" + ); +} + +#[test] +fn a_reasoning_decision_is_not_exported_as_a_tool_call() { + let now = sim_now(); + let metadata = OTSMetadata::new("task", "agent-1", OutcomeType::Success, now); + let reasoning = OTSDecision::new( + DecisionType::ReasoningStep, + OTSChoice::new("compare shipping options"), + OTSConsequence::success(), + ); + let invocation = OTSDecision::new( + DecisionType::ToolSelection, + OTSChoice::new("Ship"), + OTSConsequence::success(), + ); + let trajectory = OTSTrajectory::new(metadata).with_turn( + OTSTurn::new(1, now) + .with_decision(reasoning) + .with_decision(invocation), + ); + + let atif = to_atif(&trajectory, None).expect("export"); + let step = &atif.steps[0]; + let called: Vec<&str> = step + .tool_calls + .iter() + .map(|call| call.function_name.as_str()) + .collect(); + assert_eq!( + called, + vec!["Ship"], + "a reasoning step names a thought, not a callable" + ); + assert_eq!( + step.observation + .as_ref() + .expect("observation") + .results + .len(), + 1, + "no environment result is fabricated for a thought" + ); + assert_eq!( + step.extra["temper.decisions"] + .as_array() + .expect("decisions") + .len(), + 2, + "both decisions are still carried verbatim" + ); +} + +#[test] +fn agent_version_reports_the_agent_release_not_the_spec() { + let now = sim_now(); + let metadata = OTSMetadata::new("task", "agent-1", OutcomeType::Success, now) + .with_spec_version("sha256:abcd") + .with_harness("temperpaw"); + let trajectory = OTSTrajectory::new(metadata.clone()) + .with_system_message(OTSSystemMessage::new("be helpful", now)); + + let atif = to_atif(&trajectory, None).expect("export"); + assert_eq!( + atif.agent.version, UNKNOWN_AGENT_VERSION, + "a spec hash is not an agent release" + ); + assert_eq!(atif.agent.extra["temper.spec_version"], "sha256:abcd"); + + let versioned = OTSTrajectory::new(metadata.with_agent_version("temperpaw 3.2")) + .with_system_message(OTSSystemMessage::new("be helpful", now)); + assert_eq!( + to_atif(&versioned, None).expect("export").agent.version, + "temperpaw 3.2" + ); +} diff --git a/crates/temper-ots/src/atif/types.rs b/crates/temper-ots/src/atif/types.rs new file mode 100644 index 000000000..97964bd56 --- /dev/null +++ b/crates/temper-ots/src/atif/types.rs @@ -0,0 +1,218 @@ +//! The ATIF v1.7 document schema as Rust types. +//! +//! Field semantics follow the Harbor RFC; the OTS-to-ATIF mapping that fills +//! them lives in the parent module. + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; + +/// The ATIF schema version this exporter emits. +pub const ATIF_SCHEMA_VERSION: &str = "ATIF-v1.7"; + +/// Originator of an ATIF step. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum AtifSource { + /// A system prompt or system-initiated event. + System, + /// A message from the user. + User, + /// An agent turn: response, tool calls, and their observations. + Agent, +} + +/// An ATIF v1.7 trajectory document. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct AtifTrajectory { + /// ATIF compatibility marker, always [`ATIF_SCHEMA_VERSION`]. + pub schema_version: String, + + /// Run-scoped identifier, supplied by the caller. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub session_id: Option, + + /// Document-scoped identifier, from the OTS trajectory id. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub trajectory_id: Option, + + /// The agent system that produced the trajectory. + pub agent: AtifAgent, + + /// The full interaction history. + pub steps: Vec, + + /// Aggregate statistics for the whole trajectory. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub final_metrics: Option, + + /// Embedded subagent trajectories. Always absent on an OTS export — see + /// the module docs on subagent embedding. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub subagent_trajectories: Option>, + + /// Trajectory-level values ATIF has no field for. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub extra: BTreeMap, +} + +/// The agent system identification block. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct AtifAgent { + /// Name of the agent system. + pub name: String, + + /// Version identifier of the agent system. + pub version: String, + + /// Default model for the run. Never set on an OTS export — OTS records no + /// model identity. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub model_name: Option, + + /// Agent-level values ATIF has no field for. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub extra: BTreeMap, +} + +/// One step of the interaction history. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct AtifStep { + /// Ordinal index, starting at 1. + pub step_id: i64, + + /// ISO-8601 timestamp of the step. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub timestamp: Option, + + /// Who produced the step. + pub source: AtifSource, + + /// The dialogue message. Required by ATIF, may be empty. + pub message: String, + + /// The agent's explicit internal reasoning. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reasoning_content: Option, + + /// Tool invocations issued by this step. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub tool_calls: Vec, + + /// Environment feedback for this step. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub observation: Option, + + /// LLM operational and RL data for this step. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub metrics: Option, + + /// Step-level values ATIF has no field for. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub extra: BTreeMap, +} + +/// A single tool invocation. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct AtifToolCall { + /// Identifier correlating this call with its observation result. + pub tool_call_id: String, + + /// The invoked function or action. + pub function_name: String, + + /// Arguments passed to the function. Always a JSON object. + pub arguments: serde_json::Value, +} + +/// Environment feedback for a step. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct AtifObservation { + /// One entry per tool call or action result. + pub results: Vec, +} + +/// A single observation result. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct AtifObservationResult { + /// The `tool_call_id` this result answers, when the result came from a + /// structured tool call. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_call_id: Option, + + /// The result payload. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub content: Option, + + /// Result-level values ATIF has no field for. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub extra: BTreeMap, +} + +/// Per-step LLM metrics. +#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] +pub struct AtifMetrics { + /// Input token count. Derived from `prompt_token_ids`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub prompt_tokens: Option, + + /// Output token count. Derived from `completion_token_ids`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub completion_tokens: Option, + + /// Monetary cost. Never set on an OTS export. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cost_usd: Option, + + /// Prompt token ids exactly as the serving stack produced them. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub prompt_token_ids: Option>, + + /// Completion token ids exactly as the serving stack produced them. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub completion_token_ids: Option>, + + /// Per-token log probabilities. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub logprobs: Option>, + + /// Metric-level values ATIF has no field for. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub extra: BTreeMap, +} + +/// Aggregate statistics for the whole trajectory. +#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] +pub struct AtifFinalMetrics { + /// Sum of the derived per-step prompt token counts. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub total_prompt_tokens: Option, + + /// Sum of the derived per-step completion token counts. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub total_completion_tokens: Option, + + /// Total cost. Never set on an OTS export. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub total_cost_usd: Option, + + /// Number of emitted steps. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub total_steps: Option, + + /// Trajectory-level metrics ATIF has no field for. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub extra: BTreeMap, +} + +impl AtifMetrics { + pub(super) fn is_empty(&self) -> bool { + self.prompt_tokens.is_none() + && self.completion_tokens.is_none() + && self.cost_usd.is_none() + && self.prompt_token_ids.is_none() + && self.completion_token_ids.is_none() + && self.logprobs.is_none() + && self.extra.is_empty() + } +} diff --git a/crates/temper-ots/src/builder.rs b/crates/temper-ots/src/builder.rs index 94ab83be5..669aded41 100644 --- a/crates/temper-ots/src/builder.rs +++ b/crates/temper-ots/src/builder.rs @@ -84,15 +84,23 @@ impl TrajectoryBuilder { turn.decisions.push(decision); } - /// End the current turn, recording its duration. Panics if no turn is in progress. + /// End the current turn, recording its duration. /// /// Duration is computed as the difference between `end_time` and the /// turn's start timestamp. + /// + /// Panics if no turn is in progress, or if the turn's completion-side + /// token signals do not line up — a misaligned turn is a malformed + /// training sample, and sealing it here is the last point at which the + /// producer can still be told which turn was wrong. pub fn end_turn(&mut self, end_time: DateTime) { let mut turn = self .current_turn .take() .expect("Cannot end turn: no turn in progress"); + if let Err(error) = turn.validate_token_signals() { + panic!("Cannot end turn: {error}"); + } let duration_ms = (end_time - turn.timestamp).num_milliseconds() as f64; turn.duration_ms = Some(duration_ms); self.turns.push(turn); @@ -103,15 +111,73 @@ impl TrajectoryBuilder { self.system_message = Some(system_message); } - /// Build the final trajectory, consuming the builder. + /// Record the actor spec version (hash or version) this run executed under. + pub fn set_spec_version(&mut self, spec_version: impl Into) { + self.metadata.spec_version = Some(spec_version.into()); + } + + /// Record the harness that is driving this run (e.g. "temperpaw"). + pub fn set_harness(&mut self, harness: impl Into) { + self.metadata.harness = Some(harness.into()); + } + + /// Attach serving-stack token IDs to the current turn. /// - /// If a turn is still in progress, it is automatically ended using - /// `sim_now()` as the end time. + /// Panics if no turn is in progress, or if a completion-aligned signal is + /// already set at a different length — see [`OTSTurn::response_mask`]. + pub fn set_turn_token_ids( + &mut self, + prompt_token_ids: Vec, + completion_token_ids: Vec, + ) { + let turn = self + .current_turn + .take() + .expect("Cannot set token ids: no turn in progress"); + self.current_turn = Some( + turn.with_prompt_token_ids(prompt_token_ids) + .with_completion_token_ids(completion_token_ids), + ); + } + + /// Attach the per-token response mask to the current turn. + /// + /// `1` marks a model-generated token, `0` a tool or otherwise injected + /// token, one entry per completion token. Panics if no turn is in + /// progress, if an entry is outside `{0, 1}`, or if a completion-aligned + /// signal is already set at a different length. + pub fn set_turn_response_mask(&mut self, response_mask: Vec) { + let turn = self + .current_turn + .take() + .expect("Cannot set response mask: no turn in progress"); + self.current_turn = Some(turn.with_response_mask(response_mask)); + } + + /// Attach per-token log probabilities to the current turn. /// + /// One entry per completion token. Panics if no turn is in progress, or + /// if a completion-aligned signal is already set at a different length. + pub fn set_turn_logprobs(&mut self, logprobs: Vec) { + let turn = self + .current_turn + .take() + .expect("Cannot set logprobs: no turn in progress"); + self.current_turn = Some(turn.with_logprobs(logprobs)); + } + /// Build a snapshot of the current trajectory without consuming the builder. /// /// Useful for mid-session uploads where the session should continue /// recording new turns after the upload. + /// + /// The in-progress turn goes through the same token-signal check + /// [`Self::end_turn`] applies. A mid-turn flush is the one path that + /// carries an unsealed turn into a document, and a turn whose + /// completion-side signals do not yet agree is a malformed training + /// sample: the upload endpoint refuses the whole document, so producing + /// one here would lose the snapshot at the far end for a fault that is + /// visible right here. pub fn snapshot(&self) -> OTSTrajectory { let mut metadata = self.metadata.clone(); let now = sim_now(); // determinism-ok: sim_now is DST-safe @@ -120,6 +186,9 @@ impl TrajectoryBuilder { let mut turns = self.turns.clone(); if let Some(ref current) = self.current_turn { + if let Err(error) = current.validate_token_signals() { + panic!("Cannot snapshot trajectory: {error}"); + } turns.push(current.clone()); } @@ -158,200 +227,5 @@ impl TrajectoryBuilder { } #[cfg(test)] -mod tests { - use super::*; - use crate::models::{ - DecisionType, MessageRole, OTSChoice, OTSConsequence, OTSMessageContent, OutcomeType, - }; - use temper_runtime::scheduler::sim_now; - - #[test] - fn test_builder_basic_flow() { - let now = sim_now(); - let metadata = OTSMetadata::new("Test task", "agent_1", OutcomeType::Success, now); - let context = OTSContext::new(); - let mut builder = TrajectoryBuilder::new(metadata, context); - - builder.start_turn(now); - builder.add_message(OTSMessage::new( - MessageRole::User, - OTSMessageContent::text("Hello"), - now, - )); - builder.add_message(OTSMessage::new( - MessageRole::Assistant, - OTSMessageContent::text("Hi there"), - now, - )); - builder.end_turn(now); - - let trajectory = builder.build(); - assert_eq!(trajectory.turns.len(), 1); - assert_eq!(trajectory.turns[0].messages.len(), 2); - assert_eq!(trajectory.turns[0].turn_id, 1); - } - - #[test] - fn test_builder_multiple_turns() { - let now = sim_now(); - let metadata = OTSMetadata::new("Multi-turn", "agent_2", OutcomeType::Success, now); - let mut builder = TrajectoryBuilder::new(metadata, OTSContext::new()); - - builder.start_turn(now); - builder.add_message(OTSMessage::new( - MessageRole::User, - OTSMessageContent::text("Turn 1"), - now, - )); - builder.end_turn(now); - - builder.start_turn(now); - builder.add_message(OTSMessage::new( - MessageRole::User, - OTSMessageContent::text("Turn 2"), - now, - )); - builder.end_turn(now); - - let trajectory = builder.build(); - assert_eq!(trajectory.turns.len(), 2); - assert_eq!(trajectory.turns[0].turn_id, 1); - assert_eq!(trajectory.turns[1].turn_id, 2); - } - - #[test] - fn test_builder_with_decisions() { - let now = sim_now(); - let metadata = OTSMetadata::new("Decision task", "agent_3", OutcomeType::Success, now); - let mut builder = TrajectoryBuilder::new(metadata, OTSContext::new()); - - builder.start_turn(now); - let decision = OTSDecision::new( - DecisionType::ToolSelection, - OTSChoice::new("search"), - OTSConsequence::success(), - ); - builder.add_decision(decision); - builder.end_turn(now); - - let trajectory = builder.build(); - assert_eq!(trajectory.turns[0].decisions.len(), 1); - } - - #[test] - fn test_builder_with_system_message() { - let now = sim_now(); - let metadata = OTSMetadata::new("Sys msg task", "agent_4", OutcomeType::Success, now); - let mut builder = TrajectoryBuilder::new(metadata, OTSContext::new()); - - builder.set_system_message(OTSSystemMessage::new("You are helpful", now)); - - let trajectory = builder.build(); - assert!(trajectory.system_message.is_some()); - assert_eq!( - trajectory.system_message.unwrap().content, - "You are helpful" - ); - } - - #[test] - fn test_builder_auto_closes_turn() { - let now = sim_now(); - let metadata = OTSMetadata::new("Auto-close", "agent_5", OutcomeType::Success, now); - let mut builder = TrajectoryBuilder::new(metadata, OTSContext::new()); - - builder.start_turn(now); - builder.add_message(OTSMessage::new( - MessageRole::User, - OTSMessageContent::text("Unclosed turn"), - now, - )); - - // Build should auto-close the turn - let trajectory = builder.build(); - assert_eq!(trajectory.turns.len(), 1); - } - - #[test] - fn test_builder_sets_end_timestamp() { - let now = sim_now(); - let metadata = OTSMetadata::new("End time", "agent_6", OutcomeType::Success, now); - let builder = TrajectoryBuilder::new(metadata, OTSContext::new()); - - let trajectory = builder.build(); - assert!(trajectory.metadata.timestamp_end.is_some()); - assert!(trajectory.metadata.duration_ms.is_some()); - } - - #[test] - fn test_snapshot_does_not_consume_builder() { - let now = sim_now(); - let metadata = OTSMetadata::new("Snapshot", "agent-snap", OutcomeType::Success, now); - let mut builder = TrajectoryBuilder::new(metadata, OTSContext::new()); - - builder.start_turn(now); - builder.add_message(OTSMessage::new( - MessageRole::User, - OTSMessageContent::text("in-progress"), - now, - )); - - let snapshot = builder.snapshot(); - assert_eq!( - snapshot.turns.len(), - 1, - "snapshot should include in-progress turn" - ); - - // Builder should remain usable after snapshot. - builder.end_turn(now); - let final_trajectory = builder.build(); - assert_eq!(final_trajectory.turns.len(), 1); - } - - #[test] - #[should_panic(expected = "Cannot start a new turn while one is in progress")] - fn test_builder_double_start_panics() { - let now = sim_now(); - let metadata = OTSMetadata::new("Double start", "agent_7", OutcomeType::Success, now); - let mut builder = TrajectoryBuilder::new(metadata, OTSContext::new()); - - builder.start_turn(now); - builder.start_turn(now); // Should panic - } - - #[test] - #[should_panic(expected = "Cannot end turn: no turn in progress")] - fn test_builder_end_without_start_panics() { - let now = sim_now(); - let metadata = OTSMetadata::new("No start", "agent_8", OutcomeType::Success, now); - let mut builder = TrajectoryBuilder::new(metadata, OTSContext::new()); - - builder.end_turn(now); // Should panic - } - - #[test] - #[should_panic(expected = "Cannot add message: no turn in progress")] - fn test_builder_message_without_turn_panics() { - let now = sim_now(); - let metadata = OTSMetadata::new("No turn", "agent_9", OutcomeType::Success, now); - let mut builder = TrajectoryBuilder::new(metadata, OTSContext::new()); - - builder.add_message(OTSMessage::new( - MessageRole::User, - OTSMessageContent::text("Orphan"), - now, - )); - } - - #[test] - fn test_builder_empty_trajectory() { - let now = sim_now(); - let metadata = OTSMetadata::new("Empty", "agent_10", OutcomeType::Failure, now); - let builder = TrajectoryBuilder::new(metadata, OTSContext::new()); - - let trajectory = builder.build(); - assert!(trajectory.turns.is_empty()); - assert_eq!(trajectory.version, "0.1.0"); - } -} +#[path = "builder_test.rs"] +mod builder_test; diff --git a/crates/temper-ots/src/builder_test.rs b/crates/temper-ots/src/builder_test.rs new file mode 100644 index 000000000..416509437 --- /dev/null +++ b/crates/temper-ots/src/builder_test.rs @@ -0,0 +1,333 @@ +//! Tests for [`TrajectoryBuilder`]. + +use super::*; +use crate::models::{ + DecisionType, MessageRole, OTSChoice, OTSConsequence, OTSMessageContent, OutcomeType, +}; +use temper_runtime::scheduler::sim_now; + +#[test] +fn test_builder_basic_flow() { + let now = sim_now(); + let metadata = OTSMetadata::new("Test task", "agent_1", OutcomeType::Success, now); + let context = OTSContext::new(); + let mut builder = TrajectoryBuilder::new(metadata, context); + + builder.start_turn(now); + builder.add_message(OTSMessage::new( + MessageRole::User, + OTSMessageContent::text("Hello"), + now, + )); + builder.add_message(OTSMessage::new( + MessageRole::Assistant, + OTSMessageContent::text("Hi there"), + now, + )); + builder.end_turn(now); + + let trajectory = builder.build(); + assert_eq!(trajectory.turns.len(), 1); + assert_eq!(trajectory.turns[0].messages.len(), 2); + assert_eq!(trajectory.turns[0].turn_id, 1); +} + +#[test] +fn test_builder_multiple_turns() { + let now = sim_now(); + let metadata = OTSMetadata::new("Multi-turn", "agent_2", OutcomeType::Success, now); + let mut builder = TrajectoryBuilder::new(metadata, OTSContext::new()); + + builder.start_turn(now); + builder.add_message(OTSMessage::new( + MessageRole::User, + OTSMessageContent::text("Turn 1"), + now, + )); + builder.end_turn(now); + + builder.start_turn(now); + builder.add_message(OTSMessage::new( + MessageRole::User, + OTSMessageContent::text("Turn 2"), + now, + )); + builder.end_turn(now); + + let trajectory = builder.build(); + assert_eq!(trajectory.turns.len(), 2); + assert_eq!(trajectory.turns[0].turn_id, 1); + assert_eq!(trajectory.turns[1].turn_id, 2); +} + +#[test] +fn test_builder_with_decisions() { + let now = sim_now(); + let metadata = OTSMetadata::new("Decision task", "agent_3", OutcomeType::Success, now); + let mut builder = TrajectoryBuilder::new(metadata, OTSContext::new()); + + builder.start_turn(now); + let decision = OTSDecision::new( + DecisionType::ToolSelection, + OTSChoice::new("search"), + OTSConsequence::success(), + ); + builder.add_decision(decision); + builder.end_turn(now); + + let trajectory = builder.build(); + assert_eq!(trajectory.turns[0].decisions.len(), 1); +} + +#[test] +fn test_builder_with_system_message() { + let now = sim_now(); + let metadata = OTSMetadata::new("Sys msg task", "agent_4", OutcomeType::Success, now); + let mut builder = TrajectoryBuilder::new(metadata, OTSContext::new()); + + builder.set_system_message(OTSSystemMessage::new("You are helpful", now)); + + let trajectory = builder.build(); + assert!(trajectory.system_message.is_some()); + assert_eq!( + trajectory.system_message.unwrap().content, + "You are helpful" + ); +} + +#[test] +fn test_builder_auto_closes_turn() { + let now = sim_now(); + let metadata = OTSMetadata::new("Auto-close", "agent_5", OutcomeType::Success, now); + let mut builder = TrajectoryBuilder::new(metadata, OTSContext::new()); + + builder.start_turn(now); + builder.add_message(OTSMessage::new( + MessageRole::User, + OTSMessageContent::text("Unclosed turn"), + now, + )); + + // Build should auto-close the turn + let trajectory = builder.build(); + assert_eq!(trajectory.turns.len(), 1); +} + +#[test] +fn test_builder_sets_end_timestamp() { + let now = sim_now(); + let metadata = OTSMetadata::new("End time", "agent_6", OutcomeType::Success, now); + let builder = TrajectoryBuilder::new(metadata, OTSContext::new()); + + let trajectory = builder.build(); + assert!(trajectory.metadata.timestamp_end.is_some()); + assert!(trajectory.metadata.duration_ms.is_some()); +} + +#[test] +fn test_snapshot_does_not_consume_builder() { + let now = sim_now(); + let metadata = OTSMetadata::new("Snapshot", "agent-snap", OutcomeType::Success, now); + let mut builder = TrajectoryBuilder::new(metadata, OTSContext::new()); + + builder.start_turn(now); + builder.add_message(OTSMessage::new( + MessageRole::User, + OTSMessageContent::text("in-progress"), + now, + )); + + let snapshot = builder.snapshot(); + assert_eq!( + snapshot.turns.len(), + 1, + "snapshot should include in-progress turn" + ); + + // Builder should remain usable after snapshot. + builder.end_turn(now); + let final_trajectory = builder.build(); + assert_eq!(final_trajectory.turns.len(), 1); +} + +#[test] +#[should_panic(expected = "Cannot start a new turn while one is in progress")] +fn test_builder_double_start_panics() { + let now = sim_now(); + let metadata = OTSMetadata::new("Double start", "agent_7", OutcomeType::Success, now); + let mut builder = TrajectoryBuilder::new(metadata, OTSContext::new()); + + builder.start_turn(now); + builder.start_turn(now); // Should panic +} + +#[test] +#[should_panic(expected = "Cannot end turn: no turn in progress")] +fn test_builder_end_without_start_panics() { + let now = sim_now(); + let metadata = OTSMetadata::new("No start", "agent_8", OutcomeType::Success, now); + let mut builder = TrajectoryBuilder::new(metadata, OTSContext::new()); + + builder.end_turn(now); // Should panic +} + +#[test] +#[should_panic(expected = "Cannot add message: no turn in progress")] +fn test_builder_message_without_turn_panics() { + let now = sim_now(); + let metadata = OTSMetadata::new("No turn", "agent_9", OutcomeType::Success, now); + let mut builder = TrajectoryBuilder::new(metadata, OTSContext::new()); + + builder.add_message(OTSMessage::new( + MessageRole::User, + OTSMessageContent::text("Orphan"), + now, + )); +} + +#[test] +fn test_builder_records_spec_version_and_harness() { + let now = sim_now(); + let metadata = OTSMetadata::new("Provenance", "agent-prov", OutcomeType::Success, now); + let mut builder = TrajectoryBuilder::new(metadata, OTSContext::new()); + + builder.set_spec_version("sha256:abc123"); + builder.set_harness("temperpaw"); + + let trajectory = builder.build(); + assert_eq!( + trajectory.metadata.spec_version.as_deref(), + Some("sha256:abc123") + ); + assert_eq!(trajectory.metadata.harness.as_deref(), Some("temperpaw")); +} + +#[test] +fn test_builder_records_turn_token_signals() { + let now = sim_now(); + let metadata = OTSMetadata::new("Token signals", "agent-tok", OutcomeType::Success, now); + let mut builder = TrajectoryBuilder::new(metadata, OTSContext::new()); + + builder.start_turn(now); + // Two completion tokens, so two mask entries and two log probabilities. + builder.set_turn_token_ids(vec![1, 2, 3], vec![4, 5]); + builder.set_turn_response_mask(vec![1, 0]); + builder.set_turn_logprobs(vec![-0.1, -0.2]); + builder.end_turn(now); + + let trajectory = builder.build(); + let turn = &trajectory.turns[0]; + assert_eq!( + turn.prompt_token_ids.as_deref(), + Some([1u32, 2, 3].as_ref()) + ); + assert_eq!( + turn.completion_token_ids.as_deref(), + Some([4u32, 5].as_ref()) + ); + assert_eq!(turn.response_mask.as_deref(), Some([1u8, 0].as_ref())); + assert_eq!(turn.logprobs.as_ref().map(Vec::len), Some(2)); +} + +#[test] +#[should_panic(expected = "no completion_token_ids to align it against")] +fn test_builder_end_turn_rejects_a_mask_with_no_completion_tokens() { + let now = sim_now(); + let metadata = OTSMetadata::new("Dangling mask", "agent-tok", OutcomeType::Success, now); + let mut builder = TrajectoryBuilder::new(metadata, OTSContext::new()); + + builder.start_turn(now); + builder.set_turn_response_mask(vec![1, 0]); + builder.end_turn(now); +} + +#[test] +#[should_panic(expected = "completion-side signals are aligned position for position")] +fn test_builder_rejects_a_misaligned_mask() { + let now = sim_now(); + let metadata = OTSMetadata::new("Misaligned", "agent-tok", OutcomeType::Success, now); + let mut builder = TrajectoryBuilder::new(metadata, OTSContext::new()); + + builder.start_turn(now); + builder.set_turn_token_ids(vec![1, 2, 3], vec![4, 5]); + builder.set_turn_response_mask(vec![1, 0, 1]); +} + +#[test] +fn test_builder_snapshot_carries_token_signals() { + let now = sim_now(); + let metadata = OTSMetadata::new("Snapshot tokens", "agent-snap", OutcomeType::Success, now); + let mut builder = TrajectoryBuilder::new(metadata, OTSContext::new()); + + builder.set_harness("claude-code"); + builder.start_turn(now); + builder.set_turn_token_ids(vec![7, 8], vec![9]); + + let snapshot = builder.snapshot(); + assert_eq!(snapshot.metadata.harness.as_deref(), Some("claude-code")); + assert_eq!( + snapshot.turns[0].completion_token_ids.as_deref(), + Some([9u32].as_ref()) + ); +} + +#[test] +#[should_panic(expected = "Cannot set token ids: no turn in progress")] +fn test_builder_token_ids_without_turn_panics() { + let now = sim_now(); + let metadata = OTSMetadata::new("No turn", "agent-tok-2", OutcomeType::Success, now); + let mut builder = TrajectoryBuilder::new(metadata, OTSContext::new()); + + builder.set_turn_token_ids(vec![1], vec![2]); +} + +#[test] +#[should_panic(expected = "Cannot set response mask: no turn in progress")] +fn test_builder_response_mask_without_turn_panics() { + let now = sim_now(); + let metadata = OTSMetadata::new("No turn", "agent-tok-3", OutcomeType::Success, now); + let mut builder = TrajectoryBuilder::new(metadata, OTSContext::new()); + + builder.set_turn_response_mask(vec![1]); +} + +#[test] +#[should_panic(expected = "Cannot set logprobs: no turn in progress")] +fn test_builder_logprobs_without_turn_panics() { + let now = sim_now(); + let metadata = OTSMetadata::new("No turn", "agent-tok-4", OutcomeType::Success, now); + let mut builder = TrajectoryBuilder::new(metadata, OTSContext::new()); + + builder.set_turn_logprobs(vec![-0.1]); +} + +#[test] +fn test_builder_empty_trajectory() { + let now = sim_now(); + let metadata = OTSMetadata::new("Empty", "agent_10", OutcomeType::Failure, now); + let builder = TrajectoryBuilder::new(metadata, OTSContext::new()); + + let trajectory = builder.build(); + assert!(trajectory.turns.is_empty()); + assert_eq!(trajectory.version, "0.1.0"); +} + +#[test] +#[should_panic(expected = "Cannot snapshot trajectory")] +fn test_builder_snapshot_rejects_a_turn_whose_signals_do_not_align() { + // A mid-turn flush is the one path that carries an unsealed turn into a + // document. The mask here has nothing to index into, which the setters + // cannot see and the upload endpoint refuses. + let now = sim_now(); + let metadata = OTSMetadata::new( + "Snapshot misaligned", + "agent-snap-2", + OutcomeType::Success, + now, + ); + let mut builder = TrajectoryBuilder::new(metadata, OTSContext::new()); + + builder.start_turn(now); + builder.set_turn_response_mask(vec![1, 0]); + let _ = builder.snapshot(); +} diff --git a/crates/temper-ots/src/lib.rs b/crates/temper-ots/src/lib.rs index 0fb6be493..2c0439b0b 100644 --- a/crates/temper-ots/src/lib.rs +++ b/crates/temper-ots/src/lib.rs @@ -10,11 +10,16 @@ //! - **Core Models**: Complete type-safe OTS data structures //! - **DST Compatible**: All types use deterministic collections and sim-aware constructors //! - **Builder**: Incremental trajectory construction via [`TrajectoryBuilder`] +//! - **ATIF Export**: Harbor ATIF v1.7 interchange output via [`atif::to_atif`] +pub mod atif; pub mod builder; pub mod models; // Re-exports for convenience +pub use atif::{ + ATIF_SCHEMA_VERSION, AtifExportError, AtifTrajectory, UNKNOWN_AGENT_VERSION, to_atif, +}; pub use builder::TrajectoryBuilder; pub use models::{ DecisionType, EvaluatorType, MessageRole, OTSAnnotation, OTSChoice, OTSConsequence, OTSContext, diff --git a/crates/temper-ots/src/models/decision.rs b/crates/temper-ots/src/models/decision.rs index 345639bcc..e21bef915 100644 --- a/crates/temper-ots/src/models/decision.rs +++ b/crates/temper-ots/src/models/decision.rs @@ -374,6 +374,14 @@ pub struct OTSDecision { /// Optional embedding vector for similarity search #[serde(skip_serializing_if = "Option::is_none")] pub embedding: Option>, + + /// The `tool_call` id that links this decision to the observation it caused. + /// + /// Without it a decision and the tool result that followed can only be + /// paired by position, which breaks under interleaved or parallel tool + /// calls. Carries the provider's tool-call id verbatim. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cause_id: Option, } impl OTSDecision { @@ -395,6 +403,7 @@ impl OTSDecision { evaluation: None, credit_assignment: None, embedding: None, + cause_id: None, } } @@ -439,6 +448,12 @@ impl OTSDecision { self.embedding = Some(embedding); self } + + /// Set the causing `tool_call` id, linking this decision to its observation + pub fn with_cause_id(mut self, cause_id: impl Into) -> Self { + self.cause_id = Some(cause_id.into()); + self + } } #[cfg(test)] diff --git a/crates/temper-ots/src/models/enums.rs b/crates/temper-ots/src/models/enums.rs index 0604cfd2d..7aa375b3c 100644 --- a/crates/temper-ots/src/models/enums.rs +++ b/crates/temper-ots/src/models/enums.rs @@ -16,6 +16,28 @@ pub enum DecisionType { ResponseFormulation, } +impl DecisionType { + /// Whether a decision of this type names something the agent tried to + /// invoke, as opposed to something it thought or said. + /// + /// `choice.action` means different things per type. On a tool selection or + /// a parameter choice it is a callable name — a tool, or a governed + /// action — and the agent attempted to invoke it. On a reasoning step or a + /// response formulation it is free-form prose describing the thought + /// ("compare shipping options"), and nothing was invoked. + /// + /// Every consumer that reads `choice.action` as an invocation must ask + /// this first. Treating prose as an invocation fabricates environment + /// interactions in exported trajectories and reports imaginary actions in + /// conformance checks. + pub fn is_invocation(self) -> bool { + match self { + Self::ToolSelection | Self::ParameterChoice => true, + Self::ReasoningStep | Self::ResponseFormulation => false, + } + } +} + /// Trajectory outcome types #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] diff --git a/crates/temper-ots/src/models/trajectory.rs b/crates/temper-ots/src/models/trajectory.rs index 16be224f6..692d50171 100644 --- a/crates/temper-ots/src/models/trajectory.rs +++ b/crates/temper-ots/src/models/trajectory.rs @@ -60,6 +60,30 @@ pub struct OTSMetadata { /// Parent trajectory ID (for hierarchical traces) #[serde(skip_serializing_if = "Option::is_none")] pub parent_trajectory_id: Option, + + /// Hash or version of the actor spec this run executed under. + /// + /// Lets a consumer replay or score a trajectory against the exact + /// governing spec instead of whatever version is current. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub spec_version: Option, + + /// Harness that produced this trajectory (e.g. "temperpaw", "claude-code"). + /// + /// Distinct from `framework`: `framework` names the agent library, while + /// `harness` names the runner that drove the loop. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub harness: Option, + + /// Release identifier of the agent system itself (e.g. "temperpaw 3.2"). + /// + /// The build that ran, not the spec it ran under. Two runs of the same + /// build under different specs share this value; one build upgraded under + /// an unchanged spec changes it. `spec_version` answers the other + /// question, and the two are never interchangeable — a consumer grouping + /// runs by agent release must not be handed a spec hash. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub agent_version: Option, } impl OTSMetadata { @@ -86,6 +110,9 @@ impl OTSMetadata { human_reviewed: false, tags: Vec::new(), parent_trajectory_id: None, + spec_version: None, + harness: None, + agent_version: None, } } @@ -159,6 +186,24 @@ impl OTSMetadata { self.parent_trajectory_id = Some(parent_trajectory_id.into()); self } + + /// Set the spec version (hash or version of the governing actor spec) + pub fn with_spec_version(mut self, spec_version: impl Into) -> Self { + self.spec_version = Some(spec_version.into()); + self + } + + /// Set the harness that produced this trajectory + pub fn with_harness(mut self, harness: impl Into) -> Self { + self.harness = Some(harness.into()); + self + } + + /// Set the release identifier of the agent system that ran + pub fn with_agent_version(mut self, agent_version: impl Into) -> Self { + self.agent_version = Some(agent_version.into()); + self + } } /// System message at trajectory start diff --git a/crates/temper-ots/src/models/turn.rs b/crates/temper-ots/src/models/turn.rs index 12f456471..48cbbcfcf 100644 --- a/crates/temper-ots/src/models/turn.rs +++ b/crates/temper-ots/src/models/turn.rs @@ -10,8 +10,17 @@ use temper_runtime::scheduler::sim_uuid; /// One LLM interaction cycle /// -/// Contains messages and extracted decisions +/// Contains messages and extracted decisions. +/// +/// # The token-signal contract holds on every construction path +/// +/// The builder setters enforce completion-side alignment as they run, and +/// deserialization enforces the same contract through [`OTSTurnWire`]: a turn +/// that arrives over the wire misaligned is a deserialization error, not a +/// stored training sample. Direct struct literals are the one remaining path, +/// and they are inside this crate's control. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(try_from = "OTSTurnWire")] pub struct OTSTurn { /// Turn number in sequence pub turn_id: i32, @@ -45,6 +54,99 @@ pub struct OTSTurn { /// Decisions made in this turn #[serde(default, skip_serializing_if = "Vec::is_empty")] pub decisions: Vec, + + /// Token IDs of the prompt exactly as the serving stack tokenized it. + /// + /// RL consumers train on token IDs; re-tokenizing the rendered text + /// drifts from what the model actually saw. Populated only when the + /// serving stack exposes them, absent otherwise. + /// + /// Prompt-aligned, and the only prompt-side signal on the turn: nothing + /// else indexes into it. Prompt tokens are never trained on, so they carry + /// neither a mask nor log probabilities. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub prompt_token_ids: Option>, + + /// Token IDs of the completion exactly as the serving stack emitted them. + /// + /// The alignment anchor for the completion side: `response_mask` and + /// `logprobs` index into this array position for position. See + /// [`OTSTurn::validate_token_signals`]. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub completion_token_ids: Option>, + + /// Per-token loss mask over the completion: `1` for model-generated + /// tokens, `0` for tool output or otherwise injected tokens. + /// + /// A multi-turn completion interleaves what the model wrote with what the + /// environment injected back into the same response segment; the mask is + /// what tells them apart at training time. Completion-aligned: exactly one + /// entry per `completion_token_ids` entry, each `0` or `1`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub response_mask: Option>, + + /// Per-token log probabilities reported by the serving stack. + /// + /// Completion-aligned: exactly one entry per `completion_token_ids` entry. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub logprobs: Option>, +} + +/// The deserialization shape of [`OTSTurn`]. +/// +/// Exists so that every parsed turn passes through +/// [`OTSTurn::validate_token_signals`]. Without it, `serde` would build the +/// struct field by field and a misaligned upload would be accepted as valid +/// RL data. Field set, defaults, and names mirror [`OTSTurn`] exactly. +#[derive(Deserialize)] +struct OTSTurnWire { + turn_id: i32, + span_id: String, + #[serde(default)] + parent_span_id: Option, + timestamp: DateTime, + #[serde(default)] + duration_ms: Option, + #[serde(default)] + error: bool, + #[serde(default)] + turn_reward: Option, + #[serde(default)] + messages: Vec, + #[serde(default)] + decisions: Vec, + #[serde(default)] + prompt_token_ids: Option>, + #[serde(default)] + completion_token_ids: Option>, + #[serde(default)] + response_mask: Option>, + #[serde(default)] + logprobs: Option>, +} + +impl TryFrom for OTSTurn { + type Error = String; + + fn try_from(wire: OTSTurnWire) -> Result { + let turn = Self { + turn_id: wire.turn_id, + span_id: wire.span_id, + parent_span_id: wire.parent_span_id, + timestamp: wire.timestamp, + duration_ms: wire.duration_ms, + error: wire.error, + turn_reward: wire.turn_reward, + messages: wire.messages, + decisions: wire.decisions, + prompt_token_ids: wire.prompt_token_ids, + completion_token_ids: wire.completion_token_ids, + response_mask: wire.response_mask, + logprobs: wire.logprobs, + }; + turn.validate_token_signals()?; + Ok(turn) + } } impl OTSTurn { @@ -62,6 +164,10 @@ impl OTSTurn { turn_reward: None, messages: Vec::new(), decisions: Vec::new(), + prompt_token_ids: None, + completion_token_ids: None, + response_mask: None, + logprobs: None, } } @@ -118,94 +224,115 @@ impl OTSTurn { self.decisions = decisions; self } -} -#[cfg(test)] -mod tests { - use super::*; - use crate::models::{DecisionType, MessageRole, OTSChoice, OTSConsequence, OTSMessageContent}; - use temper_runtime::scheduler::sim_now; - - #[test] - fn test_turn_serialization() { - let timestamp = sim_now(); - let turn = OTSTurn::new(1, timestamp) - .with_duration_ms(150.5) - .with_turn_reward(0.85); - - let json_str = serde_json::to_string(&turn).unwrap(); - let parsed: OTSTurn = serde_json::from_str(&json_str).unwrap(); - - assert_eq!(parsed.turn_id, 1); - assert_eq!(parsed.duration_ms, Some(150.5)); - assert_eq!(parsed.turn_reward, Some(0.85)); - assert!(!parsed.error); - } - - #[test] - fn test_turn_with_messages_and_decisions() { - let timestamp = sim_now(); - let message = OTSMessage::new( - MessageRole::User, - OTSMessageContent::text("Hello"), - timestamp, - ); - let decision = OTSDecision::new( - DecisionType::ToolSelection, - OTSChoice::new("search"), - OTSConsequence::success(), - ); - - let turn = OTSTurn::new(1, timestamp) - .with_message(message) - .with_decision(decision); - - assert_eq!(turn.messages.len(), 1); - assert_eq!(turn.decisions.len(), 1); - - let json_str = serde_json::to_string(&turn).unwrap(); - let parsed: OTSTurn = serde_json::from_str(&json_str).unwrap(); - - assert_eq!(parsed.messages.len(), 1); - assert_eq!(parsed.decisions.len(), 1); + /// Set the prompt token IDs + pub fn with_prompt_token_ids(mut self, prompt_token_ids: Vec) -> Self { + self.prompt_token_ids = Some(prompt_token_ids); + self } - #[test] - fn test_turn_minimal() { - let timestamp = sim_now(); - let turn = OTSTurn::new(1, timestamp); - - let json_str = serde_json::to_string(&turn).unwrap(); - - // Optional fields should not appear - assert!(!json_str.contains("\"parent_span_id\"")); - assert!(!json_str.contains("\"duration_ms\"")); - assert!(!json_str.contains("\"turn_reward\"")); - - // Empty vectors should not appear - assert!(!json_str.contains("\"messages\"")); - assert!(!json_str.contains("\"decisions\"")); - - // Error defaults to false but should appear - assert!(json_str.contains("\"error\":false")); + /// Set the completion token IDs. + /// + /// Panics if a completion-aligned signal is already set at a different + /// length. + pub fn with_completion_token_ids(mut self, completion_token_ids: Vec) -> Self { + self.assert_completion_aligned("completion_token_ids", completion_token_ids.len()); + self.completion_token_ids = Some(completion_token_ids); + self } - #[test] - fn test_turn_with_error() { - let timestamp = sim_now(); - let turn = OTSTurn::new(1, timestamp).with_error(true); - - assert!(turn.error); + /// Set the per-token response mask (`1` = model token, `0` = injected token). + /// + /// Panics if any entry is outside `{0, 1}`, or if a completion-aligned + /// signal is already set at a different length. + pub fn with_response_mask(mut self, response_mask: Vec) -> Self { + if let Some(value) = response_mask.iter().find(|value| **value > 1) { + panic!("response_mask entries must be 0 or 1, got {value}"); + } + self.assert_completion_aligned("response_mask", response_mask.len()); + self.response_mask = Some(response_mask); + self + } - let json_str = serde_json::to_string(&turn).unwrap(); - assert!(json_str.contains("\"error\":true")); + /// Set the per-token log probabilities. + /// + /// Panics if a completion-aligned signal is already set at a different + /// length. + pub fn with_logprobs(mut self, logprobs: Vec) -> Self { + self.assert_completion_aligned("logprobs", logprobs.len()); + self.logprobs = Some(logprobs); + self } - #[test] - fn test_turn_with_parent_span() { - let timestamp = sim_now(); - let turn = OTSTurn::new(1, timestamp).with_parent_span_id("parent-span-123"); + /// Check that the completion-side signals agree. + /// + /// The setters catch every pairwise length disagreement whatever order + /// they are called in, but they cannot see a signal that is never set: + /// a mask or a logprob array with no `completion_token_ids` has nothing + /// to index into and is meaningless. This is the whole-turn check, and it + /// runs on three paths: [`TrajectoryBuilder::end_turn`](crate::TrajectoryBuilder) + /// before a turn is sealed, deserialization of any turn that arrives over + /// the wire, and any caller that wants to check a hand-built turn. + pub fn validate_token_signals(&self) -> Result<(), String> { + if let Some(response_mask) = &self.response_mask + && let Some(value) = response_mask.iter().find(|value| **value > 1) + { + return Err(format!( + "turn {} has a response_mask entry of {value}; the mask is a per-token loss \ + switch and every entry is 0 or 1", + self.turn_id + )); + } + let completion = self.completion_token_ids.as_ref().map(Vec::len); + for (name, len) in [ + ("response_mask", self.response_mask.as_ref().map(Vec::len)), + ("logprobs", self.logprobs.as_ref().map(Vec::len)), + ] { + let Some(len) = len else { continue }; + match completion { + None => { + return Err(format!( + "turn {} carries {name} with no completion_token_ids to align it against", + self.turn_id + )); + } + Some(completion) if completion != len => { + return Err(format!( + "turn {} has {len} {name} entries but {completion} completion_token_ids; \ + completion-side signals are aligned position for position", + self.turn_id + )); + } + Some(_) => {} + } + } + Ok(()) + } - assert_eq!(turn.parent_span_id, Some("parent-span-123".to_string())); + /// Panic when `len` disagrees with a completion-aligned signal that is + /// already set. Order-independent: whichever setter runs last sees at + /// least one counterpart and trips. + fn assert_completion_aligned(&self, name: &str, len: usize) { + for (other_name, other_len) in [ + ( + "completion_token_ids", + self.completion_token_ids.as_ref().map(Vec::len), + ), + ("response_mask", self.response_mask.as_ref().map(Vec::len)), + ("logprobs", self.logprobs.as_ref().map(Vec::len)), + ] { + if let Some(other_len) = other_len + && other_len != len + { + panic!( + "{name} has {len} entries but {other_name} has {other_len}; \ + completion-side signals are aligned position for position" + ); + } + } } } + +#[cfg(test)] +#[path = "turn_test.rs"] +mod tests; diff --git a/crates/temper-ots/src/models/turn_test.rs b/crates/temper-ots/src/models/turn_test.rs new file mode 100644 index 000000000..011d40460 --- /dev/null +++ b/crates/temper-ots/src/models/turn_test.rs @@ -0,0 +1,198 @@ +use super::*; +use crate::models::{DecisionType, MessageRole, OTSChoice, OTSConsequence, OTSMessageContent}; +use temper_runtime::scheduler::sim_now; + +#[test] +fn test_turn_serialization() { + let timestamp = sim_now(); + let turn = OTSTurn::new(1, timestamp) + .with_duration_ms(150.5) + .with_turn_reward(0.85); + + let json_str = serde_json::to_string(&turn).unwrap(); + let parsed: OTSTurn = serde_json::from_str(&json_str).unwrap(); + + assert_eq!(parsed.turn_id, 1); + assert_eq!(parsed.duration_ms, Some(150.5)); + assert_eq!(parsed.turn_reward, Some(0.85)); + assert!(!parsed.error); +} + +#[test] +fn test_turn_with_messages_and_decisions() { + let timestamp = sim_now(); + let message = OTSMessage::new( + MessageRole::User, + OTSMessageContent::text("Hello"), + timestamp, + ); + let decision = OTSDecision::new( + DecisionType::ToolSelection, + OTSChoice::new("search"), + OTSConsequence::success(), + ); + + let turn = OTSTurn::new(1, timestamp) + .with_message(message) + .with_decision(decision); + + assert_eq!(turn.messages.len(), 1); + assert_eq!(turn.decisions.len(), 1); + + let json_str = serde_json::to_string(&turn).unwrap(); + let parsed: OTSTurn = serde_json::from_str(&json_str).unwrap(); + + assert_eq!(parsed.messages.len(), 1); + assert_eq!(parsed.decisions.len(), 1); +} + +#[test] +fn test_turn_minimal() { + let timestamp = sim_now(); + let turn = OTSTurn::new(1, timestamp); + + let json_str = serde_json::to_string(&turn).unwrap(); + + // Optional fields should not appear + assert!(!json_str.contains("\"parent_span_id\"")); + assert!(!json_str.contains("\"duration_ms\"")); + assert!(!json_str.contains("\"turn_reward\"")); + + // Empty vectors should not appear + assert!(!json_str.contains("\"messages\"")); + assert!(!json_str.contains("\"decisions\"")); + + // Error defaults to false but should appear + assert!(json_str.contains("\"error\":false")); +} + +#[test] +fn test_turn_with_error() { + let timestamp = sim_now(); + let turn = OTSTurn::new(1, timestamp).with_error(true); + + assert!(turn.error); + + let json_str = serde_json::to_string(&turn).unwrap(); + assert!(json_str.contains("\"error\":true")); +} + +#[test] +fn completion_side_signals_must_agree_in_length() { + let timestamp = sim_now(); + let turn = OTSTurn::new(1, timestamp) + .with_completion_token_ids(vec![4, 5]) + .with_response_mask(vec![1, 0]) + .with_logprobs(vec![-0.1, -0.2]); + assert!(turn.validate_token_signals().is_ok()); +} + +#[test] +#[should_panic(expected = "response_mask has 3 entries but completion_token_ids has 2")] +fn a_mask_longer_than_the_completion_panics() { + let timestamp = sim_now(); + OTSTurn::new(1, timestamp) + .with_completion_token_ids(vec![4, 5]) + .with_response_mask(vec![1, 0, 1]); +} + +#[test] +#[should_panic(expected = "completion_token_ids has 2 entries but response_mask has 3")] +fn the_alignment_check_does_not_depend_on_setter_order() { + let timestamp = sim_now(); + OTSTurn::new(1, timestamp) + .with_response_mask(vec![1, 0, 1]) + .with_completion_token_ids(vec![4, 5]); +} + +#[test] +#[should_panic(expected = "response_mask entries must be 0 or 1, got 2")] +fn a_mask_outside_the_zero_one_domain_panics() { + let timestamp = sim_now(); + OTSTurn::new(1, timestamp).with_response_mask(vec![1, 2]); +} + +#[test] +fn a_signal_with_nothing_to_align_against_is_rejected_by_validation() { + let timestamp = sim_now(); + // The setters cannot catch this: there is no counterpart to compare + // against, so only the whole-turn check sees it. + let turn = OTSTurn::new(1, timestamp).with_response_mask(vec![1, 0]); + let error = turn + .validate_token_signals() + .expect_err("a mask with no completion tokens is meaningless"); + assert!(error.contains("no completion_token_ids"), "{error}"); +} + +#[test] +fn a_turn_with_no_token_signals_validates() { + let timestamp = sim_now(); + assert!( + OTSTurn::new(1, timestamp).validate_token_signals().is_ok(), + "token signals are optional; absent is not misaligned" + ); +} + +#[test] +fn deserialization_rejects_a_misaligned_turn() { + let json = serde_json::json!({ + "turn_id": 1, + "span_id": "span-1", + "timestamp": "2026-01-01T00:00:00Z", + "completion_token_ids": [4, 5], + "logprobs": [-0.1, -0.2, -0.3], + }); + let error = serde_json::from_value::(json) + .expect_err("a turn with three logprobs over two tokens is not valid RL data"); + assert!(error.to_string().contains("logprobs"), "{error}"); +} + +#[test] +fn deserialization_rejects_a_signal_with_nothing_to_align_against() { + let json = serde_json::json!({ + "turn_id": 1, + "span_id": "span-1", + "timestamp": "2026-01-01T00:00:00Z", + "response_mask": [1, 0], + }); + let error = serde_json::from_value::(json) + .expect_err("a mask with no completion tokens indexes into nothing"); + assert!( + error.to_string().contains("no completion_token_ids"), + "{error}" + ); +} + +#[test] +fn deserialization_rejects_a_mask_outside_the_zero_one_domain() { + let json = serde_json::json!({ + "turn_id": 1, + "span_id": "span-1", + "timestamp": "2026-01-01T00:00:00Z", + "completion_token_ids": [4, 5], + "response_mask": [1, 7], + }); + let error = serde_json::from_value::(json) + .expect_err("a mask entry of 7 is not a loss switch"); + assert!(error.to_string().contains("0 or 1"), "{error}"); +} + +#[test] +fn an_aligned_turn_round_trips_through_serde() { + let timestamp = sim_now(); + let turn = OTSTurn::new(1, timestamp) + .with_completion_token_ids(vec![4, 5]) + .with_response_mask(vec![1, 0]) + .with_logprobs(vec![-0.1, -0.2]); + let json = serde_json::to_string(&turn).expect("serialize"); + let parsed: OTSTurn = serde_json::from_str(&json).expect("aligned turns deserialize"); + assert_eq!(parsed, turn); +} + +#[test] +fn test_turn_with_parent_span() { + let timestamp = sim_now(); + let turn = OTSTurn::new(1, timestamp).with_parent_span_id("parent-span-123"); + + assert_eq!(turn.parent_span_id, Some("parent-span-123".to_string())); +} diff --git a/crates/temper-ots/tests/atif_export.rs b/crates/temper-ots/tests/atif_export.rs new file mode 100644 index 000000000..1a7629be0 --- /dev/null +++ b/crates/temper-ots/tests/atif_export.rs @@ -0,0 +1,498 @@ +//! Golden tests for the OTS -> ATIF v1.7 export. +//! +//! Three shapes are pinned: +//! +//! 1. A fully populated trajectory — every OTS field the mapping touches is +//! set, and the resulting ATIF document is asserted field by field. +//! 2. A minimal trajectory — nothing optional is set beyond the one step ATIF +//! requires, and the export must still be a valid ATIF document with the +//! optional keys absent rather than present-and-null. +//! 3. A trajectory with nothing in it at all, which is not exportable: ATIF +//! v1.7 models a trajectory as its step list and has no valid stepless +//! document. + +use temper_ots::atif::{ATIF_SCHEMA_VERSION, AtifExportError, UNKNOWN_AGENT_VERSION, to_atif}; +use temper_ots::models::{ + ContentType, DecisionType, MessageRole, OTSAlternative, OTSChoice, OTSConsequence, OTSContext, + OTSCreditAssignment, OTSDecision, OTSDecisionEvaluation, OTSEntity, OTSMessage, + OTSMessageContent, OTSMetadata, OTSSystemMessage, OTSTrajectory, OTSTurn, OutcomeType, +}; + +fn at(offset_seconds: i64) -> chrono::DateTime { + "2026-04-28T10:00:00Z" + .parse::>() + .expect("fixture timestamp parses") + + chrono::Duration::seconds(offset_seconds) +} + +/// A trajectory with every mapped OTS field populated. +fn full_trajectory() -> OTSTrajectory { + let metadata = OTSMetadata::new( + "Refund the customer's last order", + "agent-support-1", + OutcomeType::Success, + at(0), + ) + .with_domain("customer_support") + .with_timestamp_end(at(30)) + .with_duration_ms(30_000.0) + .with_framework("langchain") + .with_harness("temperpaw") + .with_agent_version("temperpaw 3.2") + .with_spec_version("sha256:9f2c") + .with_environment("production") + .with_feedback_score(0.9) + .with_human_reviewed(true) + .with_tag("high_priority") + .with_parent_trajectory_id("traj-parent-0001"); + + let decision = OTSDecision::new( + DecisionType::ToolSelection, + OTSChoice::new("RefundOrder") + .with_arguments(serde_json::json!({"order_id": "order-7"})) + .with_rationale("The order is already returned") + .with_confidence(0.95), + OTSConsequence::success().with_result_summary("Refund issued: $42.00"), + ) + .with_decision_id("decision-1") + .with_cause_id("call-abc") + .with_alternatives( + "tools", + vec![OTSAlternative::new("CancelOrder").with_rejected_reason("Order already shipped")], + ) + .with_evaluation(OTSDecisionEvaluation::new("model_eval", 0.9)) + .with_credit_assignment(OTSCreditAssignment::new(0.8).with_pivotal(true)); + + let turn = OTSTurn::new(1, at(5)) + .with_span_id("span-1") + .with_parent_span_id("span-0") + .with_duration_ms(1_250.0) + .with_turn_reward(0.75) + .with_message( + OTSMessage::new( + MessageRole::User, + OTSMessageContent::text("Please refund my order"), + at(5), + ) + .with_message_id("message-user-1"), + ) + .with_message( + OTSMessage::new( + MessageRole::Assistant, + OTSMessageContent::text("Issuing the refund now."), + at(6), + ) + .with_message_id("message-assistant-1") + .with_reasoning("The order is in Returned, so RefundOrder is legal."), + ) + .with_message( + OTSMessage::new( + MessageRole::Assistant, + OTSMessageContent::widget(serde_json::json!({"kind": "refund_receipt"})), + at(7), + ) + .with_message_id("message-assistant-2"), + ) + .with_message( + OTSMessage::new( + MessageRole::Tool, + OTSMessageContent::tool_response(serde_json::json!({"status": "ok"})), + at(8), + ) + .with_message_id("message-tool-1"), + ) + .with_decision(decision) + .with_prompt_token_ids(vec![10, 11, 12]) + .with_completion_token_ids(vec![20, 21]) + .with_response_mask(vec![1, 1]) + .with_logprobs(vec![-0.1, -0.2]); + + OTSTrajectory::new(metadata) + .with_trajectory_id("traj-full-0001") + .with_context(OTSContext::new().with_entity(OTSEntity::new("tool", "refund"))) + .with_system_message(OTSSystemMessage::new("You are a support agent", at(0))) + .with_turn(turn) + .with_final_reward(0.95) +} + +#[test] +fn full_trajectory_exports_field_by_field() { + let atif = to_atif(&full_trajectory(), Some("session-42")).expect("export"); + + // -- Root ------------------------------------------------------------ + assert_eq!(atif.schema_version, ATIF_SCHEMA_VERSION); + assert_eq!(atif.schema_version, "ATIF-v1.7"); + assert_eq!(atif.session_id.as_deref(), Some("session-42")); + assert_eq!(atif.trajectory_id.as_deref(), Some("traj-full-0001")); + assert!( + atif.subagent_trajectories.is_none(), + "OTS points child -> parent, so a single document embeds no subagents" + ); + + // -- Agent ----------------------------------------------------------- + assert_eq!( + atif.agent.name, "temperpaw", + "harness names the agent system and outranks framework" + ); + assert_eq!( + atif.agent.version, "temperpaw 3.2", + "agent.version is the agent system's release, never the spec hash" + ); + assert!( + atif.agent.model_name.is_none(), + "OTS records no model identity" + ); + assert_eq!(atif.agent.extra["temper.agent_id"], "agent-support-1"); + assert_eq!(atif.agent.extra["temper.framework"], "langchain"); + assert_eq!(atif.agent.extra["temper.harness"], "temperpaw"); + assert_eq!(atif.agent.extra["temper.spec_version"], "sha256:9f2c"); + + // -- Steps ----------------------------------------------------------- + assert_eq!(atif.steps.len(), 3); + + let system = &atif.steps[0]; + assert_eq!(system.step_id, 1); + assert_eq!( + serde_json::to_value(system.source).unwrap(), + serde_json::json!("system") + ); + assert_eq!(system.message, "You are a support agent"); + assert_eq!( + system.timestamp.as_deref(), + Some("2026-04-28T10:00:00+00:00") + ); + assert!(system.tool_calls.is_empty()); + assert!(system.observation.is_none()); + assert!(system.metrics.is_none()); + + let user = &atif.steps[1]; + assert_eq!(user.step_id, 2); + assert_eq!( + serde_json::to_value(user.source).unwrap(), + serde_json::json!("user") + ); + assert_eq!(user.message, "Please refund my order"); + assert_eq!(user.timestamp.as_deref(), Some("2026-04-28T10:00:05+00:00")); + + let agent = &atif.steps[2]; + assert_eq!(agent.step_id, 3); + assert_eq!( + serde_json::to_value(agent.source).unwrap(), + serde_json::json!("agent") + ); + assert_eq!(agent.message, "Issuing the refund now."); + assert_eq!( + agent.reasoning_content.as_deref(), + Some("The order is in Returned, so RefundOrder is legal.") + ); + assert_eq!( + agent.timestamp.as_deref(), + Some("2026-04-28T10:00:05+00:00") + ); + + // -- Tool calls ------------------------------------------------------ + assert_eq!(agent.tool_calls.len(), 1); + let call = &agent.tool_calls[0]; + assert_eq!( + call.tool_call_id, "call-abc", + "cause_id is the provider tool-call id and wins over decision_id" + ); + assert_eq!(call.function_name, "RefundOrder"); + assert_eq!(call.arguments, serde_json::json!({"order_id": "order-7"})); + + // -- Observation ----------------------------------------------------- + let observation = agent.observation.as_ref().expect("observation"); + assert_eq!(observation.results.len(), 2); + assert_eq!( + observation.results[0].source_call_id.as_deref(), + Some("call-abc") + ); + assert_eq!( + observation.results[0].content.as_deref(), + Some("Refund issued: $42.00") + ); + assert_eq!( + observation.results[0].extra["temper.consequence"], + serde_json::json!({"success": true, "error_type": null}) + ); + assert!( + observation.results[1].source_call_id.is_none(), + "OTS tool messages carry no call linkage" + ); + assert_eq!( + observation.results[1].content.as_deref(), + Some(r#"{"status":"ok"}"#) + ); + + // -- Metrics --------------------------------------------------------- + let metrics = agent.metrics.as_ref().expect("metrics"); + assert_eq!(metrics.prompt_token_ids.as_deref(), Some(&[10, 11, 12][..])); + assert_eq!(metrics.completion_token_ids.as_deref(), Some(&[20, 21][..])); + assert_eq!(metrics.logprobs.as_deref(), Some(&[-0.1, -0.2][..])); + assert_eq!( + metrics.prompt_tokens, + Some(3), + "counts are derived from the token-id arrays" + ); + assert_eq!(metrics.completion_tokens, Some(2)); + assert!(metrics.cost_usd.is_none(), "OTS records no cost"); + assert_eq!( + metrics.extra["temper.response_mask"], + serde_json::json!([1, 1]) + ); + assert_eq!(metrics.extra["temper.turn_reward"], serde_json::json!(0.75)); + + // -- Step extra ------------------------------------------------------ + assert_eq!(agent.extra["temper.turn_id"], serde_json::json!(1)); + assert_eq!(agent.extra["temper.span_id"], "span-1"); + assert_eq!(agent.extra["temper.parent_span_id"], "span-0"); + assert_eq!( + agent.extra["temper.duration_ms"], + serde_json::json!(1_250.0) + ); + assert!( + !agent.extra.contains_key("temper.error"), + "a clean turn records no error flag" + ); + + let decisions = agent.extra["temper.decisions"] + .as_array() + .expect("decisions array"); + assert_eq!(decisions.len(), 1); + assert_eq!(decisions[0]["decision_id"], "decision-1"); + assert_eq!(decisions[0]["decision_type"], "tool_selection"); + assert_eq!( + decisions[0]["choice"]["confidence"], + serde_json::json!(0.95) + ); + assert_eq!( + decisions[0]["choice"]["rationale"], + "The order is already returned" + ); + assert_eq!( + decisions[0]["alternatives"]["tools"][0]["action"], + "CancelOrder" + ); + assert_eq!(decisions[0]["evaluation"]["evaluator_id"], "model_eval"); + assert_eq!( + decisions[0]["credit_assignment"]["impact"], + serde_json::json!(0.8) + ); + + let assistant_content = agent.extra["temper.assistant_content"] + .as_array() + .expect("assistant content array"); + assert_eq!( + assistant_content.len(), + 1, + "only non-text assistant payloads land in extra; text is the step message" + ); + assert_eq!( + assistant_content[0]["type"], + serde_json::to_value(ContentType::Widget).unwrap() + ); + + // -- Final metrics --------------------------------------------------- + let final_metrics = atif.final_metrics.as_ref().expect("final metrics"); + assert_eq!(final_metrics.total_prompt_tokens, Some(3)); + assert_eq!(final_metrics.total_completion_tokens, Some(2)); + assert_eq!(final_metrics.total_steps, Some(3)); + assert!(final_metrics.total_cost_usd.is_none()); + + // -- Root extra ------------------------------------------------------ + assert_eq!(atif.extra["temper.ots_version"], "0.1.0"); + assert_eq!(atif.extra["temper.final_reward"], serde_json::json!(0.95)); + let metadata = &atif.extra["temper.metadata"]; + assert_eq!( + metadata["task_description"], + "Refund the customer's last order" + ); + assert_eq!(metadata["outcome"], "success"); + assert_eq!(metadata["spec_version"], "sha256:9f2c"); + assert_eq!(metadata["harness"], "temperpaw"); + assert_eq!(metadata["domain"], "customer_support"); + assert_eq!(metadata["environment"], "production"); + assert_eq!(metadata["feedback_score"], serde_json::json!(0.9)); + assert_eq!(metadata["human_reviewed"], serde_json::json!(true)); + assert_eq!(metadata["tags"], serde_json::json!(["high_priority"])); + assert_eq!(metadata["parent_trajectory_id"], "traj-parent-0001"); + assert_eq!(atif.extra["temper.context"]["entities"][0]["id"], "refund"); +} + +#[test] +fn minimal_trajectory_exports_valid_atif_with_optionals_absent() { + let metadata = OTSMetadata::new("minimal task", "agent-2", OutcomeType::Failure, at(0)); + // One step is the ATIF floor, so the smallest exportable trajectory is a + // system message and nothing else. + let trajectory = OTSTrajectory::new(metadata) + .with_trajectory_id("traj-minimal-0001") + .with_system_message(OTSSystemMessage::new("be brief", at(0))); + + let atif = to_atif(&trajectory, None).expect("a system message is a step"); + assert_eq!(atif.schema_version, "ATIF-v1.7"); + assert!(atif.session_id.is_none()); + assert_eq!(atif.steps.len(), 1); + assert_eq!( + atif.agent.name, "agent-2", + "with neither harness nor framework the agent id names the system" + ); + assert_eq!( + atif.agent.version, UNKNOWN_AGENT_VERSION, + "an unreported agent release is unknown, not the OTS format version" + ); + + let json = serde_json::to_value(&atif).expect("serialize ATIF"); + let object = json.as_object().expect("ATIF root is an object"); + assert_eq!(object["schema_version"], "ATIF-v1.7"); + assert_eq!(object["trajectory_id"], "traj-minimal-0001"); + assert_eq!( + object["steps"].as_array().expect("steps").len(), + 1, + "ATIF v1.7 requires at least one step" + ); + + // Optional keys must be absent, not present-and-null. + for absent in ["session_id", "subagent_trajectories"] { + assert!( + !object.contains_key(absent), + "unset optional `{absent}` must be omitted, not null" + ); + } + assert!( + !object["agent"] + .as_object() + .unwrap() + .contains_key("model_name") + ); + + // A trajectory with no turns still reports its step total. + let final_metrics = &object["final_metrics"]; + assert_eq!(final_metrics["total_steps"], serde_json::json!(1)); + assert!( + !final_metrics + .as_object() + .unwrap() + .contains_key("total_prompt_tokens"), + "no turn carried token ids, so no total is invented" + ); + + // No context and no final reward: neither key is fabricated. + let extra = object["extra"].as_object().expect("root extra"); + assert!(!extra.contains_key("temper.context")); + assert!(!extra.contains_key("temper.final_reward")); + assert_eq!(extra["temper.ots_version"], "0.1.0"); +} + +#[test] +fn a_trajectory_with_no_steps_is_refused_rather_than_exported_stepless() { + let metadata = OTSMetadata::new("nothing ran", "agent-2", OutcomeType::Failure, at(0)); + let trajectory = OTSTrajectory::new(metadata).with_trajectory_id("traj-empty-0001"); + + assert_eq!( + to_atif(&trajectory, Some("session-empty")), + Err(AtifExportError::NoSteps), + "a session finalized before anything ran has no valid ATIF document" + ); +} + +#[test] +fn thinking_decisions_do_not_become_environment_interactions() { + let metadata = OTSMetadata::new("think then act", "agent-4", OutcomeType::Success, at(0)); + let reasoning = OTSDecision::new( + DecisionType::ReasoningStep, + OTSChoice::new("compare shipping options"), + OTSConsequence::success(), + ) + .with_decision_id("decision-think"); + let response = OTSDecision::new( + DecisionType::ResponseFormulation, + OTSChoice::new("apologise and offer a refund"), + OTSConsequence::success(), + ) + .with_decision_id("decision-say"); + let invocation = OTSDecision::new( + DecisionType::ParameterChoice, + OTSChoice::new("ShipOrder").with_arguments(serde_json::json!({"carrier": "dhl"})), + OTSConsequence::success().with_result_summary("Shipped"), + ) + .with_decision_id("decision-ship"); + + let trajectory = OTSTrajectory::new(metadata) + .with_trajectory_id("traj-thinking-0001") + .with_turn( + OTSTurn::new(1, at(1)) + .with_span_id("span-1") + .with_decision(reasoning) + .with_decision(response) + .with_decision(invocation), + ); + + let atif = to_atif(&trajectory, None).expect("export"); + let step = &atif.steps[0]; + let called: Vec<&str> = step + .tool_calls + .iter() + .map(|call| call.function_name.as_str()) + .collect(); + assert_eq!( + called, + vec!["ShipOrder"], + "only tool selections and parameter choices name a callable" + ); + let results = &step.observation.as_ref().expect("observation").results; + assert_eq!( + results.len(), + 1, + "a thought produces no environment observation" + ); + assert_eq!(results[0].source_call_id.as_deref(), Some("decision-ship")); + assert_eq!( + step.extra["temper.decisions"] + .as_array() + .expect("decisions") + .len(), + 3, + "every decision is still carried verbatim in extra" + ); +} + +#[test] +fn errored_turn_and_fallback_call_id_are_recorded() { + let metadata = OTSMetadata::new("failing task", "agent-3", OutcomeType::Failure, at(0)); + let decision = OTSDecision::new( + DecisionType::ToolSelection, + OTSChoice::new("ShipOrder"), + OTSConsequence::failure().with_error_type("IllegalTransition"), + ) + .with_decision_id("decision-9"); + let turn = OTSTurn::new(1, at(1)) + .with_span_id("span-9") + .with_error(true) + .with_decision(decision); + let trajectory = OTSTrajectory::new(metadata) + .with_trajectory_id("traj-error-0001") + .with_turn(turn); + + let atif = to_atif(&trajectory, Some("session-9")).expect("export"); + assert_eq!(atif.steps.len(), 1); + let step = &atif.steps[0]; + assert_eq!(step.message, "", "no assistant text means an empty message"); + assert_eq!(step.extra["temper.error"], serde_json::json!(true)); + assert_eq!( + step.tool_calls[0].tool_call_id, "decision-9", + "without cause_id the decision id keeps the call correlated" + ); + assert_eq!(step.tool_calls[0].arguments, serde_json::json!({})); + + let results = &step.observation.as_ref().expect("observation").results; + assert_eq!(results[0].source_call_id.as_deref(), Some("decision-9")); + assert!(results[0].content.is_none()); + assert_eq!( + results[0].extra["temper.consequence"], + serde_json::json!({"success": false, "error_type": "IllegalTransition"}) + ); + assert!( + step.metrics.is_none(), + "a turn with no token signal and no reward emits no metrics block" + ); +} diff --git a/crates/temper-ots/tests/fixtures/trajectory_pre_jcs.json b/crates/temper-ots/tests/fixtures/trajectory_pre_jcs.json new file mode 100644 index 000000000..abc90991f --- /dev/null +++ b/crates/temper-ots/tests/fixtures/trajectory_pre_jcs.json @@ -0,0 +1,76 @@ +{ + "trajectory_id": "traj-pre-jcs-0001", + "version": "0.1.0", + "metadata": { + "task_description": "Refund the duplicate charge on invoice INV-42", + "domain": "customer_support", + "timestamp_start": "2026-04-28T10:00:00Z", + "timestamp_end": "2026-04-28T10:00:12Z", + "duration_ms": 12000.0, + "agent_id": "agent-support-1", + "framework": "temper", + "environment": "production", + "outcome": "success", + "feedback_score": 0.9, + "human_reviewed": false, + "tags": ["billing", "refund"], + "parent_trajectory_id": "traj-parent-0001" + }, + "context": { + "user": { + "id": "user-7", + "handle": "alice" + } + }, + "system_message": { + "content": "You are a billing support agent.", + "timestamp": "2026-04-28T10:00:00Z" + }, + "turns": [ + { + "turn_id": 1, + "span_id": "span-0001", + "timestamp": "2026-04-28T10:00:01Z", + "duration_ms": 4200.0, + "error": false, + "turn_reward": 0.8, + "messages": [ + { + "message_id": "msg-0001", + "role": "user", + "timestamp": "2026-04-28T10:00:01Z", + "content": { + "type": "text", + "text": "I was charged twice for invoice INV-42." + } + } + ], + "decisions": [ + { + "decision_id": "dec-0001", + "decision_type": "tool_selection", + "state": { + "context_summary": "Duplicate charge reported", + "available_actions": ["lookup_invoice", "issue_refund"] + }, + "choice": { + "action": "lookup_invoice", + "arguments": { + "invoice_id": "INV-42" + }, + "confidence": 0.95 + }, + "consequence": { + "success": true, + "result_summary": "Found two charges" + }, + "credit_assignment": { + "impact": 0.6, + "pivotal": true + } + } + ] + } + ], + "final_reward": 0.95 +} diff --git a/crates/temper-ots/tests/schema_compat.rs b/crates/temper-ots/tests/schema_compat.rs new file mode 100644 index 000000000..adddb64eb --- /dev/null +++ b/crates/temper-ots/tests/schema_compat.rs @@ -0,0 +1,192 @@ +//! Schema-compatibility tests for the JCS trajectory-core OTS extensions. +//! +//! The extension fields (`spec_version`/`harness`/`agent_version` on +//! metadata, token IDs, response mask and logprobs on turns, `cause_id` on +//! decisions) are optional and additive. Two properties must hold for every stored row: +//! +//! 1. Rows written before the extension still deserialize (back-compat). +//! 2. A trajectory carrying every extension field round-trips losslessly. + +use temper_ots::models::{ + DecisionType, MessageRole, OTSChoice, OTSConsequence, OTSContext, OTSDecision, OTSMessage, + OTSMessageContent, OTSMetadata, OTSSystemMessage, OTSTrajectory, OTSTurn, OutcomeType, +}; + +/// A trajectory serialized before the JCS extension fields existed. +const PRE_JCS_TRAJECTORY: &str = include_str!("fixtures/trajectory_pre_jcs.json"); + +fn fixed_timestamp() -> chrono::DateTime { + "2026-04-28T10:00:00Z" + .parse::>() + .expect("fixture timestamp parses") +} + +#[test] +fn pre_extension_trajectory_still_deserializes() { + let trajectory: OTSTrajectory = + serde_json::from_str(PRE_JCS_TRAJECTORY).expect("old-shape trajectory must still parse"); + + assert_eq!(trajectory.trajectory_id, "traj-pre-jcs-0001"); + assert_eq!(trajectory.version, "0.1.0"); + assert_eq!(trajectory.metadata.agent_id, "agent-support-1"); + assert_eq!(trajectory.metadata.outcome, OutcomeType::Success); + assert_eq!(trajectory.turns.len(), 1); + assert_eq!(trajectory.turns[0].decisions.len(), 1); + + // Every extension field is absent, not defaulted to a wrong value. + assert!(trajectory.metadata.spec_version.is_none()); + assert!(trajectory.metadata.harness.is_none()); + assert!(trajectory.metadata.agent_version.is_none()); + assert!(trajectory.turns[0].prompt_token_ids.is_none()); + assert!(trajectory.turns[0].completion_token_ids.is_none()); + assert!(trajectory.turns[0].response_mask.is_none()); + assert!(trajectory.turns[0].logprobs.is_none()); + assert!(trajectory.turns[0].decisions[0].cause_id.is_none()); +} + +#[test] +fn pre_extension_trajectory_reserializes_without_extension_keys() { + let trajectory: OTSTrajectory = + serde_json::from_str(PRE_JCS_TRAJECTORY).expect("old-shape trajectory must still parse"); + let reserialized = serde_json::to_string(&trajectory).expect("serialize"); + + for absent_key in [ + "spec_version", + "harness", + "agent_version", + "prompt_token_ids", + "completion_token_ids", + "response_mask", + "logprobs", + "cause_id", + ] { + assert!( + !reserialized.contains(absent_key), + "unset extension field '{absent_key}' must not be emitted: {reserialized}" + ); + } +} + +#[test] +fn fully_populated_extension_fields_round_trip() { + let now = fixed_timestamp(); + + let metadata = OTSMetadata::new( + "Run the governed refund workflow", + "agent-support-1", + OutcomeType::Success, + now, + ) + .with_spec_version("sha256:9f1c2d") + .with_harness("temperpaw") + .with_agent_version("temperpaw 3.2"); + + let decision = OTSDecision::new( + DecisionType::ToolSelection, + OTSChoice::new("lookup_invoice").with_confidence(0.95), + OTSConsequence::success().with_result_summary("Found two charges"), + ) + .with_decision_id("dec-0001") + .with_cause_id("toolu_01ABCDEF"); + + let turn = OTSTurn::new(1, now) + .with_span_id("span-0001") + .with_message(OTSMessage::new( + MessageRole::Assistant, + OTSMessageContent::text("Looking up the invoice."), + now, + )) + .with_decision(decision) + .with_prompt_token_ids(vec![128000, 9906, 1917]) + // Completion-side signals are aligned position for position: four + // completion tokens, four mask entries, four log probabilities. The + // `0` marks a token the environment injected into the response. + .with_completion_token_ids(vec![40, 2846, 389, 433]) + .with_response_mask(vec![1, 1, 0, 1]) + .with_logprobs(vec![-0.01, -0.42, -1.75, -0.03]); + + let trajectory = OTSTrajectory::new(metadata) + .with_trajectory_id("traj-jcs-0001") + .with_context(OTSContext::new()) + .with_system_message(OTSSystemMessage::new("You are a billing agent.", now)) + .with_turn(turn) + .with_final_reward(0.95); + + let json = serde_json::to_string(&trajectory).expect("serialize"); + let parsed: OTSTrajectory = serde_json::from_str(&json).expect("deserialize"); + + assert_eq!(parsed, trajectory, "round-trip must be lossless"); + assert_eq!( + parsed.metadata.spec_version.as_deref(), + Some("sha256:9f1c2d") + ); + assert_eq!(parsed.metadata.harness.as_deref(), Some("temperpaw")); + assert_eq!( + parsed.metadata.agent_version.as_deref(), + Some("temperpaw 3.2"), + "the agent release is a field of its own, never folded into spec_version" + ); + + let parsed_turn = &parsed.turns[0]; + assert_eq!( + parsed_turn.prompt_token_ids.as_deref(), + Some([128000u32, 9906, 1917].as_ref()) + ); + assert_eq!( + parsed_turn.completion_token_ids.as_deref(), + Some([40u32, 2846, 389, 433].as_ref()) + ); + assert_eq!( + parsed_turn.response_mask.as_deref(), + Some([1u8, 1, 0, 1].as_ref()), + "completion-aligned: one entry per completion token, 1 for model \ + tokens and 0 for tool or otherwise injected tokens" + ); + assert_eq!( + parsed_turn.logprobs.as_deref(), + Some([-0.01f64, -0.42, -1.75, -0.03].as_ref()) + ); + assert_eq!( + parsed_turn.decisions[0].cause_id.as_deref(), + Some("toolu_01ABCDEF"), + "cause_id links the decision to the observation it produced" + ); +} + +#[test] +fn extension_field_names_match_the_cross_repo_contract() { + // temper and temperpaw must agree on the wire names verbatim; a rename on + // either side silently drops data instead of failing loudly. + let now = fixed_timestamp(); + let metadata = OTSMetadata::new("contract", "agent", OutcomeType::Success, now) + .with_spec_version("v1") + .with_harness("claude-code") + .with_agent_version("claude-code 2.4"); + let turn = OTSTurn::new(1, now) + .with_decision( + OTSDecision::new( + DecisionType::ToolSelection, + OTSChoice::new("search"), + OTSConsequence::success(), + ) + .with_cause_id("toolu_1"), + ) + .with_prompt_token_ids(vec![1]) + .with_completion_token_ids(vec![2]) + .with_response_mask(vec![1]) + .with_logprobs(vec![-0.5]); + let trajectory = OTSTrajectory::new(metadata).with_turn(turn); + + let json = serde_json::to_value(&trajectory).expect("serialize"); + let metadata_json = &json["metadata"]; + assert_eq!(metadata_json["spec_version"], "v1"); + assert_eq!(metadata_json["harness"], "claude-code"); + assert_eq!(metadata_json["agent_version"], "claude-code 2.4"); + + let turn_json = &json["turns"][0]; + assert_eq!(turn_json["prompt_token_ids"], serde_json::json!([1])); + assert_eq!(turn_json["completion_token_ids"], serde_json::json!([2])); + assert_eq!(turn_json["response_mask"], serde_json::json!([1])); + assert_eq!(turn_json["logprobs"], serde_json::json!([-0.5])); + assert_eq!(turn_json["decisions"][0]["cause_id"], "toolu_1"); +} diff --git a/crates/temper-platform/Cargo.toml b/crates/temper-platform/Cargo.toml index f29554c8e..365e52958 100644 --- a/crates/temper-platform/Cargo.toml +++ b/crates/temper-platform/Cargo.toml @@ -10,7 +10,7 @@ description = "Dogfooded hosting platform for Temper — verify-and-deploy pipel temper-spec = { workspace = true } temper-verify = { workspace = true } temper-jit = { workspace = true } -temper-server = { workspace = true } +temper-server = { workspace = true, features = ["observe"] } temper-store-turso = { workspace = true } temper-observe = { workspace = true } temper-runtime = { workspace = true } @@ -32,6 +32,8 @@ sha2 = { workspace = true } toml = { workspace = true } async-trait = { workspace = true } base64 = "0.22" +futures = { workspace = true } +tempfile = "3.27.0" [dev-dependencies] temper-codegen = { workspace = true } @@ -39,4 +41,3 @@ tokio-test = { workspace = true } tower = { workspace = true } hyper = { workspace = true } wiremock = { workspace = true } -tempfile = "3.27.0" diff --git a/crates/temper-platform/src/bearer_auth.rs b/crates/temper-platform/src/bearer_auth.rs index ea5ce2b2c..3cae859b9 100644 --- a/crates/temper-platform/src/bearer_auth.rs +++ b/crates/temper-platform/src/bearer_auth.rs @@ -1,384 +1,193 @@ -//! Bearer token authentication middleware. +//! Tenant-scoped bearer authentication middleware. //! -//! Every non-health-check request must include `Authorization: Bearer `. -//! The middleware resolves agent credentials first, then falls back to the -//! global `TEMPER_API_KEY` for admin/operator access. -//! -//! See ADR-0033: Platform-Assigned Agent Identity. +//! Every protected request resolves a bearer credential in the requested +//! tenant and receives one typed [`temper_authz::AuthenticatedRequestContext`]. +//! `TEMPER_API_KEY` may bootstrap a normal tenant credential, but it has no +//! special runtime fallback or deployment-wide Admin authority. use crate::state::PlatformState; use axum::extract::{Request, State}; -use axum::http::StatusCode; +use axum::http::{Method, StatusCode}; use axum::middleware::Next; -use axum::response::Response; +use axum::response::{IntoResponse, Response}; +use base64::Engine as _; use temper_runtime::tenant::TenantId; -/// Marker extension for requests that were already authenticated by an -/// outer layer and have trusted principal headers injected server-side. -#[derive(Debug, Clone, Copy)] -pub struct PreAuthenticatedRequest; +const BASIC_CREDENTIAL_DECODE_BUDGET: usize = 8 * 1024; -/// Axum middleware that validates Bearer token authentication and resolves -/// agent identity from credentials. -/// -/// Resolution order: -/// 1. Health check paths → passthrough (no auth needed) -/// 2. No `api_token` configured → passthrough (local dev mode) -/// 3. Try agent credential resolution → if match, set `ResolvedIdentity` extension -/// 4. Try global `TEMPER_API_KEY` match → admin/operator access -/// 5. No match → 401 Unauthorized +/// Resolve the request's bearer credential and attach typed authority. pub async fn bearer_auth_check( State(state): State, mut req: Request, next: Next, ) -> Result { - // Allow health checks without auth (Railway probes these paths). - if req.method() == axum::http::Method::GET - && (req.uri().path() == "/tdata" || req.uri().path() == "/healthz") - { - return Ok(next.run(req).await); - } + let bearer = bearer_credential(&req); + let tenant = requested_tenant(&req)?; - // Allow identity resolution endpoint without auth — the token in the - // request body IS the credential being resolved (self-resolving). - if req.method() == axum::http::Method::POST && req.uri().path() == "/api/identity/resolve" { + if bearer + .as_deref() + .is_some_and(temper_server::internal_invocation::is_internal_invocation_bearer) + { + let token = bearer.as_deref().ok_or(StatusCode::UNAUTHORIZED)?; + let authenticated = state + .server + .internal_invocation_credentials + .consume_for_request(token, &tenant, req.method(), req.uri()) + .map_err(|_| StatusCode::UNAUTHORIZED)?; + // The capability has already been consumed; do not expose it to + // downstream handlers, logs, or accidental forwarding. + req.headers_mut().remove("authorization"); + req.extensions_mut().insert(authenticated); return Ok(next.run(req).await); } - let Some(ref _expected) = state.api_token else { - // No API key configured — passthrough (local dev mode). - return Ok(next.run(req).await); - }; - - if req.extensions().get::().is_some() - && req.headers().contains_key("x-temper-principal-kind") - && req.headers().contains_key("x-temper-principal-id") - { + if is_public_request(&req) { + if !req.uri().path().starts_with("/webhooks/") { + req.headers_mut().remove("authorization"); + } return Ok(next.run(req).await); } - let Some(auth_header) = req.headers().get("authorization") else { - return Err(StatusCode::UNAUTHORIZED); - }; - - let auth_str = auth_header.to_str().map_err(|_| StatusCode::UNAUTHORIZED)?; - - let Some(token) = auth_str.strip_prefix("Bearer ") else { - return Err(StatusCode::UNAUTHORIZED); + let matched_endpoint = match state.server.http_endpoint_tables.get(&tenant).await { + Some(table) => { + table + .match_request(req.method().as_str(), req.uri().path()) + .await + } + None => None, }; + let request_method = req.method().as_str().to_string(); + let request_path = req.uri().path().to_string(); - let has_explicit_principal = req.headers().contains_key("x-temper-principal-kind") - && req.headers().contains_key("x-temper-principal-id"); - let matches_global_api_key = state - .api_token - .as_ref() - .is_some_and(|expected| constant_time_eq(token.as_bytes(), expected.as_bytes())); - - // ADR-0043 guest override path: internal loopback callers may present the - // platform API key while explicitly declaring the principal they are acting - // as. Preserve those headers instead of collapsing the request into the - // bootstrapped operator credential. - if matches_global_api_key && has_explicit_principal { - return Ok(next.run(req).await); - } - - // Step 1: Try to resolve as an agent credential. - let tenant = extract_tenant(&req); - if let Some(identity) = state - .identity_resolver - .resolve(&state.server, &tenant, token) - .await + if let Some(token) = request_credential(&req) + && let Some(identity) = temper_server::identity::IdentityResolver::new() + .resolve(&state.server, &tenant, &token) + .await { - // Agent credential resolved — inject into request extensions. - req.extensions_mut().insert(identity); + let session_id = temper_server::request_context::session_id_from_headers(req.headers()); + let intent = temper_server::request_context::intent_from_headers(req.headers()); + let verified_session = match session_id.as_deref() { + Some(sid) => state + .server + .session_grant_verified(tenant.as_str(), &identity.agent_instance_id, sid) + .await + .then_some(sid), + None => None, + }; + let security_context = temper_authz::SecurityContext::from_resolved_identity( + &identity.agent_instance_id, + &identity.agent_type_name, + verified_session, + ); + let authenticated = + temper_authz::AuthenticatedRequestContext::new(tenant.clone(), security_context) + .with_intent(intent) + .with_session_id(session_id); + + req.extensions_mut().insert(authenticated); + if let Some(matched) = matched_endpoint { + req.extensions_mut() + .insert(temper_server::http_endpoint::AdmittedHttpEndpoint::new( + tenant.clone(), + &request_method, + &request_path, + matched, + )); + } + // The credential has served its only purpose. Downstream handlers and + // tenant WASM modules receive typed authority, never the reusable secret. + req.headers_mut().remove("authorization"); return Ok(next.run(req).await); } - // Step 2: Fall back to global API key (admin/operator access). - if matches_global_api_key { - if !req.headers().contains_key("x-temper-principal-kind") { - req.headers_mut().insert( - "x-temper-principal-kind", - "admin" - .parse() - .expect("valid x-temper-principal-kind header"), - ); - } - if !req.headers().contains_key("x-temper-principal-id") { - req.headers_mut().insert( - "x-temper-principal-id", - "api-key-holder" - .parse() - .expect("valid x-temper-principal-id header"), - ); + // Routes declared public have an explicit anonymous Customer context. + // This lets foreign-protocol adapters serve public reads without the old + // System fallback or trusting a guest-returned principal. + if let Some(matched) = matched_endpoint { + if matched.route.requires_auth { + return Ok(( + StatusCode::UNAUTHORIZED, + [("www-authenticate", "Basic realm=\"Temper\"")], + ) + .into_response()); } + req.headers_mut().remove("authorization"); + req.extensions_mut() + .insert(temper_authz::AuthenticatedRequestContext::new( + tenant.clone(), + temper_authz::SecurityContext::anonymous(), + )); + req.extensions_mut() + .insert(temper_server::http_endpoint::AdmittedHttpEndpoint::new( + tenant, + &request_method, + &request_path, + matched, + )); return Ok(next.run(req).await); } - // No match — reject. Err(StatusCode::UNAUTHORIZED) } -/// Extract tenant ID from request headers, defaulting to "default". -fn extract_tenant(req: &Request) -> TenantId { - req.headers() - .get("x-tenant-id") - .and_then(|v| v.to_str().ok()) - .map(TenantId::new) - .unwrap_or_default() +fn authorization_parts(req: &Request) -> Option<(&str, &str)> { + let value = req.headers().get("authorization")?.to_str().ok()?; + let (scheme, credential) = value.split_once(' ')?; + let credential = credential.trim(); + (!credential.is_empty()).then_some((scheme, credential)) } -/// Constant-time byte comparison. -fn constant_time_eq(a: &[u8], b: &[u8]) -> bool { - if a.len() != b.len() { - return false; - } - let mut diff = 0u8; - for (x, y) in a.iter().zip(b.iter()) { - diff |= x ^ y; - } - diff == 0 +fn bearer_credential(req: &Request) -> Option { + let (scheme, credential) = authorization_parts(req)?; + (scheme.eq_ignore_ascii_case("bearer") + && credential.len() <= temper_server::identity::MAX_CREDENTIAL_BYTES) + .then(|| credential.to_string()) } -#[cfg(test)] -mod tests { - use super::*; - use axum::Router; - use axum::body::Body; - use axum::body::to_bytes; - use axum::extract::Extension; - use axum::http::HeaderMap; - use axum::http::Request as HttpRequest; - use axum::middleware; - use axum::routing::get; - use std::collections::BTreeMap; - use temper_server::identity::ResolvedIdentity; - use tower::ServiceExt; - - async fn ok_handler() -> &'static str { - "ok" - } +fn request_credential(req: &Request) -> Option { + let (scheme, credential) = authorization_parts(req)?; + if scheme.eq_ignore_ascii_case("bearer") { + return (credential.len() <= temper_server::identity::MAX_CREDENTIAL_BYTES) + .then(|| credential.to_string()); + } + if !scheme.eq_ignore_ascii_case("basic") || credential.len() > BASIC_CREDENTIAL_DECODE_BUDGET { + return None; + } + let decoded = base64::engine::general_purpose::STANDARD + .decode(credential) + .ok()?; + if decoded.len() > BASIC_CREDENTIAL_DECODE_BUDGET { + return None; + } + let decoded = std::str::from_utf8(&decoded).ok()?; + let (username, password) = decoded.split_once(':')?; + let token = if password.is_empty() { + username + } else { + password + }; + (!token.is_empty()).then(|| token.to_string()) +} - async fn inspect_identity_handler( - headers: HeaderMap, - resolved_identity: Option>, - ) -> String { - let resolved = resolved_identity - .map(|Extension(identity)| identity.agent_type_name) - .unwrap_or_else(|| "none".to_string()); - let principal_kind = headers - .get("x-temper-principal-kind") - .and_then(|v| v.to_str().ok()) - .unwrap_or(""); - let principal_id = headers - .get("x-temper-principal-id") - .and_then(|v| v.to_str().ok()) - .unwrap_or(""); - let agent_type = headers - .get("x-temper-agent-type") - .and_then(|v| v.to_str().ok()) - .unwrap_or(""); - format!( - "resolved={resolved};principal_kind={principal_kind};principal_id={principal_id};agent_type={agent_type}" +fn is_public_request(req: &Request) -> bool { + temper_server::authz::is_public_kernel_request(req.method(), req.uri().path()) + || matches!( + (req.method(), req.uri().path()), + (&Method::GET, "/healthz") | (&Method::POST, "/api/identity/resolve") ) - } - - fn app_with_token(token: Option) -> Router { - let mut state = PlatformState::new(None); - state.api_token = token; - Router::new() - .route("/tdata", get(ok_handler)) - .route("/healthz", get(ok_handler)) - .route("/tdata/Orders", get(ok_handler)) - .route("/api/specs", get(ok_handler)) - .layer(middleware::from_fn_with_state( - state.clone(), - bearer_auth_check, - )) - .with_state(state) - } - - #[tokio::test] - async fn no_token_configured_passes_all() { - let app = app_with_token(None); - let resp = app - .oneshot(HttpRequest::get("/api/specs").body(Body::empty()).unwrap()) - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - } - - #[tokio::test] - async fn health_check_passes_without_auth() { - let app = app_with_token(Some("secret123".into())); - let resp = app - .clone() - .oneshot(HttpRequest::get("/tdata").body(Body::empty()).unwrap()) - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - - let resp_healthz = app - .oneshot(HttpRequest::get("/healthz").body(Body::empty()).unwrap()) - .await - .unwrap(); - assert_eq!(resp_healthz.status(), StatusCode::OK); - } - - #[tokio::test] - async fn valid_bearer_passes() { - let app = app_with_token(Some("secret123".into())); - let resp = app - .oneshot( - HttpRequest::get("/api/specs") - .header("authorization", "Bearer secret123") - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - } - - #[tokio::test] - async fn missing_auth_returns_401() { - let app = app_with_token(Some("secret123".into())); - let resp = app - .oneshot(HttpRequest::get("/api/specs").body(Body::empty()).unwrap()) - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); - } - - #[tokio::test] - async fn wrong_token_returns_401() { - let app = app_with_token(Some("secret123".into())); - let resp = app - .oneshot( - HttpRequest::get("/api/specs") - .header("authorization", "Bearer wrong") - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); - } - - #[tokio::test] - async fn non_bearer_scheme_returns_401() { - let app = app_with_token(Some("secret123".into())); - let resp = app - .oneshot( - HttpRequest::get("/api/specs") - .header("authorization", "Basic secret123") - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); - } - - #[tokio::test] - async fn tdata_subpath_requires_auth() { - let app = app_with_token(Some("secret123".into())); - // /tdata/Orders is NOT the health check path — requires auth. - let resp = app - .oneshot( - HttpRequest::get("/tdata/Orders") - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); - } - - #[tokio::test] - async fn pre_authenticated_request_bypasses_bearer_requirement() { - async fn mark_pre_authenticated( - mut req: Request, - next: Next, - ) -> Result { - req.extensions_mut().insert(PreAuthenticatedRequest); - req.headers_mut() - .insert("x-tenant-id", "default".parse().unwrap()); - req.headers_mut() - .insert("x-temper-principal-kind", "admin".parse().unwrap()); - req.headers_mut() - .insert("x-temper-principal-id", "dashboard-user".parse().unwrap()); - Ok(next.run(req).await) - } - - let mut state = PlatformState::new(None); - state.api_token = Some("secret123".into()); - let app = Router::new() - .route("/tdata/Orders", get(ok_handler)) - .layer(middleware::from_fn_with_state( - state.clone(), - bearer_auth_check, - )) - .layer(middleware::from_fn(mark_pre_authenticated)) - .with_state(state); - - let resp = app - .oneshot( - HttpRequest::get("/tdata/Orders") - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - } - - #[tokio::test] - async fn global_api_key_preserves_explicit_principal_headers() { - let mut state = PlatformState::new(None); - state.api_token = Some("secret123".into()); - crate::bootstrap::bootstrap_system_tenant(&state, &BTreeMap::new()); - crate::bootstrap::bootstrap_agent_specs(&state, "default", false, &BTreeMap::new()); - crate::bootstrap::bootstrap_operator_credential(&state, "secret123", "default").await; - - let app = Router::new() - .route("/inspect", get(inspect_identity_handler)) - .layer(middleware::from_fn_with_state( - state.clone(), - bearer_auth_check, - )) - .with_state(state); - - let resp = app - .oneshot( - HttpRequest::get("/inspect") - .header("authorization", "Bearer secret123") - .header("x-tenant-id", "default") - .header("x-temper-principal-kind", "agent") - .header("x-temper-principal-id", "system") - .header("x-temper-agent-type", "system") - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - assert_eq!(resp.status(), StatusCode::OK); - - let body = to_bytes(resp.into_body(), usize::MAX).await.unwrap(); - let body = String::from_utf8(body.to_vec()).unwrap(); - assert!( - body.contains("resolved=none"), - "global API key with explicit principal headers must not inject operator identity: {body}" - ); - assert!(body.contains("principal_kind=agent")); - assert!(body.contains("principal_id=system")); - assert!(body.contains("agent_type=system")); - } +} - #[test] - fn constant_time_eq_works() { - assert!(constant_time_eq(b"hello", b"hello")); - assert!(!constant_time_eq(b"hello", b"world")); - assert!(!constant_time_eq(b"hello", b"hell")); - } +fn requested_tenant(req: &Request) -> Result { + let tenant = req + .headers() + .get("x-tenant-id") + .and_then(|value| value.to_str().ok()) + .map(str::trim) + .filter(|tenant| !tenant.is_empty()) + .unwrap_or("default"); + TenantId::try_new(tenant).map_err(|_| StatusCode::BAD_REQUEST) } + +#[cfg(test)] +#[path = "bearer_auth/tests.rs"] +mod tests; diff --git a/crates/temper-platform/src/bearer_auth/tests.rs b/crates/temper-platform/src/bearer_auth/tests.rs new file mode 100644 index 000000000..726b58f7d --- /dev/null +++ b/crates/temper-platform/src/bearer_auth/tests.rs @@ -0,0 +1,543 @@ +use super::*; +use axum::Router; +use axum::body::Body; +use axum::extract::Extension; +use axum::http::Request as HttpRequest; +use axum::middleware; +use axum::routing::{get, post}; +use std::collections::BTreeMap; +use temper_authz::AuthenticatedRequestContext; +use tower::ServiceExt; + +async fn ok_handler() -> &'static str { + "ok" +} + +/// Reports where the request's session landed: the Cedar-visible +/// `context_attrs["sessionId"]` vs the telemetry-only request context. +async fn session_probe(Extension(context): Extension) -> String { + format!( + "cedar={:?} telemetry={:?} principal={}", + context + .security_context() + .context_attrs + .get("sessionId") + .and_then(|v| v.as_str()), + context.session_id(), + context.security_context().principal.id, + ) +} + +async fn whoami( + Extension(context): Extension, + headers: axum::http::HeaderMap, +) -> String { + format!( + "{}:{:?}:{}:{}", + context.tenant(), + context.security_context().principal.kind, + context.security_context().principal.id, + headers.contains_key("authorization") + ) +} + +fn protocol_route(requires_auth: bool) -> temper_server::http_endpoint::HttpEndpointRoute { + temper_server::http_endpoint::HttpEndpointRoute { + id: "he-protocol".to_string(), + path_prefix: "/repo.git".to_string(), + methods: vec!["GET".to_string(), "POST".to_string()], + integration_module: "protocol-adapter".to_string(), + requires_auth, + timeout_secs: 60, + max_fuel: None, + max_memory: None, + max_response_bytes: None, + action_bridge: None, + } +} + +fn app(state: PlatformState) -> Router { + Router::new() + .route("/tdata", get(ok_handler)) + .route("/tdata/$metadata", get(ok_handler)) + .route("/tdata/$hints", get(ok_handler)) + .route("/temper-client.js", get(ok_handler)) + .route("/static/temper-client.js", get(ok_handler)) + .route("/genesis", get(ok_handler)) + .route("/genesis/{*path}", get(ok_handler)) + .route( + "/webhooks/{tenant}/{*path}", + get(ok_handler).post(ok_handler), + ) + .route("/healthz", get(ok_handler)) + .route("/api/identity/resolve", post(ok_handler)) + .route("/session-probe", get(session_probe)) + .route("/api/specs", get(ok_handler)) + .route("/whoami", get(whoami)) + .route("/repo.git/{*path}", get(whoami).post(whoami)) + .layer(middleware::from_fn_with_state( + state.clone(), + bearer_auth_check, + )) + .layer(middleware::from_fn( + temper_server::authz::strip_inbound_identity_headers, + )) + .with_state(state) +} + +#[tokio::test] +async fn no_key_mode_rejects_protected_requests() { + let response = app(PlatformState::new(None)) + .oneshot( + HttpRequest::get("/api/specs") + .header("x-temper-principal-kind", "admin") + .header("x-temper-principal-id", "attacker") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); +} + +#[tokio::test] +async fn deployment_key_has_no_unregistered_fallback() { + let mut state = PlatformState::new(None); + state.api_token = Some("deployment-root".to_string()); + let response = app(state) + .oneshot( + HttpRequest::get("/api/specs") + .header("authorization", "Bearer deployment-root") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); +} + +#[tokio::test] +async fn exact_public_routes_do_not_require_credentials() { + let app = app(PlatformState::new(None)); + for request in [ + HttpRequest::get("/tdata").body(Body::empty()).unwrap(), + HttpRequest::get("/tdata/$metadata") + .body(Body::empty()) + .unwrap(), + HttpRequest::get("/temper-client.js") + .body(Body::empty()) + .unwrap(), + HttpRequest::get("/static/temper-client.js") + .body(Body::empty()) + .unwrap(), + HttpRequest::get("/genesis/app.js") + .body(Body::empty()) + .unwrap(), + HttpRequest::post("/webhooks/tenant/provider") + .body(Body::empty()) + .unwrap(), + HttpRequest::get("/healthz").body(Body::empty()).unwrap(), + HttpRequest::post("/api/identity/resolve") + .body(Body::empty()) + .unwrap(), + ] { + let response = app.clone().oneshot(request).await.unwrap(); + assert_eq!(response.status(), StatusCode::OK); + } +} + +#[tokio::test] +async fn tenant_hints_require_a_credential() { + let response = app(PlatformState::new(None)) + .oneshot( + HttpRequest::get("/tdata/$hints") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); +} + +#[tokio::test] +async fn registered_credential_is_tenant_bound_and_headers_cannot_replace_it() { + let state = PlatformState::new(None); + crate::bootstrap::bootstrap_agent_specs(&state, "default", false, &BTreeMap::new()); + crate::bootstrap::bootstrap_operator_credential(&state, "tenant-key", "default").await; + let router = app(state); + + let response = router + .clone() + .oneshot( + HttpRequest::get("/whoami") + .header("authorization", "Bearer tenant-key") + .header("x-tenant-id", "default") + .header("x-temper-principal-kind", "admin") + .header("x-temper-principal-id", "attacker") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let body = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + assert_eq!( + String::from_utf8(body.to_vec()).unwrap(), + "default:Agent:operator:false" + ); + + let cross_tenant = router + .oneshot( + HttpRequest::get("/whoami") + .header("authorization", "Bearer tenant-key") + .header("x-tenant-id", "other") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(cross_tenant.status(), StatusCode::UNAUTHORIZED); + + let basic = base64::engine::general_purpose::STANDARD.encode("git:tenant-key"); + let state = PlatformState::new(None); + crate::bootstrap::bootstrap_agent_specs(&state, "default", false, &BTreeMap::new()); + crate::bootstrap::bootstrap_operator_credential(&state, "tenant-key", "default").await; + let response = app(state) + .oneshot( + HttpRequest::get("/whoami") + .header("authorization", format!("Basic {basic}")) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); +} + +#[tokio::test] +async fn declared_public_http_endpoint_gets_anonymous_typed_context() { + let state = PlatformState::new(None); + state + .server + .http_endpoint_tables + .table_for(&TenantId::default()) + .await + .replace(vec![protocol_route(false)]) + .await; + + let response = app(state) + .oneshot( + HttpRequest::get("/repo.git/info/refs") + .header("authorization", "Basic malformed") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let body = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + assert_eq!( + String::from_utf8(body.to_vec()).unwrap(), + "default:Customer:anonymous:false" + ); +} + +#[tokio::test] +async fn private_http_endpoint_challenges_basic_credentials_before_guest_dispatch() { + let state = PlatformState::new(None); + state + .server + .http_endpoint_tables + .table_for(&TenantId::default()) + .await + .replace(vec![protocol_route(true)]) + .await; + + let response = app(state) + .oneshot( + HttpRequest::get("/repo.git/info/refs") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + assert_eq!( + response + .headers() + .get("www-authenticate") + .and_then(|value| value.to_str().ok()), + Some("Basic realm=\"Temper\"") + ); +} + +#[test] +fn basic_credential_parser_uses_password_then_empty_password_username() { + for (decoded, expected) in [ + ("git:tenant-key", "tenant-key"), + ("tenant-key:", "tenant-key"), + ] { + let encoded = base64::engine::general_purpose::STANDARD.encode(decoded); + let request = HttpRequest::get("/") + .header("authorization", format!("Basic {encoded}")) + .body(Body::empty()) + .unwrap(); + assert_eq!(request_credential(&request), Some(expected.to_string())); + } +} + +#[test] +fn oversized_bearer_credentials_are_rejected_before_resolution() { + let oversized = "x".repeat(temper_server::identity::MAX_CREDENTIAL_BYTES + 1); + let request = HttpRequest::get("/") + .header("authorization", format!("Bearer {oversized}")) + .body(Body::empty()) + .unwrap(); + assert!(bearer_credential(&request).is_none()); + assert!(request_credential(&request).is_none()); +} + +#[tokio::test] +async fn malformed_tenant_header_is_rejected_without_panicking() { + let response = app(PlatformState::new(None)) + .oneshot( + HttpRequest::get("/healthz") + .header("x-tenant-id", ":") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); +} + +#[tokio::test] +async fn public_classifier_does_not_allow_prefix_confusion() { + let app = app(PlatformState::new(None)); + for path in ["/tdata/Orders", "/genesis-evil", "/webhooks-evil/path"] { + let response = app + .clone() + .oneshot(HttpRequest::get(path).body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED, "{path}"); + } +} + +#[tokio::test] +async fn unauthenticated_unknown_tenant_does_not_allocate_route_table() { + let state = PlatformState::new(None); + let tables = state.server.http_endpoint_tables.clone(); + let response = app(state) + .oneshot( + HttpRequest::get("/unknown-protocol") + .header("x-tenant-id", "attacker-created") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + assert_eq!(tables.tenant_count().await, 0); +} + +#[tokio::test] +async fn internal_capability_restores_exact_context_without_identity_resolution() { + let state = PlatformState::new(None); + let mut security_context = temper_authz::SecurityContext::from_resolved_identity( + "invoking-agent", + "planner", + Some("session-1"), + ); + security_context.principal.role = Some("planner-role".to_string()); + let token = state + .server + .internal_invocation_credentials + .issue_for_url( + AuthenticatedRequestContext::new(TenantId::new("tenant-a"), security_context), + "GET", + "http://127.0.0.1:3000/whoami?view=full", + ) + .expect("internal credential should issue"); + + let response = app(state) + .oneshot( + HttpRequest::get("/whoami?view=full") + .header("authorization", format!("Bearer {token}")) + .header("x-tenant-id", "tenant-a") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let body = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + assert_eq!( + String::from_utf8(body.to_vec()).unwrap(), + "tenant-a:Agent:invoking-agent:false" + ); +} + +#[tokio::test] +async fn reserved_internal_prefix_never_falls_back_to_agent_credentials() { + let state = PlatformState::new(None); + crate::bootstrap::bootstrap_agent_specs(&state, "default", false, &BTreeMap::new()); + let reserved = format!( + "{}registered-as-normal", + temper_server::internal_invocation::INTERNAL_INVOCATION_BEARER_PREFIX + ); + crate::bootstrap::bootstrap_operator_credential(&state, &reserved, "default").await; + + let response = app(state) + .oneshot( + HttpRequest::get("/whoami") + .header("authorization", format!("Bearer {reserved}")) + .header("x-tenant-id", "default") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); +} + +#[tokio::test] +async fn internal_capability_on_public_route_is_still_consumed_once() { + let state = PlatformState::new(None); + let token = state + .server + .internal_invocation_credentials + .issue_for_url( + AuthenticatedRequestContext::new( + TenantId::new("tenant-a"), + temper_authz::SecurityContext::from_resolved_identity( + "invoking-agent", + "worker", + None, + ), + ), + "GET", + "http://127.0.0.1:3000/tdata", + ) + .expect("internal credential should issue"); + let request = || { + HttpRequest::get("/tdata") + .header("authorization", format!("Bearer {token}")) + .header("x-tenant-id", "tenant-a") + .body(Body::empty()) + .unwrap() + }; + let app = app(state); + + assert_eq!( + app.clone().oneshot(request()).await.unwrap().status(), + StatusCode::OK + ); + assert_eq!( + app.oneshot(request()).await.unwrap().status(), + StatusCode::UNAUTHORIZED + ); +} + +/// A caller-asserted session header becomes `context.sessionId` — a Cedar +/// input — only when an approved decision binds that exact session to the +/// asserting principal (ADR-0157). Unvalidated assertions stay telemetry-only, +/// so session-scoped permits cannot be satisfied by replaying a header. +#[tokio::test] +async fn session_header_reaches_cedar_only_through_an_approved_grant() { + let mut state = PlatformState::new(None); + let dir = std::env::temp_dir().join(format!( + "temper-session-edge-{}-{}", + std::process::id(), + uuid::Uuid::new_v4() + )); + std::fs::create_dir_all(&dir).expect("create temp dir"); + let turso = temper_store_turso::TursoEventStore::new( + &format!("file:{}", dir.join("grants.db").display()), + None, + ) + .await + .expect("create local turso db"); + state + .server + .set_storage_stack(temper_server::storage::StorageStack::from_turso(turso)); + crate::bootstrap::bootstrap_agent_specs(&state, "default", false, &BTreeMap::new()); + crate::bootstrap::bootstrap_operator_credential(&state, "tenant-key", "default").await; + let server_state = state.server.clone(); + let router = app(state); + + let probe = |session: Option<&'static str>| { + let router = router.clone(); + async move { + let mut request = HttpRequest::get("/session-probe") + .header("authorization", "Bearer tenant-key") + .header("x-tenant-id", "default"); + if let Some(session) = session { + request = request.header("x-session-id", session); + } + let response = router + .oneshot(request.body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let body = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + String::from_utf8(body.to_vec()).unwrap() + } + }; + + // Before any grant: the asserted header is telemetry, never Cedar input. + let unvalidated = probe(Some("sess-approved")).await; + assert!( + unvalidated.starts_with("cedar=None telemetry=Some(\"sess-approved\")"), + "an unvalidated session assertion must stay out of the Cedar context: {unvalidated}" + ); + let agent_id = unvalidated + .rsplit("principal=") + .next() + .expect("probe reports the principal") + .to_string(); + + // A human approves a session-scoped decision for exactly this principal. + let mut scope = temper_authz::PolicyScopeMatrix::default_for(Some("operator")); + scope.duration = temper_authz::DurationScope::Session; + scope.session_id = Some("sess-approved".to_string()); + let mut decision = temper_server::state::PendingDecision::from_denial( + "default", + &agent_id, + "Delete", + "Order", + "order-1", + serde_json::json!({}), + "denied by policy", + None, + ); + decision.status = temper_server::state::DecisionStatus::Approved; + decision.approved_scope = Some(scope); + server_state + .persist_pending_decision(&decision) + .await + .expect("persist approved session grant"); + + // The approved (principal, session) pair now reaches Cedar. + let validated = probe(Some("sess-approved")).await; + assert!( + validated.starts_with("cedar=Some(\"sess-approved\") telemetry=Some(\"sess-approved\")"), + "the granted session must reach the Cedar context: {validated}" + ); + + // A different asserted session still does not. + let other = probe(Some("sess-other")).await; + assert!( + other.starts_with("cedar=None telemetry=Some(\"sess-other\")"), + "a session outside the grant must stay out of the Cedar context: {other}" + ); +} diff --git a/crates/temper-platform/src/bootstrap.rs b/crates/temper-platform/src/bootstrap.rs index 8cbbdd176..b7851b777 100644 --- a/crates/temper-platform/src/bootstrap.rs +++ b/crates/temper-platform/src/bootstrap.rs @@ -390,16 +390,20 @@ pub async fn persist_agent_verification( .await; } -/// Auto-register an `AgentCredential` for the global API key on bootstrap. +/// Register the bootstrap key as a normal operator credential in one tenant. /// /// When the platform boots with a `TEMPER_API_KEY` configured, this function /// ensures a corresponding `AgentType` ("operator") and `AgentCredential` -/// exist in the default tenant so the bearer auth middleware can resolve -/// the global key as a verified identity instead of falling back to -/// unverified/anonymous access. +/// exist in the given tenant so the bearer auth middleware can resolve the +/// key as a verified identity in that tenant. Registration in one tenant +/// deliberately grants no authority in any other tenant. /// -/// This is idempotent: if the entities already exist (e.g., from a previous -/// boot), the actions are no-ops (entity already in target state). +/// Also seeds a narrow Cedar permit so a verified operator can +/// `manage_policies` on that tenant's `PolicySet` (ADR-0172). The permit is +/// merged into live Cedar, persisted as a granular row, and is idempotent. +/// +/// This is idempotent: if the entities and permit already exist (e.g., from a +/// previous boot), the actions are no-ops. pub async fn bootstrap_operator_credential(state: &PlatformState, api_key: &str, tenant: &str) { use temper_server::identity::hash_token; @@ -419,7 +423,7 @@ pub async fn bootstrap_operator_credential(state: &PlatformState, api_key: &str, "Define", serde_json::json!({ "name": "operator", - "system_prompt": "Platform operator — global API key access", + "system_prompt": "Tenant-scoped platform operator", "tool_set": "local", "model": "none", "max_turns": "0", @@ -432,11 +436,7 @@ pub async fn bootstrap_operator_credential(state: &PlatformState, api_key: &str, // Step 2: Create and issue AgentCredential for the API key hash. let key_hash = hash_token(api_key); - let key_prefix = if api_key.len() >= 8 { - &api_key[..8] - } else { - api_key - }; + let key_prefix = api_key.chars().take(8).collect::(); let _ = state .server @@ -450,7 +450,7 @@ pub async fn bootstrap_operator_credential(state: &PlatformState, api_key: &str, "agent_instance_id": instance_id, "key_hash": key_hash, "key_prefix": key_prefix, - "description": "Auto-registered credential for global TEMPER_API_KEY", + "description": "Tenant-scoped operator bootstrap credential", "created_by": "bootstrap", "expires_at": "" }), @@ -458,6 +458,8 @@ pub async fn bootstrap_operator_credential(state: &PlatformState, api_key: &str, ) .await; + crate::operator_manage_policies::seed_operator_manage_policies(state, tenant).await; + tracing::info!( "Operator credential bootstrapped for tenant '{tenant}' (key_hash={}...)", &key_hash[..8] diff --git a/crates/temper-platform/src/genesis_install.rs b/crates/temper-platform/src/genesis_install.rs index 3cf880ba0..b0614e2d5 100644 --- a/crates/temper-platform/src/genesis_install.rs +++ b/crates/temper-platform/src/genesis_install.rs @@ -4,24 +4,43 @@ //! governed action has succeeded, then materializes the pinned Genesis commit //! into the platform's app installer. -use std::collections::BTreeSet; -use std::path::{Component, Path, PathBuf}; +use std::collections::{BTreeMap, BTreeSet}; +use std::io::Read as _; +use std::path::{Path, PathBuf}; use std::time::Instant; -use base64::Engine as _; use serde::{Deserialize, Serialize}; use serde_json::Value; +use sha2::Digest as _; use temper_runtime::tenant::TenantId; use temper_server::platform_store::InstalledAppRecord; use temper_server::state::{BoundActionHook, BoundActionHookContext, DispatchCommand, ServerState}; +mod blob_materialization; +mod bundle_transport; +mod bundles; +mod cache_paths; +use blob_materialization::{ + MAX_GENESIS_TREE_CANONICAL_BYTES, blob_content_len, canonical_field_len, git_object_body, + materialize_blob_content_field, read_canonical_field_bounded, +}; +use bundles::{ + GenesisBundleBudget, MAX_GENESIS_BUNDLE_APPS, MAX_GENESIS_BUNDLE_FILE_BYTES, + collect_bundle_files, materialize_registry_app_closure_via_bundle, +}; +#[cfg(test)] +use bundles::{safe_bundle_relative_path, write_bundle_app}; +use cache_paths::{ + app_cache_dir, replace_directory, validate_git_object_id, validate_identity_component, +}; + use crate::os_apps::{ AppManifest, InstallResult, OsAppReconcileResult, add_os_apps_dir_preferred, os_app_bundle_digest, reconcile_os_app, resolve_os_app_install_order, }; use crate::state::PlatformState; -const FIELD_OVERFLOW_REF_KEY: &str = "__temper_blob_ref"; +const MAX_GENESIS_MANIFEST_BYTES: u64 = 1024 * 1024; #[derive(Debug, Clone, Deserialize)] pub struct GenesisRegistryInstallRequest { @@ -764,6 +783,8 @@ fn parse_registry_app_ref(app_ref: &str) -> Result { if owner.is_empty() || name.is_empty() { return Err("Genesis app ref must include non-empty owner and app name".to_string()); } + validate_identity_component("owner", owner)?; + validate_identity_component("app name", name)?; let version_hash = match version_hash { Some(hash) if hash.trim().is_empty() => { return Err("Genesis app ref hash must not be empty".to_string()); @@ -785,6 +806,8 @@ async fn materialize_git_registry_app( version_hash: Option<&str>, app_dir: &Path, ) -> Result { + validate_identity_component("owner", owner)?; + validate_identity_component("app name", name)?; let remote = registry_git_url(registry_url, owner, name); let git_dir = app_dir.join(".git"); if app_dir.exists() && !git_dir.is_dir() { @@ -812,6 +835,7 @@ async fn materialize_git_registry_app( } if let Some(hash) = version_hash { + let hash = validate_git_object_id(hash)?; run_git(Some(app_dir), &["checkout", "--detach", hash]).await?; } @@ -841,7 +865,7 @@ async fn materialize_registry_app_closure( continue; } - let app_dir = cache_root.join(&app_ref.name); + let app_dir = app_cache_dir(cache_root, &app_ref.name)?; let resolved_hash = materialize_git_registry_app( registry_url, &app_ref.owner, @@ -869,124 +893,6 @@ async fn materialize_registry_app_closure( Ok(materialized_refs) } -async fn materialize_registry_app_closure_via_bundle( - registry_url: &str, - registry_tenant: &str, - root_ref: RegistryAppRef, - cache_root: &Path, -) -> Result, String> { - let Some(version_hash) = root_ref.version_hash.as_deref() else { - return Err("bundle fetch requires a pinned root app ref".to_string()); - }; - let bundle_url = format!( - "{}/api/genesis/apps/{}/{}/versions/{}/bundle", - registry_url.trim_end_matches('/'), - root_ref.owner, - root_ref.name, - version_hash.trim_start_matches('@') - ); - let response = reqwest::Client::new() - .get(&bundle_url) - .header("X-Tenant-Id", registry_tenant) - .send() - .await - .map_err(|error| format!("request Genesis bundle {bundle_url}: {error}"))?; - let status = response.status(); - if !status.is_success() { - let body = response.text().await.unwrap_or_default(); - return Err(format!( - "request Genesis bundle {bundle_url} returned {status}: {}", - body.trim() - )); - } - let bundle: GenesisRegistryBundleResponse = response - .json() - .await - .map_err(|error| format!("decode Genesis bundle {bundle_url}: {error}"))?; - - if cache_root.exists() { - std::fs::remove_dir_all(cache_root).map_err(|error| { - format!( - "clear Genesis registry bundle cache '{}': {error}", - cache_root.display() - ) - })?; - } - std::fs::create_dir_all(cache_root).map_err(|error| { - format!( - "create Genesis registry bundle cache '{}': {error}", - cache_root.display() - ) - })?; - - let mut refs = Vec::new(); - for app in bundle.apps { - let app_dir = cache_root.join(&app.name); - write_bundle_app(&app_dir, &app)?; - refs.push(RegistryAppRef { - owner: app.owner, - name: app.name, - version_hash: Some(app.version_hash), - }); - } - Ok(refs) -} - -fn write_bundle_app(app_dir: &Path, app: &GenesisRegistryBundleApp) -> Result<(), String> { - if app_dir.exists() { - std::fs::remove_dir_all(app_dir).map_err(|error| { - format!("clear Genesis bundle app '{}': {error}", app_dir.display()) - })?; - } - std::fs::create_dir_all(app_dir) - .map_err(|error| format!("create Genesis bundle app '{}': {error}", app_dir.display()))?; - - for file in &app.files { - let rel = safe_bundle_relative_path(&file.path)?; - let path = app_dir.join(&rel); - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent).map_err(|error| { - format!("create bundle file parent '{}': {error}", parent.display()) - })?; - } - let bytes = base64::engine::general_purpose::STANDARD - .decode(&file.content_base64) - .map_err(|error| format!("decode bundle file '{}': {error}", file.path))?; - std::fs::write(&path, bytes) - .map_err(|error| format!("write bundle file '{}': {error}", path.display()))?; - } - Ok(()) -} - -fn safe_bundle_relative_path(path: &str) -> Result { - let rel = PathBuf::from(path); - if rel.as_os_str().is_empty() { - return Err("bundle file path must not be empty".to_string()); - } - let mut safe = PathBuf::new(); - for component in rel.components() { - match component { - Component::Normal(part) => { - if part == "target" || part == ".git" { - return Err(format!( - "bundle file path '{}' contains forbidden component '{}'", - path, - part.to_string_lossy() - )); - } - safe.push(part); - } - _ => { - return Err(format!( - "bundle file path '{}' must be relative and must not contain '..'", - path - )); - } - } - } - Ok(safe) -} - fn registry_git_url(registry_url: &str, owner: &str, name: &str) -> String { format!( "{}/{}/{}.git", @@ -1091,7 +997,7 @@ impl BoundActionHook for GenesisInstallHook { }, ) .await?; - let app_dir = cache_root.join(&name); + let app_dir = app_cache_dir(&cache_root, &name)?; add_os_apps_dir_preferred(cache_root); let mut platform = self.platform.clone(); @@ -1196,6 +1102,8 @@ fn resolve_install_app_ref( latest_version_hash: &str, requested_app_ref: Option<&str>, ) -> Result { + validate_identity_component("owner", owner)?; + validate_identity_component("app name", name)?; let latest = latest_version_hash.trim_start_matches('@'); let Some(raw_app_ref) = requested_app_ref .map(str::trim) @@ -1345,6 +1253,45 @@ struct GenesisAppBundle { version_hash: String, } +#[derive(Default)] +struct GenesisClosureAdmission { + versions_by_app: BTreeMap<(String, String), String>, + owners_by_directory: BTreeMap, +} + +impl GenesisClosureAdmission { + fn admit(&mut self, app: &GenesisAppBundle) -> Result { + let app_key = (app.owner.clone(), app.name.clone()); + if let Some(version) = self.versions_by_app.get(&app_key) { + if version != &app.version_hash { + return Err(format!( + "Genesis dependency {}/{} resolves to conflicting versions '{}' and '{}'", + app.owner, app.name, version, app.version_hash + )); + } + return Ok(false); + } + if let Some(owner) = self.owners_by_directory.get(&app.name) + && owner != &app.owner + { + return Err(format!( + "Genesis dependencies {owner}/{} and {}/{} collide on cache directory '{}'", + app.name, app.owner, app.name, app.name + )); + } + if self.versions_by_app.len() >= MAX_GENESIS_BUNDLE_APPS { + return Err(format!( + "Genesis dependency closure exceeds app budget {MAX_GENESIS_BUNDLE_APPS}" + )); + } + self.owners_by_directory + .insert(app.name.clone(), app.owner.clone()); + self.versions_by_app + .insert(app_key, app.version_hash.clone()); + Ok(true) + } +} + pub async fn export_genesis_registry_bundle( platform: &PlatformState, registry_tenant: &str, @@ -1369,10 +1316,18 @@ pub async fn export_genesis_registry_bundle( ); let cache_root = genesis_cache_root(&platform.server, &app_ref); let closure = resolve_genesis_app_closure(&platform.server, &tenant, root).await?; + if closure.len() > MAX_GENESIS_BUNDLE_APPS { + return Err(format!( + "Genesis bundle closure contains {} apps; budget is {MAX_GENESIS_BUNDLE_APPS}", + closure.len() + )); + } let mut apps = Vec::new(); + let mut bundle_budget = GenesisBundleBudget::new(); + let mut materialization_budget = GenesisBundleBudget::new(); for app in closure { - let app_dir = cache_root.join(&app.name); + let app_dir = app_cache_dir(&cache_root, &app.name)?; let started = Instant::now(); materialize_commit_tree( &platform.server, @@ -1380,9 +1335,10 @@ pub async fn export_genesis_registry_bundle( &app.repository_id, &app.version_hash, &app_dir, + &mut materialization_budget, ) .await?; - let files = collect_bundle_files(&app_dir)?; + let files = collect_bundle_files(&app_dir, &mut bundle_budget)?; tracing::info!( registry_tenant = %registry_tenant, app = %app.name, @@ -1455,12 +1411,12 @@ async fn resolve_genesis_app_closure( root: GenesisAppBundle, ) -> Result, String> { let mut stack = vec![root]; - let mut seen = BTreeSet::new(); + let mut admission = GenesisClosureAdmission::default(); let mut closure = Vec::new(); + let mut materialization_budget = GenesisBundleBudget::new(); while let Some(app) = stack.pop() { - let key = format!("{}/{}", app.owner, app.name); - if !seen.insert(key) { + if !admission.admit(&app)? { continue; } let cache_root = genesis_cache_root( @@ -1472,13 +1428,14 @@ async fn resolve_genesis_app_closure( app.version_hash.trim_start_matches('@') ), ); - let app_dir = cache_root.join(&app.name); + let app_dir = app_cache_dir(&cache_root, &app.name)?; materialize_commit_tree( state, tenant, &app.repository_id, &app.version_hash, &app_dir, + &mut materialization_budget, ) .await?; for dependency in read_manifest_dependencies(&app_dir)?.into_iter().rev() { @@ -1490,9 +1447,7 @@ async fn resolve_genesis_app_closure( dependency, app.name ) })?; - if !seen.contains(&format!("{}/{}", dependency.owner, dependency.name)) { - stack.push(dependency); - } + stack.push(dependency); } closure.push(app); } @@ -1500,62 +1455,6 @@ async fn resolve_genesis_app_closure( Ok(closure) } -fn collect_bundle_files(app_dir: &Path) -> Result, String> { - let mut paths = Vec::new(); - collect_bundle_file_paths(app_dir, app_dir, &mut paths)?; - paths.sort(); - let mut files = Vec::new(); - for path in paths { - let rel = path - .strip_prefix(app_dir) - .map_err(|error| format!("strip bundle path '{}': {error}", path.display()))? - .to_string_lossy() - .replace('\\', "/"); - let bytes = std::fs::read(&path) - .map_err(|error| format!("read bundle file '{}': {error}", path.display()))?; - files.push(GenesisRegistryBundleFile { - path: rel, - content_base64: base64::engine::general_purpose::STANDARD.encode(bytes), - }); - } - Ok(files) -} - -fn collect_bundle_file_paths( - root: &Path, - dir: &Path, - paths: &mut Vec, -) -> Result<(), String> { - let mut entries: Vec<_> = std::fs::read_dir(dir) - .map_err(|error| format!("read bundle directory '{}': {error}", dir.display()))? - .filter_map(|entry| entry.ok()) - .collect(); - entries.sort_by_key(|entry| entry.file_name()); - for entry in entries { - let path = entry.path(); - let rel = path - .strip_prefix(root) - .map_err(|error| format!("strip bundle path '{}': {error}", path.display()))?; - if rel.components().any(|component| { - matches!(component, Component::Normal(part) if part == "target" || part == ".git") - }) { - if path.is_dir() { - tracing::warn!( - path = %path.display(), - "Skipping forbidden generated directory in Genesis bundle export" - ); - } - continue; - } - if path.is_dir() { - collect_bundle_file_paths(root, &path, paths)?; - } else if path.is_file() { - paths.push(path); - } - } - Ok(()) -} - async fn materialize_app_closure( state: &ServerState, tenant: &TenantId, @@ -1563,21 +1462,23 @@ async fn materialize_app_closure( root: GenesisAppBundle, ) -> Result, String> { let mut stack = vec![root]; - let mut seen = BTreeSet::new(); + let mut admission = GenesisClosureAdmission::default(); let mut materialized = Vec::new(); + let mut materialization_budget = GenesisBundleBudget::new(); while let Some(app) = stack.pop() { - if !seen.insert(app.name.clone()) { + if !admission.admit(&app)? { continue; } - let app_dir = cache_root.join(&app.name); + let app_dir = app_cache_dir(cache_root, &app.name)?; materialize_commit_tree( state, tenant, &app.repository_id, &app.version_hash, &app_dir, + &mut materialization_budget, ) .await?; materialized.push(app.name.clone()); @@ -1591,9 +1492,7 @@ async fn materialize_app_closure( dependency, app.name ) })?; - if !seen.contains(&dependency.name) { - stack.push(dependency); - } + stack.push(dependency); } } @@ -1602,8 +1501,33 @@ async fn materialize_app_closure( fn read_manifest_dependencies(app_dir: &Path) -> Result, String> { let path = app_dir.join("app.toml"); - let content = std::fs::read_to_string(&path) + let metadata = std::fs::symlink_metadata(&path) + .map_err(|error| format!("stat Genesis app manifest '{}': {error}", path.display()))?; + if metadata.file_type().is_symlink() || !metadata.is_file() { + return Err(format!( + "Genesis app manifest '{}' must be a regular file", + path.display() + )); + } + if metadata.len() > MAX_GENESIS_MANIFEST_BYTES { + return Err(format!( + "Genesis app manifest '{}' is {} bytes; budget is {MAX_GENESIS_MANIFEST_BYTES}", + path.display(), + metadata.len() + )); + } + let mut content = String::with_capacity(metadata.len() as usize); + std::fs::File::open(&path) + .map_err(|error| format!("open Genesis app manifest '{}': {error}", path.display()))? + .take(MAX_GENESIS_MANIFEST_BYTES.saturating_add(1)) + .read_to_string(&mut content) .map_err(|error| format!("read Genesis app manifest '{}': {error}", path.display()))?; + if content.len() as u64 > MAX_GENESIS_MANIFEST_BYTES { + return Err(format!( + "Genesis app manifest '{}' exceeded its byte budget while reading", + path.display() + )); + } let manifest: AppManifest = toml::from_str(&content) .map_err(|error| format!("parse Genesis app manifest '{}': {error}", path.display()))?; Ok(manifest.dependencies) @@ -1734,21 +1658,40 @@ async fn materialize_commit_tree( repository_id: &str, version_hash: &str, app_dir: &Path, + budget: &mut GenesisBundleBudget, ) -> Result<(), String> { - let commit_id = version_hash.trim_start_matches('@'); + let commit_id = validate_git_object_id(version_hash)?; let commit = load_genesis_object(state, tenant, "Commit", repository_id, commit_id) .await? .ok_or_else(|| format!("Genesis commit {commit_id} not found for {repository_id}"))?; let tree_sha = string_field(&commit.state.fields, "TreeSha") .ok_or_else(|| format!("Genesis commit {commit_id} is missing TreeSha"))?; - - if app_dir.exists() { - std::fs::remove_dir_all(app_dir) - .map_err(|e| format!("clear Genesis app cache '{}': {e}", app_dir.display()))?; - } - std::fs::create_dir_all(app_dir) - .map_err(|e| format!("create Genesis app cache '{}': {e}", app_dir.display()))?; - materialize_tree(state, tenant, repository_id, &tree_sha, app_dir).await + let parent = app_dir.parent().ok_or_else(|| { + format!( + "Genesis app cache '{}' has no parent directory", + app_dir.display() + ) + })?; + std::fs::create_dir_all(parent).map_err(|error| { + format!( + "create Genesis app cache parent '{}': {error}", + parent.display() + ) + })?; + let staged = tempfile::Builder::new() + .prefix(".genesis-tree-") + .tempdir_in(parent) + .map_err(|error| format!("create staged Genesis app cache: {error}"))?; + materialize_tree( + state, + tenant, + repository_id, + &tree_sha, + staged.path(), + budget, + ) + .await?; + replace_directory(staged.keep(), app_dir) } async fn materialize_tree( @@ -1757,22 +1700,35 @@ async fn materialize_tree( repository_id: &str, tree_sha: &str, dir: &Path, + budget: &mut GenesisBundleBudget, ) -> Result<(), String> { - let mut stack = vec![(tree_sha.to_string(), dir.to_path_buf())]; - while let Some((current_tree, current_dir)) = stack.pop() { + let mut stack = vec![(tree_sha.to_string(), dir.to_path_buf(), 0usize)]; + while let Some((current_tree, current_dir, depth)) = stack.pop() { std::fs::create_dir_all(¤t_dir) .map_err(|e| format!("create directory '{}': {e}", current_dir.display()))?; let tree = load_genesis_object(state, tenant, "Tree", repository_id, ¤t_tree) .await? .ok_or_else(|| format!("Genesis tree {current_tree} not found for {repository_id}"))?; - let canonical = string_field_resolved(state, tenant, &tree.state.fields, "CanonicalBytes") - .await? - .ok_or_else(|| format!("Genesis tree {current_tree} is missing CanonicalBytes"))?; - for entry in parse_tree_entries(&decode_git_object_body(&canonical, "tree")?)? { + let canonical_bytes = canonical_field_len(&tree.state.fields, "tree")?; + budget.consume_tree(¤t_dir, canonical_bytes)?; + let canonical = read_canonical_field_bounded( + state, + tenant, + &tree.state.fields, + "tree", + MAX_GENESIS_TREE_CANONICAL_BYTES, + ) + .await + .map_err(|error| format!("read Genesis tree {current_tree}: {error}"))?; + for entry in parse_tree_entries(git_object_body(&canonical, "tree")?)? { validate_tree_entry_name(&entry.name)?; let path = current_dir.join(&entry.name); + let entry_depth = depth + .checked_add(1) + .ok_or_else(|| "Genesis tree depth overflowed usize".to_string())?; + budget.consume_tree_entry(&path, entry_depth)?; if entry.is_tree() { - stack.push((entry.object_sha, path)); + stack.push((entry.object_sha, path, entry_depth)); continue; } let blob = load_genesis_object(state, tenant, "Blob", repository_id, &entry.object_sha) @@ -1791,15 +1747,27 @@ async fn materialize_tree( entry.object_sha, blob_repository, repository_id )); } - let content = string_field_resolved(state, tenant, &blob.state.fields, "Content") - .await? - .ok_or_else(|| format!("Genesis blob {} is missing Content", entry.object_sha))?; + let content_bytes = blob_content_len(&blob.state.fields)?; + budget.consume_file(&path, content_bytes)?; if let Some(parent) = path.parent() { std::fs::create_dir_all(parent) .map_err(|e| format!("create directory '{}': {e}", parent.display()))?; } - std::fs::write(&path, decode_blob_content(&content)) - .map_err(|e| format!("write Genesis app file '{}': {e}", path.display()))?; + materialize_blob_content_field( + state, + tenant, + &blob.state.fields, + &path, + MAX_GENESIS_BUNDLE_FILE_BYTES, + ) + .await + .map_err(|error| { + format!( + "materialize Genesis blob {} to '{}': {error}", + entry.object_sha, + path.display() + ) + })?; } } Ok(()) @@ -1974,30 +1942,6 @@ fn validate_tree_entry_name(name: &str) -> Result<(), String> { Ok(()) } -fn decode_blob_content(value: &str) -> Vec { - base64::engine::general_purpose::STANDARD - .decode(value) - .unwrap_or_else(|_| value.as_bytes().to_vec()) -} - -fn decode_git_object_body(value: &str, expected_kind: &str) -> Result, String> { - let canonical = base64::engine::general_purpose::STANDARD - .decode(value) - .map_err(|e| format!("CanonicalBytes must be base64: {e}"))?; - let Some(nul) = canonical.iter().position(|byte| *byte == 0) else { - return Err("CanonicalBytes missing git object header terminator".to_string()); - }; - let header = std::str::from_utf8(&canonical[..nul]) - .map_err(|e| format!("CanonicalBytes header is not UTF-8: {e}"))?; - let expected_prefix = format!("{expected_kind} "); - if !header.starts_with(&expected_prefix) { - return Err(format!( - "CanonicalBytes header must start with '{expected_prefix}'" - )); - } - Ok(canonical[nul + 1..].to_vec()) -} - fn string_field(value: &Value, key: &str) -> Option { value .get(key) @@ -2006,52 +1950,14 @@ fn string_field(value: &Value, key: &str) -> Option { .map(ToString::to_string) } -async fn string_field_resolved( - state: &ServerState, - tenant: &TenantId, - value: &Value, - key: &str, -) -> Result, String> { - let Some(field) = value - .get(key) - .or_else(|| value.get("fields").and_then(|fields| fields.get(key))) - else { - return Ok(None); - }; - if let Some(value) = field.as_str() { - return Ok(Some(value.to_string())); - } - - let Some(blob_key) = field - .as_object() - .and_then(|object| object.get(FIELD_OVERFLOW_REF_KEY)) - .and_then(Value::as_str) - else { - return Ok(None); - }; - let Some(bytes) = state - .get_blob_with_legacy_fallback(tenant, blob_key) - .await - .map_err(|error| format!("read Genesis field overflow blob {blob_key}: {error}"))? - else { - return Err(format!("Genesis field overflow blob {blob_key} not found")); - }; - let restored: Value = serde_json::from_slice(&bytes) - .map_err(|error| format!("decode Genesis field overflow blob {blob_key}: {error}"))?; - restored - .as_str() - .map(ToString::to_string) - .ok_or_else(|| format!("Genesis field overflow blob {blob_key} is not a string")) - .map(Some) -} - fn genesis_cache_root(state: &ServerState, app_ref: &str) -> PathBuf { let root = if state.data_dir.as_os_str().is_empty() { std::env::temp_dir().join("temper-genesis-app-cache") } else { state.data_dir.join("genesis-app-cache") }; - root.join(sanitize_fragment(app_ref)) + let digest = format!("{:x}", sha2::Sha256::digest(app_ref.as_bytes())); + root.join(format!("{}-{}", sanitize_fragment(app_ref), &digest[..16])) } fn genesis_source_tenants() -> Vec { @@ -2105,6 +2011,9 @@ fn sanitize_fragment(input: &str) -> String { #[cfg(test)] mod tests { use base64::Engine as _; + use sha2::Digest as _; + use temper_runtime::ActorSystem; + use temper_spec::csdl::CsdlDocument; use super::*; @@ -2184,6 +2093,25 @@ mod tests { assert_eq!(sanitize_fragment("../"), "item"); } + #[test] + fn genesis_cache_roots_do_not_collide_after_sanitizing() { + let data_dir = tempfile::tempdir().expect("data dir"); + let mut state = ServerState::new( + ActorSystem::new("genesis-cache-key-test"), + CsdlDocument { + version: "4.0".to_string(), + schemas: Vec::new(), + }, + String::new(), + ); + state.data_dir = data_dir.path().to_path_buf(); + + assert_ne!( + genesis_cache_root(&state, "owner/app@a-b"), + genesis_cache_root(&state, "owner/app@a/b") + ); + } + #[test] fn parses_pinned_registry_app_refs() { let parsed = parse_registry_app_ref("temperpaw/paw-agent@abc123").expect("valid app ref"); @@ -2192,6 +2120,32 @@ mod tests { assert_eq!(parsed.version_hash.as_deref(), Some("abc123")); assert!(parse_registry_app_ref("paw-agent").is_err()); assert!(parse_registry_app_ref("temperpaw/paw-agent@").is_err()); + assert!(parse_registry_app_ref("../paw-agent@abc123").is_err()); + assert!(parse_registry_app_ref("temperpaw/../../target@abc123").is_err()); + assert!(parse_registry_app_ref("temperpaw//tmp/target@abc123").is_err()); + } + + #[test] + fn closure_admission_rejects_version_and_directory_collisions() { + let app = |owner: &str, name: &str, version: &str| GenesisAppBundle { + owner: owner.to_string(), + name: name.to_string(), + repository_id: format!("repo-{owner}-{name}"), + version_hash: version.to_string(), + }; + let mut admission = GenesisClosureAdmission::default(); + assert!(admission.admit(&app("acme", "notes", "v1")).unwrap()); + assert!(!admission.admit(&app("acme", "notes", "v1")).unwrap()); + + let version_error = admission + .admit(&app("acme", "notes", "v2")) + .expect_err("one dependency cannot resolve to two versions"); + assert!(version_error.contains("conflicting versions")); + + let directory_error = admission + .admit(&app("other", "notes", "v1")) + .expect_err("different owners cannot share one cache directory"); + assert!(directory_error.contains("collide on cache directory")); } #[test] @@ -2338,6 +2292,32 @@ mod tests { assert!(safe_bundle_relative_path(".git/config").is_err()); } + #[cfg(unix)] + #[test] + fn bundle_collection_rejects_symbolic_links() { + use std::os::unix::fs::symlink; + + let root = tempfile::tempdir().expect("bundle root"); + let outside = tempfile::NamedTempFile::new().expect("outside file"); + symlink(outside.path(), root.path().join("leak")).expect("symlink"); + + let error = collect_bundle_files(root.path(), &mut GenesisBundleBudget::new()) + .expect_err("symlink must be rejected"); + assert!(error.contains("symbolic link")); + } + + #[test] + fn manifest_read_rejects_oversized_file_before_allocation() { + let root = tempfile::tempdir().expect("app root"); + let manifest = std::fs::File::create(root.path().join("app.toml")).expect("manifest"); + manifest + .set_len(MAX_GENESIS_MANIFEST_BYTES + 1) + .expect("oversized manifest"); + + let error = read_manifest_dependencies(root.path()).expect_err("manifest budget"); + assert!(error.contains("budget")); + } + #[test] fn write_bundle_app_materializes_base64_files() { let temp_dir = std::env::temp_dir().join(format!( @@ -2387,4 +2367,130 @@ mod tests { fn source_tenants_default_to_default() { assert!(genesis_source_tenants().contains(&"default".to_string())); } + + fn blob_stream_test_state(data_dir: &Path) -> ServerState { + let mut state = ServerState::new( + ActorSystem::new("genesis-stream-test"), + CsdlDocument { + version: "4.0".to_string(), + schemas: Vec::new(), + }, + String::new(), + ); + state.data_dir = data_dir.to_path_buf(); + state + } + + async fn write_overflow_object( + data_dir: &Path, + key: &str, + serialized: &[u8], + ) -> std::path::PathBuf { + let path = data_dir.join("blobs").join(key); + tokio::fs::create_dir_all(path.parent().expect("overflow parent")) + .await + .expect("create overflow parent"); + tokio::fs::write(&path, serialized) + .await + .expect("write overflow object"); + path + } + + #[tokio::test] + async fn genesis_materializes_large_blob_content_from_stream() { + let data_dir = tempfile::tempdir().expect("Genesis data dir"); + let output_dir = tempfile::tempdir().expect("Genesis output dir"); + let state = blob_stream_test_state(data_dir.path()); + let content = vec![0x6bu8; 2 * 1024 * 1024]; + let serialized = + serde_json::to_vec(&base64::engine::general_purpose::STANDARD.encode(&content)) + .expect("serialize overflow JSON string"); + let key = format!( + "field-overflow/sha256/{:x}.json", + sha2::Sha256::digest(&serialized) + ); + write_overflow_object(data_dir.path(), &key, &serialized).await; + let fields = serde_json::json!({ + "Size": content.len() as u64, + "Content": { + "__temper_blob_ref": key, + "__temper_blob_size": serialized.len() as u64, + "__temper_blob_encoding": "json", + "__temper_blob_encoding": "json", + } + }); + let destination = output_dir.path().join("large.bin"); + + materialize_blob_content_field( + &state, + &TenantId::default(), + &fields, + &destination, + MAX_GENESIS_BUNDLE_FILE_BYTES, + ) + .await + .expect("stream materialization"); + + assert_eq!( + tokio::fs::read(&destination) + .await + .expect("materialized file"), + content + ); + } + + #[tokio::test] + async fn genesis_malformed_stream_never_replaces_destination() { + let data_dir = tempfile::tempdir().expect("Genesis data dir"); + let output_dir = tempfile::tempdir().expect("Genesis output dir"); + let state = blob_stream_test_state(data_dir.path()); + let serialized = b"\"!!!!\""; + let key = format!( + "field-overflow/sha256/{:x}.json", + sha2::Sha256::digest(serialized) + ); + write_overflow_object(data_dir.path(), &key, serialized).await; + let fields = serde_json::json!({ + "Size": 3, + "Content": { + "__temper_blob_ref": key, + "__temper_blob_size": 6, + "__temper_blob_encoding": "json", + "__temper_blob_encoding": "json", + } + }); + let destination = output_dir.path().join("existing.bin"); + tokio::fs::write(&destination, b"existing") + .await + .expect("seed destination"); + + let error = materialize_blob_content_field( + &state, + &TenantId::default(), + &fields, + &destination, + MAX_GENESIS_BUNDLE_FILE_BYTES, + ) + .await + .expect_err("malformed stream must fail"); + + assert!(error.contains("decode Genesis Blob.Content")); + assert_eq!( + tokio::fs::read(&destination) + .await + .expect("existing destination"), + b"existing" + ); + let staged = std::fs::read_dir(output_dir.path()) + .expect("list output dir") + .filter_map(Result::ok) + .filter(|entry| { + entry + .file_name() + .to_string_lossy() + .starts_with(".genesis-blob-") + }) + .count(); + assert_eq!(staged, 0, "RAII removes failed staged files"); + } } diff --git a/crates/temper-platform/src/genesis_install/blob_materialization.rs b/crates/temper-platform/src/genesis_install/blob_materialization.rs new file mode 100644 index 000000000..230f7f2ad --- /dev/null +++ b/crates/temper-platform/src/genesis_install/blob_materialization.rs @@ -0,0 +1,391 @@ +//! Bounded decoding and atomic publication of Genesis git object fields. + +use std::io::Read as _; +use std::path::Path; +use std::time::Duration; + +use futures::StreamExt as _; +use serde_json::Value; +use temper_runtime::tenant::TenantId; +use temper_server::state::ServerState; +use tokio::io::AsyncWriteExt as _; + +pub(super) const MAX_GENESIS_TREE_CANONICAL_BYTES: u64 = 16 * 1024 * 1024; +const GENESIS_STREAM_CHUNK_BYTES: usize = 64 * 1024; +const GENESIS_FILE_CREATE_TIMEOUT: Duration = Duration::from_secs(30); +const GENESIS_MATERIALIZATION_TIMEOUT: Duration = Duration::from_secs(30 * 60); + +pub(super) fn git_object_body<'a>( + canonical: &'a [u8], + expected_kind: &str, +) -> Result<&'a [u8], String> { + let Some(nul) = canonical.iter().position(|byte| *byte == 0) else { + return Err("CanonicalBytes missing git object header terminator".to_string()); + }; + let header = std::str::from_utf8(&canonical[..nul]) + .map_err(|error| format!("CanonicalBytes header is not UTF-8: {error}"))?; + let body = &canonical[nul + 1..]; + let expected_header = format!("{expected_kind} {}", body.len()); + if header != expected_header { + return Err(format!( + "CanonicalBytes header must be '{expected_header}', got '{header}'" + )); + } + Ok(body) +} + +fn u64_field(value: &Value, key: &str) -> Option { + value + .get(key) + .or_else(|| value.get("fields").and_then(|fields| fields.get(key))) + .and_then(Value::as_u64) +} + +fn encoded_json_base64_len(decoded_bytes: u64) -> Result { + decoded_bytes + .checked_add(2) + .map(|bytes| bytes / 3) + .and_then(|groups| groups.checked_mul(4)) + .and_then(|base64_bytes| base64_bytes.checked_add(2)) + .ok_or_else(|| "base64 JSON length overflowed u64".to_string()) +} + +fn decoded_field_len(value: &Value, kind: Option<&str>) -> Result { + let raw_size = u64_field(value, "Size") + .ok_or_else(|| "Genesis object is missing a non-negative Size".to_string())?; + match kind { + Some(kind) => raw_size + .checked_add(format!("{kind} {raw_size}\0").len() as u64) + .ok_or_else(|| "Genesis canonical object length overflowed u64".to_string()), + None => Ok(raw_size), + } +} + +pub(super) fn canonical_field_len(value: &Value, expected_kind: &str) -> Result { + decoded_field_len(value, Some(expected_kind)) +} + +pub(super) fn blob_content_len(value: &Value) -> Result { + decoded_field_len(value, None) +} + +pub(super) async fn read_canonical_field_bounded( + state: &ServerState, + tenant: &TenantId, + value: &Value, + expected_kind: &str, + max_bytes: u64, +) -> Result, String> { + let expected_bytes = decoded_field_len(value, Some(expected_kind))?; + if expected_bytes > max_bytes { + return Err(format!( + "Genesis {expected_kind} canonical object is {expected_bytes} bytes; budget is {max_bytes}" + )); + } + let Some(field) = value.get("CanonicalBytes").or_else(|| { + value + .get("fields") + .and_then(|fields| fields.get("CanonicalBytes")) + }) else { + return Err("Genesis object is missing CanonicalBytes".to_string()); + }; + + if let Some(encoded) = field.as_str() { + return decode_inline_base64_bounded(encoded, expected_bytes, max_bytes); + } + + let descriptor = temper_server::blobs::field_overflow_descriptor(field) + .ok_or_else(|| "Genesis CanonicalBytes has an invalid overflow descriptor".to_string())?; + read_overflow_base64_bounded(state, tenant, descriptor, expected_bytes, max_bytes).await +} + +fn decode_inline_base64_bounded( + encoded: &str, + expected_bytes: u64, + max_bytes: u64, +) -> Result, String> { + if expected_bytes > max_bytes { + return Err(format!( + "decoded Genesis field is {expected_bytes} bytes; budget is {max_bytes}" + )); + } + let mut decoder = base64::read::DecoderReader::new( + encoded.as_bytes(), + &base64::engine::general_purpose::STANDARD, + ); + let mut decoded = Vec::with_capacity(expected_bytes as usize); + decoder + .by_ref() + .take(max_bytes.saturating_add(1)) + .read_to_end(&mut decoded) + .map_err(|error| format!("decode inline Genesis base64 field: {error}"))?; + if decoded.len() as u64 != expected_bytes { + return Err(format!( + "decoded Genesis field is {} bytes; expected {expected_bytes}", + decoded.len() + )); + } + Ok(decoded) +} + +async fn read_overflow_base64_bounded( + state: &ServerState, + tenant: &TenantId, + descriptor: temper_server::blobs::FieldOverflowDescriptor<'_>, + expected_bytes: u64, + max_bytes: u64, +) -> Result, String> { + if expected_bytes > max_bytes { + return Err(format!( + "decoded Genesis field is {expected_bytes} bytes; budget is {max_bytes}" + )); + } + let expected_encoded = encoded_json_base64_len(expected_bytes)?; + if descriptor.serialized_bytes != expected_encoded { + return Err(format!( + "Genesis overflow descriptor is {} bytes; expected {expected_encoded}", + descriptor.serialized_bytes + )); + } + let encoded = match state + .stream_blob_object(tenant, descriptor.key, descriptor.serialized_bytes) + .await? + { + temper_server::blob_store::BlobStreamRead::Found(stream) => stream, + temper_server::blob_store::BlobStreamRead::Missing => { + return Err(format!( + "Genesis field overflow blob {} not found", + descriptor.key + )); + } + temper_server::blob_store::BlobStreamRead::TooLarge { .. } => { + return Err(format!( + "Genesis field overflow blob {} exceeds its descriptor", + descriptor.key + )); + } + }; + if encoded.content_length() != descriptor.serialized_bytes { + return Err(format!( + "Genesis field overflow blob {} length does not match its descriptor", + descriptor.key + )); + } + let encoded = encoded.verify_sha256(descriptor.sha256); + let mut stream = + temper_server::blob_store::decode_json_base64_stream(encoded, expected_bytes).into_stream(); + let mut decoded = Vec::with_capacity(expected_bytes as usize); + while let Some(chunk) = stream.next().await { + let chunk = chunk.map_err(|error| { + format!( + "decode Genesis field overflow blob {}: {error}", + descriptor.key + ) + })?; + if decoded.len().saturating_add(chunk.len()) as u64 > max_bytes { + return Err(format!( + "decoded Genesis field overflow blob {} exceeded {max_bytes} bytes", + descriptor.key + )); + } + decoded.extend_from_slice(&chunk); + } + if decoded.len() as u64 != expected_bytes { + return Err(format!( + "decoded Genesis field overflow blob {} is {} bytes; expected {expected_bytes}", + descriptor.key, + decoded.len() + )); + } + Ok(decoded) +} + +pub(super) async fn materialize_blob_content_field( + state: &ServerState, + tenant: &TenantId, + value: &Value, + destination: &Path, + max_bytes: u64, +) -> Result<(), String> { + let deadline = tokio::time::Instant::now() + GENESIS_MATERIALIZATION_TIMEOUT; // determinism-ok: production file I/O deadline + let expected_bytes = decoded_field_len(value, None)?; + if expected_bytes > max_bytes { + return Err(format!( + "Genesis Blob.Content is {expected_bytes} bytes; budget is {max_bytes}" + )); + } + let Some(field) = value + .get("Content") + .or_else(|| value.get("fields").and_then(|fields| fields.get("Content"))) + else { + return Err("Genesis Blob is missing Content".to_string()); + }; + let parent = destination.parent().ok_or_else(|| { + format!( + "Genesis destination '{}' has no parent", + destination.display() + ) + })?; + let parent = parent.to_path_buf(); + let staged = tokio::time::timeout( + GENESIS_FILE_CREATE_TIMEOUT, + // Production materialization filesystem boundary; never simulation-visible. + tokio::task::spawn_blocking(move || { + tempfile::Builder::new() + .prefix(".genesis-blob-") + .tempfile_in(parent) + }), + ) + .await + .map_err(|_| "create staged Genesis file timed out".to_string())? + .map_err(|error| format!("staged Genesis file task failed: {error}"))? + .map_err(|error| format!("create staged Genesis file: {error}"))?; + let (file, staged_path) = staged.into_parts(); + let mut output = tokio::fs::File::from_std(file); + let written = if let Some(encoded) = field.as_str() { + write_inline_base64(&mut output, encoded, expected_bytes, deadline).await? + } else { + let descriptor = temper_server::blobs::field_overflow_descriptor(field) + .ok_or_else(|| "Genesis Blob.Content has an invalid overflow descriptor".to_string())?; + write_overflow_base64( + state, + tenant, + &mut output, + descriptor, + expected_bytes, + deadline, + ) + .await? + }; + if written != expected_bytes { + return Err(format!( + "Genesis Blob.Content decoded {written} bytes; expected {expected_bytes}" + )); + } + tokio::time::timeout_at(deadline, output.flush()) + .await + .map_err(|_| "flush staged Genesis file exceeded materialization deadline".to_string())? + .map_err(|error| format!("flush staged Genesis file: {error}"))?; + tokio::time::timeout_at(deadline, output.sync_data()) + .await + .map_err(|_| "sync staged Genesis file exceeded materialization deadline".to_string())? + .map_err(|error| format!("sync staged Genesis file: {error}"))?; + drop(output); + staged_path.persist(destination).map_err(|error| { + format!( + "publish staged Genesis file '{}': {}", + destination.display(), + error.error + ) + })?; + Ok(()) +} + +async fn write_inline_base64( + output: &mut tokio::fs::File, + encoded: &str, + expected_bytes: u64, + deadline: tokio::time::Instant, +) -> Result { + let mut decoder = base64::read::DecoderReader::new( + encoded.as_bytes(), + &base64::engine::general_purpose::STANDARD, + ); + let mut buffer = vec![0u8; GENESIS_STREAM_CHUNK_BYTES]; + let mut written = 0u64; + loop { + let read = decoder + .read(&mut buffer) + .map_err(|error| format!("decode inline Genesis Blob.Content: {error}"))?; + if read == 0 { + break; + } + written = written + .checked_add(read as u64) + .ok_or_else(|| "Genesis Blob.Content byte count overflowed u64".to_string())?; + if written > expected_bytes { + return Err("Genesis Blob.Content exceeds its declared Size".to_string()); + } + tokio::time::timeout_at(deadline, output.write_all(&buffer[..read])) + .await + .map_err(|_| { + "writing inline Genesis Blob.Content exceeded materialization deadline".to_string() + })? + .map_err(|error| format!("write staged Genesis file: {error}"))?; + } + Ok(written) +} + +async fn write_overflow_base64( + state: &ServerState, + tenant: &TenantId, + output: &mut tokio::fs::File, + descriptor: temper_server::blobs::FieldOverflowDescriptor<'_>, + expected_bytes: u64, + deadline: tokio::time::Instant, +) -> Result { + let expected_encoded = encoded_json_base64_len(expected_bytes)?; + if descriptor.serialized_bytes != expected_encoded { + return Err(format!( + "Genesis Blob.Content descriptor is {} bytes; expected {expected_encoded}", + descriptor.serialized_bytes + )); + } + let encoded = match state + .stream_blob_object(tenant, descriptor.key, descriptor.serialized_bytes) + .await? + { + temper_server::blob_store::BlobStreamRead::Found(stream) => stream, + temper_server::blob_store::BlobStreamRead::Missing => { + return Err(format!( + "Genesis Blob.Content overflow object {} not found", + descriptor.key + )); + } + temper_server::blob_store::BlobStreamRead::TooLarge { .. } => { + return Err(format!( + "Genesis Blob.Content overflow object {} exceeds its descriptor", + descriptor.key + )); + } + }; + if encoded.content_length() != descriptor.serialized_bytes { + return Err(format!( + "Genesis Blob.Content overflow object {} length does not match its descriptor", + descriptor.key + )); + } + let encoded = encoded.verify_sha256(descriptor.sha256); + let mut stream = + temper_server::blob_store::decode_json_base64_stream(encoded, expected_bytes).into_stream(); + let mut written = 0u64; + loop { + let next = tokio::time::timeout_at(deadline, stream.next()) + .await + .map_err(|_| { + "reading Genesis Blob.Content exceeded materialization deadline".to_string() + })?; + let Some(chunk) = next else { + break; + }; + let chunk = chunk.map_err(|error| { + format!( + "decode Genesis Blob.Content overflow object {}: {error}", + descriptor.key + ) + })?; + written = written + .checked_add(chunk.len() as u64) + .ok_or_else(|| "Genesis Blob.Content byte count overflowed u64".to_string())?; + if written > expected_bytes { + return Err("Genesis Blob.Content exceeds its declared Size".to_string()); + } + tokio::time::timeout_at(deadline, output.write_all(&chunk)) + .await + .map_err(|_| { + "writing Genesis Blob.Content exceeded materialization deadline".to_string() + })? + .map_err(|error| format!("write staged Genesis file: {error}"))?; + } + Ok(written) +} diff --git a/crates/temper-platform/src/genesis_install/bundle_transport.rs b/crates/temper-platform/src/genesis_install/bundle_transport.rs new file mode 100644 index 000000000..765102d9c --- /dev/null +++ b/crates/temper-platform/src/genesis_install/bundle_transport.rs @@ -0,0 +1,63 @@ +//! Bounded transport decoding for remote Genesis bundle responses. + +use std::time::Duration; + +use futures::StreamExt as _; + +use super::GenesisRegistryBundleResponse; + +const MAX_BUNDLE_RESPONSE_BYTES: usize = 128 * 1024 * 1024; +const MAX_ERROR_RESPONSE_BYTES: usize = 64 * 1024; +const BUNDLE_RESPONSE_IDLE_TIMEOUT: Duration = Duration::from_secs(30); +const BUNDLE_RESPONSE_TOTAL_TIMEOUT: Duration = Duration::from_secs(5 * 60); + +pub(super) async fn decode_bundle_response( + response: reqwest::Response, + url: &str, +) -> Result { + let status = response.status(); + let max_bytes = if status.is_success() { + MAX_BUNDLE_RESPONSE_BYTES + } else { + MAX_ERROR_RESPONSE_BYTES + }; + if response + .content_length() + .is_some_and(|length| length > max_bytes as u64) + { + return Err(format!( + "Genesis bundle response from {url} exceeds {max_bytes} bytes" + )); + } + + let deadline = tokio::time::Instant::now() + BUNDLE_RESPONSE_TOTAL_TIMEOUT; + let mut stream = response.bytes_stream(); + let mut body = Vec::new(); + loop { + let now = tokio::time::Instant::now(); + if now >= deadline { + return Err(format!("Genesis bundle response from {url} timed out")); + } + let wait = BUNDLE_RESPONSE_IDLE_TIMEOUT.min(deadline.saturating_duration_since(now)); + let next = tokio::time::timeout(wait, stream.next()) + .await + .map_err(|_| format!("Genesis bundle response from {url} stalled"))?; + let Some(chunk) = next else { break }; + let chunk = + chunk.map_err(|error| format!("read Genesis bundle response {url}: {error}"))?; + if body.len().saturating_add(chunk.len()) > max_bytes { + return Err(format!( + "Genesis bundle response from {url} exceeds {max_bytes} bytes" + )); + } + body.extend_from_slice(&chunk); + } + + if !status.is_success() { + return Err(format!( + "request Genesis bundle {url} returned {status}: {}", + String::from_utf8_lossy(&body).trim() + )); + } + serde_json::from_slice(&body).map_err(|error| format!("decode Genesis bundle {url}: {error}")) +} diff --git a/crates/temper-platform/src/genesis_install/bundles.rs b/crates/temper-platform/src/genesis_install/bundles.rs new file mode 100644 index 000000000..efe112bab --- /dev/null +++ b/crates/temper-platform/src/genesis_install/bundles.rs @@ -0,0 +1,470 @@ +//! Bounded import, export, and publication of Genesis registry bundles. + +use std::collections::BTreeSet; +use std::io::Read as _; +use std::path::{Component, Path, PathBuf}; + +use base64::Engine as _; + +use super::bundle_transport::decode_bundle_response; +use super::cache_paths::{ + app_cache_dir, replace_directory, validate_git_object_id, validate_identity_component, +}; +use super::{ + GenesisRegistryBundleApp, GenesisRegistryBundleFile, GenesisRegistryBundleResponse, + RegistryAppRef, parse_registry_app_ref, +}; + +const MAX_GENESIS_BUNDLE_FILES: usize = 4096; +pub(super) const MAX_GENESIS_BUNDLE_APPS: usize = 256; +pub(super) const MAX_GENESIS_BUNDLE_FILE_BYTES: u64 = 16 * 1024 * 1024; +const MAX_GENESIS_BUNDLE_TOTAL_BYTES: u64 = 64 * 1024 * 1024; +const MAX_GENESIS_TREE_OBJECTS: usize = 8192; +const MAX_GENESIS_TREE_ENTRIES: usize = 16_384; +const MAX_GENESIS_TREE_DEPTH: usize = 128; +const MAX_GENESIS_TREE_TOTAL_BYTES: u64 = 64 * 1024 * 1024; + +pub(super) async fn materialize_registry_app_closure_via_bundle( + registry_url: &str, + registry_tenant: &str, + root_ref: RegistryAppRef, + cache_root: &Path, +) -> Result, String> { + let Some(version_hash) = root_ref.version_hash.as_deref() else { + return Err("bundle fetch requires a pinned root app ref".to_string()); + }; + let version_hash = validate_git_object_id(version_hash)?; + let bundle_url = format!( + "{}/api/genesis/apps/{}/{}/versions/{}/bundle", + registry_url.trim_end_matches('/'), + root_ref.owner, + root_ref.name, + version_hash + ); + let response = reqwest::Client::new() + .get(&bundle_url) + .header("X-Tenant-Id", registry_tenant) + .send() + .await + .map_err(|error| format!("request Genesis bundle {bundle_url}: {error}"))?; + let bundle = decode_bundle_response(response, &bundle_url).await?; + validate_registry_bundle(&bundle, &root_ref, registry_tenant)?; + + let cache_parent = cache_root.parent().ok_or_else(|| { + format!( + "Genesis registry bundle cache '{}' has no parent", + cache_root.display() + ) + })?; + std::fs::create_dir_all(cache_parent).map_err(|error| { + format!( + "create Genesis registry bundle cache parent '{}': {error}", + cache_parent.display() + ) + })?; + let staged_cache = tempfile::Builder::new() + .prefix(".genesis-bundle-") + .tempdir_in(cache_parent) + .map_err(|error| format!("create staged Genesis bundle cache: {error}"))?; + + let mut refs = Vec::new(); + for app in bundle.apps { + validate_identity_component("owner", &app.owner)?; + let app_dir = app_cache_dir(staged_cache.path(), &app.name)?; + write_bundle_app(&app_dir, &app)?; + refs.push(RegistryAppRef { + owner: app.owner, + name: app.name, + version_hash: Some(app.version_hash), + }); + } + replace_directory(staged_cache.keep(), cache_root)?; + Ok(refs) +} + +fn validate_registry_bundle( + bundle: &GenesisRegistryBundleResponse, + root_ref: &RegistryAppRef, + registry_tenant: &str, +) -> Result<(), String> { + if bundle.registry_tenant != registry_tenant { + return Err(format!( + "Genesis bundle tenant '{}' does not match requested tenant '{registry_tenant}'", + bundle.registry_tenant + )); + } + let bundle_ref = parse_registry_app_ref(&bundle.app_ref)?; + let expected_hash = validate_git_object_id( + root_ref + .version_hash + .as_deref() + .ok_or_else(|| "Genesis bundle root is not pinned".to_string())?, + )?; + let bundle_hash = validate_git_object_id( + bundle_ref + .version_hash + .as_deref() + .ok_or_else(|| "Genesis bundle response app_ref is not pinned".to_string())?, + )?; + if bundle_ref.owner != root_ref.owner + || bundle_ref.name != root_ref.name + || bundle_hash != expected_hash + { + return Err( + "Genesis bundle response does not match the requested pinned app ref".to_string(), + ); + } + if bundle.apps.len() > MAX_GENESIS_BUNDLE_APPS { + return Err(format!( + "Genesis bundle contains {} apps; budget is {MAX_GENESIS_BUNDLE_APPS}", + bundle.apps.len() + )); + } + + let mut app_names = BTreeSet::new(); + let mut file_count = 0usize; + let mut decoded_budget = MAX_GENESIS_BUNDLE_TOTAL_BYTES; + let max_encoded_file_bytes = MAX_GENESIS_BUNDLE_FILE_BYTES + .checked_add(2) + .map(|bytes| bytes / 3) + .and_then(|groups| groups.checked_mul(4)) + .expect("static bundle file budget must encode without overflow"); + let mut root_matches = 0usize; + for app in &bundle.apps { + validate_identity_component("owner", &app.owner)?; + validate_identity_component("app name", &app.name)?; + let app_hash = validate_git_object_id(&app.version_hash)?; + if !app_names.insert(app.name.as_str()) { + return Err(format!( + "Genesis bundle contains duplicate app directory name '{}'", + app.name + )); + } + if app.owner == root_ref.owner && app.name == root_ref.name { + root_matches += 1; + if app_hash != expected_hash { + return Err( + "Genesis bundle root app version does not match requested hash".to_string(), + ); + } + } + for file in &app.files { + file_count = file_count + .checked_add(1) + .ok_or_else(|| "Genesis bundle file count overflowed usize".to_string())?; + if file_count > MAX_GENESIS_BUNDLE_FILES { + return Err(format!( + "Genesis bundle file count exceeds budget {MAX_GENESIS_BUNDLE_FILES}" + )); + } + safe_bundle_relative_path(&file.path)?; + if file.content_base64.len() as u64 > max_encoded_file_bytes { + return Err(format!( + "Genesis bundle file '{}' exceeds the encoded per-file budget", + file.path + )); + } + let decoded_upper_bound = (file.content_base64.len() as u64) + .div_ceil(4) + .saturating_mul(3); + if decoded_upper_bound > decoded_budget { + return Err(format!( + "Genesis bundle file '{}' exceeds the remaining aggregate byte budget", + file.path + )); + } + decoded_budget -= decoded_upper_bound; + } + } + if root_matches != 1 { + return Err("Genesis bundle must contain exactly one requested root app".to_string()); + } + Ok(()) +} + +pub(super) fn write_bundle_app( + app_dir: &Path, + app: &GenesisRegistryBundleApp, +) -> Result<(), String> { + if app_dir.exists() { + std::fs::remove_dir_all(app_dir).map_err(|error| { + format!("clear Genesis bundle app '{}': {error}", app_dir.display()) + })?; + } + std::fs::create_dir_all(app_dir) + .map_err(|error| format!("create Genesis bundle app '{}': {error}", app_dir.display()))?; + + for file in &app.files { + let rel = safe_bundle_relative_path(&file.path)?; + let path = app_dir.join(&rel); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).map_err(|error| { + format!("create bundle file parent '{}': {error}", parent.display()) + })?; + } + let bytes = base64::engine::general_purpose::STANDARD + .decode(&file.content_base64) + .map_err(|error| format!("decode bundle file '{}': {error}", file.path))?; + if bytes.len() as u64 > MAX_GENESIS_BUNDLE_FILE_BYTES { + return Err(format!( + "decoded Genesis bundle file '{}' exceeds {MAX_GENESIS_BUNDLE_FILE_BYTES} bytes", + file.path + )); + } + std::fs::write(&path, bytes) + .map_err(|error| format!("write bundle file '{}': {error}", path.display()))?; + } + Ok(()) +} + +pub(super) fn safe_bundle_relative_path(path: &str) -> Result { + let rel = PathBuf::from(path); + if rel.as_os_str().is_empty() { + return Err("bundle file path must not be empty".to_string()); + } + let mut safe = PathBuf::new(); + for component in rel.components() { + match component { + Component::Normal(part) => { + if part == "target" || part == ".git" { + return Err(format!( + "bundle file path '{}' contains forbidden component '{}'", + path, + part.to_string_lossy() + )); + } + safe.push(part); + } + _ => { + return Err(format!( + "bundle file path '{}' must be relative and must not contain '..'", + path + )); + } + } + } + Ok(safe) +} + +pub(super) struct GenesisBundleBudget { + files_remaining: usize, + bytes_remaining: u64, + tree_objects_remaining: usize, + tree_entries_remaining: usize, + tree_bytes_remaining: u64, +} + +impl GenesisBundleBudget { + pub(super) fn new() -> Self { + Self { + files_remaining: MAX_GENESIS_BUNDLE_FILES, + bytes_remaining: MAX_GENESIS_BUNDLE_TOTAL_BYTES, + tree_objects_remaining: MAX_GENESIS_TREE_OBJECTS, + tree_entries_remaining: MAX_GENESIS_TREE_ENTRIES, + tree_bytes_remaining: MAX_GENESIS_TREE_TOTAL_BYTES, + } + } + + pub(super) fn consume_file(&mut self, path: &Path, bytes: u64) -> Result<(), String> { + if self.files_remaining == 0 { + return Err(format!( + "Genesis bundle file count exceeded budget {MAX_GENESIS_BUNDLE_FILES}" + )); + } + if bytes > MAX_GENESIS_BUNDLE_FILE_BYTES { + return Err(format!( + "Genesis bundle file '{}' is {bytes} bytes; per-file budget is {MAX_GENESIS_BUNDLE_FILE_BYTES}", + path.display() + )); + } + if bytes > self.bytes_remaining { + return Err(format!( + "Genesis bundle file '{}' exceeds the remaining aggregate byte budget {}", + path.display(), + self.bytes_remaining + )); + } + self.files_remaining -= 1; + self.bytes_remaining -= bytes; + Ok(()) + } + + pub(super) fn consume_tree(&mut self, path: &Path, canonical_bytes: u64) -> Result<(), String> { + if self.tree_objects_remaining == 0 { + return Err(format!( + "Genesis tree count exceeded budget {MAX_GENESIS_TREE_OBJECTS} at '{}'", + path.display() + )); + } + if canonical_bytes > self.tree_bytes_remaining { + return Err(format!( + "Genesis tree '{}' exceeds the remaining aggregate tree byte budget {}", + path.display(), + self.tree_bytes_remaining + )); + } + self.tree_objects_remaining -= 1; + self.tree_bytes_remaining -= canonical_bytes; + Ok(()) + } + + pub(super) fn consume_tree_entry(&mut self, path: &Path, depth: usize) -> Result<(), String> { + if depth > MAX_GENESIS_TREE_DEPTH { + return Err(format!( + "Genesis tree path '{}' exceeds depth budget {MAX_GENESIS_TREE_DEPTH}", + path.display() + )); + } + if self.tree_entries_remaining == 0 { + return Err(format!( + "Genesis tree entries exceeded budget {MAX_GENESIS_TREE_ENTRIES} at '{}'", + path.display() + )); + } + self.tree_entries_remaining -= 1; + Ok(()) + } +} + +pub(super) fn collect_bundle_files( + app_dir: &Path, + budget: &mut GenesisBundleBudget, +) -> Result, String> { + let mut paths = Vec::new(); + collect_bundle_file_paths(app_dir, app_dir, &mut paths, budget.files_remaining)?; + paths.sort(); + let mut files = Vec::new(); + for path in paths { + let rel = path + .strip_prefix(app_dir) + .map_err(|error| format!("strip bundle path '{}': {error}", path.display()))? + .to_string_lossy() + .replace('\\', "/"); + let metadata = std::fs::symlink_metadata(&path) + .map_err(|error| format!("stat bundle file '{}': {error}", path.display()))?; + if metadata.file_type().is_symlink() || !metadata.is_file() { + return Err(format!( + "Genesis bundle path '{}' must be a regular file", + path.display() + )); + } + budget.consume_file(&path, metadata.len())?; + let mut bytes = Vec::with_capacity(metadata.len() as usize); + std::fs::File::open(&path) + .map_err(|error| format!("open bundle file '{}': {error}", path.display()))? + .take(metadata.len().saturating_add(1)) + .read_to_end(&mut bytes) + .map_err(|error| format!("read bundle file '{}': {error}", path.display()))?; + if bytes.len() as u64 != metadata.len() { + return Err(format!( + "Genesis bundle file '{}' changed size while reading", + path.display() + )); + } + files.push(GenesisRegistryBundleFile { + path: rel, + content_base64: base64::engine::general_purpose::STANDARD.encode(bytes), + }); + } + Ok(files) +} + +fn collect_bundle_file_paths( + root: &Path, + dir: &Path, + paths: &mut Vec, + file_budget: usize, +) -> Result<(), String> { + let mut entries: Vec<_> = std::fs::read_dir(dir) + .map_err(|error| format!("read bundle directory '{}': {error}", dir.display()))? + .filter_map(|entry| entry.ok()) + .collect(); + entries.sort_by_key(|entry| entry.file_name()); + for entry in entries { + let path = entry.path(); + let file_type = entry + .file_type() + .map_err(|error| format!("stat Genesis bundle entry '{}': {error}", path.display()))?; + if file_type.is_symlink() { + return Err(format!( + "Genesis bundle entry '{}' must not be a symbolic link", + path.display() + )); + } + let rel = path + .strip_prefix(root) + .map_err(|error| format!("strip bundle path '{}': {error}", path.display()))?; + if rel.components().any(|component| { + matches!(component, Component::Normal(part) if part == "target" || part == ".git") + }) { + if file_type.is_dir() { + tracing::warn!( + path = %path.display(), + "Skipping forbidden generated directory in Genesis bundle export" + ); + } + continue; + } + if file_type.is_dir() { + collect_bundle_file_paths(root, &path, paths, file_budget)?; + } else if file_type.is_file() { + if paths.len() >= file_budget { + return Err(format!( + "Genesis bundle file count exceeded budget {MAX_GENESIS_BUNDLE_FILES}" + )); + } + paths.push(path); + } else { + return Err(format!( + "Genesis bundle entry '{}' must be a regular file or directory", + path.display() + )); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn materialization_budget_rejects_files_trees_entries_and_depth_before_io() { + let path = Path::new("app/file"); + + let mut file_budget = GenesisBundleBudget::new(); + assert!( + file_budget + .consume_file(path, MAX_GENESIS_BUNDLE_FILE_BYTES + 1) + .expect_err("oversized file") + .contains("per-file budget") + ); + + let mut tree_budget = GenesisBundleBudget::new(); + assert!( + tree_budget + .consume_tree(path, MAX_GENESIS_TREE_TOTAL_BYTES + 1) + .expect_err("oversized aggregate tree input") + .contains("tree byte budget") + ); + assert!( + tree_budget + .consume_tree_entry(path, MAX_GENESIS_TREE_DEPTH + 1) + .expect_err("excessive depth") + .contains("depth budget") + ); + + let mut entry_budget = GenesisBundleBudget::new(); + for _ in 0..MAX_GENESIS_TREE_ENTRIES { + entry_budget + .consume_tree_entry(path, 1) + .expect("entry within budget"); + } + assert!( + entry_budget + .consume_tree_entry(path, 1) + .expect_err("entry count exhausted") + .contains("tree entries") + ); + } +} diff --git a/crates/temper-platform/src/genesis_install/cache_paths.rs b/crates/temper-platform/src/genesis_install/cache_paths.rs new file mode 100644 index 000000000..8632b9a77 --- /dev/null +++ b/crates/temper-platform/src/genesis_install/cache_paths.rs @@ -0,0 +1,150 @@ +//! Safe cache paths and staged directory publication for Genesis apps. + +use std::path::{Path, PathBuf}; + +const MAX_IDENTITY_COMPONENT_BYTES: usize = 128; + +pub(super) fn validate_identity_component(label: &str, value: &str) -> Result<(), String> { + if value.is_empty() + || value.len() > MAX_IDENTITY_COMPONENT_BYTES + || matches!(value, "." | "..") + || !value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.')) + { + return Err(format!( + "Genesis {label} must be a single ASCII identifier component (letters, digits, '.', '-', '_')" + )); + } + Ok(()) +} + +pub(super) fn validate_git_object_id(value: &str) -> Result<&str, String> { + let value = value.trim_start_matches('@'); + if value.len() != 40 + || !value + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return Err("Genesis version hash must be a 40-character lowercase Git SHA-1".to_string()); + } + Ok(value) +} + +pub(super) fn app_cache_dir(cache_root: &Path, app_name: &str) -> Result { + validate_identity_component("app name", app_name)?; + Ok(cache_root.join(app_name)) +} + +pub(super) fn replace_directory(staged: PathBuf, destination: &Path) -> Result<(), String> { + let Some(parent) = destination.parent() else { + return Err(format!( + "Genesis cache destination '{}' has no parent", + destination.display() + )); + }; + if std::fs::symlink_metadata(destination).is_err() { + return std::fs::rename(&staged, destination).map_err(|error| { + let cleanup = cleanup_staged_directory(&staged); + format!( + "publish staged Genesis cache '{}' to '{}': {error}", + staged.display(), + destination.display() + ) + &cleanup + }); + } + + let backup = tempfile::Builder::new() + .prefix(".genesis-backup-") + .tempdir_in(parent) + .map_err(|error| format!("create Genesis cache rollback directory: {error}"))?; + let previous = backup.path().join("previous"); + std::fs::rename(destination, &previous).map_err(|error| { + format!( + "stage previous Genesis cache '{}' for replacement: {error}", + destination.display() + ) + })?; + if let Err(publish_error) = std::fs::rename(&staged, destination) { + let cleanup = cleanup_staged_directory(&staged); + return match std::fs::rename(&previous, destination) { + Ok(()) => Err(format!( + "publish staged Genesis cache '{}' to '{}': {publish_error}; previous cache restored", + staged.display(), + destination.display() + ) + &cleanup), + Err(rollback_error) => { + let recovery_path = backup.keep(); + Err(format!( + "publish staged Genesis cache '{}' to '{}': {publish_error}; rollback failed: {rollback_error}; previous cache retained at '{}'{}", + staged.display(), + destination.display(), + recovery_path.display(), + cleanup, + )) + } + }; + } + Ok(()) +} + +fn cleanup_staged_directory(staged: &Path) -> String { + match std::fs::remove_dir_all(staged) { + Ok(()) => String::new(), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => String::new(), + Err(error) => format!("; failed to clean staged directory: {error}"), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn app_cache_dir_rejects_path_components() { + let root = Path::new("/safe/cache"); + for value in ["", ".", "..", "../target", "/tmp/target", "a/b", "a\\b"] { + assert!(app_cache_dir(root, value).is_err(), "accepted {value:?}"); + } + assert_eq!( + app_cache_dir(root, "safe-app_1.0").expect("safe app name"), + root.join("safe-app_1.0") + ); + } + + #[test] + fn git_object_ids_are_exact_lowercase_sha1() { + let hash = "0123456789abcdef0123456789abcdef01234567"; + assert_eq!(validate_git_object_id(hash), Ok(hash)); + assert_eq!(validate_git_object_id(&format!("@{hash}")), Ok(hash)); + for invalid in [ + "abc123", + "../target", + "-deadbeef", + "ABCDEF0123456789ABCDEF0123456789ABCDEF01", + ] { + assert!( + validate_git_object_id(invalid).is_err(), + "accepted {invalid:?}" + ); + } + } + + #[test] + fn failed_publish_restores_previous_directory() { + let root = tempfile::tempdir().expect("cache root"); + let destination = root.path().join("app"); + std::fs::create_dir(&destination).expect("destination"); + std::fs::write(destination.join("marker"), b"previous").expect("marker"); + let missing_staged = root.path().join("missing-stage"); + + let error = replace_directory(missing_staged, &destination) + .expect_err("missing staged directory must fail"); + + assert!(error.contains("previous cache restored")); + assert_eq!( + std::fs::read(destination.join("marker")).expect("restored marker"), + b"previous" + ); + } +} diff --git a/crates/temper-platform/src/lib.rs b/crates/temper-platform/src/lib.rs index 756496fc8..ae3b7eb9a 100644 --- a/crates/temper-platform/src/lib.rs +++ b/crates/temper-platform/src/lib.rs @@ -11,8 +11,8 @@ pub mod bootstrap; pub mod deploy; pub mod genesis_install; pub mod hooks; -pub mod identity_cache; pub mod integration; +mod operator_manage_policies; pub mod optimization; pub mod os_apps; pub mod protocol; diff --git a/crates/temper-platform/src/operator_manage_policies.rs b/crates/temper-platform/src/operator_manage_policies.rs new file mode 100644 index 000000000..cf1322008 --- /dev/null +++ b/crates/temper-platform/src/operator_manage_policies.rs @@ -0,0 +1,139 @@ +//! Narrow operator `manage_policies` permit seeded at credential bootstrap. +//! +//! See ADR-0172. This is ordinary Cedar — merged into live tenant policy, +//! persisted as a granular row, not a code bypass and not permit-all. + +use temper_server::authz::persist_and_activate_policy; + +use crate::state::PlatformState; + +/// Stable granular policy id for the operator bootstrap permit. +pub const OPERATOR_MANAGE_POLICIES_POLICY_ID: &str = "operator-bootstrap-manage-policies"; + +/// Cedar statement granting a verified operator `manage_policies` on this tenant. +pub fn operator_manage_policies_cedar(tenant: &str) -> String { + debug_assert!( + !tenant.is_empty() && !tenant.contains('"'), + "tenant id must be a Cedar-safe identifier" + ); + format!( + r#"permit( + principal is Agent, + action == Action::"manage_policies", + resource == PolicySet::"{tenant}" +) when {{ + principal.agent_type == "operator" && + principal.agentTypeVerified == true +}};"# + ) +} + +/// Append `statement` to `existing` when it is not already present. +pub fn merge_cedar_statement(existing: &str, statement: &str) -> String { + let statement = statement.trim(); + let existing = existing.trim_end(); + if statement.is_empty() || existing.contains(statement) { + return existing.to_string(); + } + if existing.is_empty() { + statement.to_string() + } else { + format!("{existing}\n{statement}") + } +} + +fn live_tenant_policy_text(state: &PlatformState, tenant: &str) -> String { + if let Some(active_text) = state + .server + .authz + .get_tenant_policy_text(tenant) + .filter(|policy_text| !policy_text.trim().is_empty()) + { + return active_text; + } + + state + .server + .tenant_policies + .read() + .ok() + .and_then(|policies| policies.get(tenant).cloned()) + .unwrap_or_default() +} + +/// Merge, activate, and persist the operator `manage_policies` permit for `tenant`. +/// +/// Idempotent: re-bootstrap does not duplicate the live statement or the +/// granular row. Does not replace existing app Cedar. +pub async fn seed_operator_manage_policies(state: &PlatformState, tenant: &str) { + assert!( + !tenant.is_empty() && !tenant.contains('"'), + "tenant id must be a Cedar-safe identifier" + ); + + let statement = operator_manage_policies_cedar(tenant); + let existing = live_tenant_policy_text(state, tenant); + let merged = merge_cedar_statement(&existing, &statement); + + if let Err(error) = state.server.authz.reload_tenant_policies(tenant, &merged) { + tracing::warn!( + tenant, + error = %error, + "failed to activate operator manage_policies Cedar permit" + ); + return; + } + + { + let mut policies = state + .server + .tenant_policies + .write() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + policies.insert(tenant.to_string(), merged); + } + + persist_and_activate_policy( + &state.server, + tenant, + OPERATOR_MANAGE_POLICIES_POLICY_ID, + &statement, + "bootstrap", + ) + .await; + + tracing::info!( + tenant, + policy_id = OPERATOR_MANAGE_POLICIES_POLICY_ID, + "operator manage_policies Cedar permit seeded" + ); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn merge_cedar_statement_is_idempotent_and_preserves_existing() { + let statement = operator_manage_policies_cedar("acme"); + let app = r#"permit(principal, action == Action::"read", resource is Issue);"#; + + let once = merge_cedar_statement(app, &statement); + assert!(once.contains("resource is Issue")); + assert!(once.contains(r#"Action::"manage_policies""#)); + assert_eq!(merge_cedar_statement(&once, &statement), once); + + assert_eq!(merge_cedar_statement("", &statement), statement.trim()); + } + + #[test] + fn operator_manage_policies_cedar_is_tenant_scoped() { + let acme = operator_manage_policies_cedar("acme"); + let other = operator_manage_policies_cedar("other"); + assert!(acme.contains(r#"PolicySet::"acme""#)); + assert!(!acme.contains(r#"PolicySet::"other""#)); + assert!(other.contains(r#"PolicySet::"other""#)); + assert!(acme.contains(r#"principal.agent_type == "operator""#)); + assert!(acme.contains("principal.agentTypeVerified == true")); + } +} diff --git a/crates/temper-platform/src/os_apps/mod_test.rs b/crates/temper-platform/src/os_apps/mod_test.rs index 42f887e25..a8bc15fe5 100644 --- a/crates/temper-platform/src/os_apps/mod_test.rs +++ b/crates/temper-platform/src/os_apps/mod_test.rs @@ -16,6 +16,21 @@ use temper_spec::csdl::parse_csdl; use temper_store_turso::TursoSpecVerificationUpdate; use temper_verify::cascade::VerificationCascade; +fn test_admin_security_context(principal_id: &str) -> SecurityContext { + SecurityContext { + principal: temper_authz::Principal { + id: principal_id.to_string(), + kind: temper_authz::PrincipalKind::Admin, + role: None, + acting_for: None, + agent_type: None, + attributes: HashMap::new(), + }, + context_attrs: HashMap::new(), + correlation_id: "test-admin-context".to_string(), + } +} + #[test] fn test_pm_specs_parse() { let bundle = get_os_app("project-management").expect("PM app not found"); @@ -329,10 +344,7 @@ async fn test_reconcile_os_app_repairs_missing_active_policies_for_unchanged_bun .reload_tenant_policies(tenant, "") .expect("empty tenant policy should load"); - let admin_ctx = SecurityContext::from_headers(&[ - ("X-Temper-Principal-Id".to_string(), "admin-1".to_string()), - ("X-Temper-Principal-Kind".to_string(), "admin".to_string()), - ]); + let admin_ctx = test_admin_security_context("admin-1"); let mut issue_attrs = HashMap::new(); issue_attrs.insert("id".to_string(), serde_json::json!("issue-1")); @@ -418,10 +430,7 @@ async fn test_reconcile_os_app_repairs_missing_authz_engine_policies_despite_tex ); } - let admin_ctx = SecurityContext::from_headers(&[ - ("X-Temper-Principal-Id".to_string(), "admin-1".to_string()), - ("X-Temper-Principal-Kind".to_string(), "admin".to_string()), - ]); + let admin_ctx = test_admin_security_context("admin-1"); let mut issue_attrs = HashMap::new(); issue_attrs.insert("id".to_string(), serde_json::json!("issue-1")); @@ -1310,10 +1319,7 @@ async fn test_install_os_app_activates_tenant_cedar_policies() { .await .expect("install project-management"); - let admin_ctx = SecurityContext::from_headers(&[ - ("X-Temper-Principal-Id".to_string(), "admin-1".to_string()), - ("X-Temper-Principal-Kind".to_string(), "admin".to_string()), - ]); + let admin_ctx = test_admin_security_context("admin-1"); let mut issue_attrs = HashMap::new(); issue_attrs.insert("id".to_string(), serde_json::json!("issue-1")); diff --git a/crates/temper-platform/src/router.rs b/crates/temper-platform/src/router.rs index 74f175742..e17d9b405 100644 --- a/crates/temper-platform/src/router.rs +++ b/crates/temper-platform/src/router.rs @@ -37,10 +37,6 @@ pub fn build_platform_router(state: PlatformState) -> Router { .route( "/observe/os-apps/{name}", routing::get(crate::tenant_api::get_os_app_guide), - ) - .route( - "/observe/tenants/{id}", - routing::delete(crate::tenant_api::delete_tenant), ); // Identity resolution endpoint — used by MCP server at startup. @@ -54,28 +50,84 @@ pub fn build_platform_router(state: PlatformState) -> Router { .merge(identity_api.with_state(state.server.clone())) .merge(platform_observe.with_state(state.clone())) .nest("/api", tenant_api.with_state(state.clone())) - .layer(middleware::from_fn_with_state( - state.clone(), - crate::identity_cache::invalidate_identity_cache_on_credential_mutation, - )) .layer(middleware::from_fn_with_state( state.clone(), tenant_access_check, )) .layer(middleware::from_fn_with_state(state, bearer_auth_check)) + // Defense in depth: raw authority headers are removed before tenant + // credential resolution creates a typed request context (ADR-0157). + .layer(middleware::from_fn( + temper_server::authz::strip_inbound_identity_headers, + )) } #[cfg(test)] mod tests { use super::*; use axum::body::Body; - use axum::http::{Request, StatusCode}; + use axum::http::{Method, Request, StatusCode}; + use temper_authz::{AuthenticatedRequestContext, SecurityContext}; + use temper_runtime::tenant::TenantId; use tower::ServiceExt; + const ROUTER_TEST_PRINCIPAL: &str = "router-test-agent"; + fn test_state() -> PlatformState { PlatformState::new(None) } + fn authenticated_request( + state: &PlatformState, + method: Method, + uri: &str, + body: Body, + ) -> Request { + let credential = state + .server + .internal_invocation_credentials + .issue_for_url( + AuthenticatedRequestContext::new( + TenantId::default(), + SecurityContext::from_resolved_identity( + ROUTER_TEST_PRINCIPAL, + "operator", + None, + ), + ), + method.as_str(), + &format!("http://127.0.0.1:3000{uri}"), + ) + .expect("test credential should issue"); + + Request::builder() + .method(method) + .uri(uri) + .header("authorization", format!("Bearer {credential}")) + .header("x-tenant-id", "default") + .body(body) + .expect("test request should build") + } + + fn allow_app_catalog(state: &PlatformState) { + state + .server + .authz + .reload_tenant_policies( + "default", + &format!( + r#" +permit( + principal == Agent::"{ROUTER_TEST_PRINCIPAL}", + action == Action::"read_app_catalog", + resource == AppCatalog::"all" +); +"# + ), + ) + .expect("test catalog policy should parse"); + } + #[tokio::test] async fn test_tdata_routes_accessible() { let app = build_platform_router(test_state()); @@ -94,11 +146,10 @@ mod tests { #[tokio::test] async fn test_unknown_route_returns_404() { - let app = build_platform_router(test_state()); - let response = app - .oneshot(Request::get("/nonexistent").body(Body::empty()).unwrap()) - .await - .unwrap(); + let state = test_state(); + let request = authenticated_request(&state, Method::GET, "/nonexistent", Body::empty()); + let app = build_platform_router(state); + let response = app.oneshot(request).await.unwrap(); assert_eq!(response.status(), StatusCode::NOT_FOUND); } @@ -115,16 +166,30 @@ mod tests { } #[tokio::test] - async fn test_old_routes_gone() { + async fn identity_resolve_rejects_malformed_body_tenant_without_panicking() { let app = build_platform_router(test_state()); + let response = app + .oneshot( + Request::post("/api/identity/resolve") + .header("content-type", "application/json") + .body(Body::from(r#"{"bearer_token":"x","tenant":":"}"#)) + .unwrap(), + ) + .await + .expect("malformed tenant must produce an HTTP response"); + + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + } + + #[tokio::test] + async fn test_old_routes_gone() { + let state = test_state(); + let app = build_platform_router(state.clone()); // /dev, /prod, and /odata should not exist for path in &["/dev", "/prod", "/odata"] { - let response = app - .clone() - .oneshot(Request::get(*path).body(Body::empty()).unwrap()) - .await - .unwrap(); + let request = authenticated_request(&state, Method::GET, path, Body::empty()); + let response = app.clone().oneshot(request).await.unwrap(); assert_eq!( response.status(), StatusCode::NOT_FOUND, @@ -137,11 +202,11 @@ mod tests { #[tokio::test] async fn test_get_os_apps_returns_200() { - let app = build_platform_router(test_state()); - let response = app - .oneshot(Request::get("/api/os-apps").body(Body::empty()).unwrap()) - .await - .unwrap(); + let state = test_state(); + allow_app_catalog(&state); + let request = authenticated_request(&state, Method::GET, "/api/os-apps", Body::empty()); + let app = build_platform_router(state); + let response = app.oneshot(request).await.unwrap(); assert_eq!(response.status(), StatusCode::OK); @@ -161,31 +226,26 @@ mod tests { #[tokio::test] async fn test_local_os_app_install_route_is_removed() { - let app = build_platform_router(test_state()); - let response = app - .oneshot( - Request::post("/api/os-apps/project-management/install") - .header("content-type", "application/json") - .body(Body::from(r#"{"tenant":"test-install"}"#)) - .unwrap(), - ) - .await - .unwrap(); + let state = test_state(); + let request = authenticated_request( + &state, + Method::POST, + "/api/os-apps/project-management/install", + Body::from(r#"{"tenant":"test-install"}"#), + ); + let app = build_platform_router(state); + let response = app.oneshot(request).await.unwrap(); assert_eq!(response.status(), StatusCode::NOT_FOUND); } #[tokio::test] async fn test_get_observe_os_apps_returns_200() { - let app = build_platform_router(test_state()); - let response = app - .oneshot( - Request::get("/observe/os-apps") - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); + let state = test_state(); + allow_app_catalog(&state); + let request = authenticated_request(&state, Method::GET, "/observe/os-apps", Body::empty()); + let app = build_platform_router(state); + let response = app.oneshot(request).await.unwrap(); assert_eq!(response.status(), StatusCode::OK); @@ -205,16 +265,15 @@ mod tests { #[tokio::test] async fn test_observe_local_os_app_install_route_is_removed() { - let app = build_platform_router(test_state()); - let response = app - .oneshot( - Request::post("/observe/os-apps/project-management/install") - .header("content-type", "application/json") - .body(Body::from(r#"{"tenant":"test"}"#)) - .unwrap(), - ) - .await - .unwrap(); + let state = test_state(); + let request = authenticated_request( + &state, + Method::POST, + "/observe/os-apps/project-management/install", + Body::from(r#"{"tenant":"test"}"#), + ); + let app = build_platform_router(state); + let response = app.oneshot(request).await.unwrap(); assert_eq!(response.status(), StatusCode::NOT_FOUND); } diff --git a/crates/temper-platform/src/state.rs b/crates/temper-platform/src/state.rs index f706f950c..cb71f4307 100644 --- a/crates/temper-platform/src/state.rs +++ b/crates/temper-platform/src/state.rs @@ -11,8 +11,6 @@ use temper_runtime::ActorSystem; use temper_server::ServerState; use temper_server::registry::SpecRegistry; -use temper_server::identity::IdentityResolver; - use crate::protocol::PlatformEvent; use crate::spec_store::SpecStore; @@ -31,12 +29,11 @@ pub struct PlatformState { pub broadcast_tx: broadcast::Sender, /// Anthropic API key for Claude-powered agents. pub api_key: Option, - /// Bearer token for API authentication (`TEMPER_API_KEY`). + /// Optional bootstrap token registered as a normal tenant credential. + /// It has no direct runtime authentication fallback (ADR-0157). pub api_token: Option, /// In-memory spec storage for pending tenant deployments. pub spec_store: Arc>, - /// Agent identity resolver — maps bearer tokens to verified identities. - pub identity_resolver: Arc, } /// Default broadcast channel capacity. @@ -46,19 +43,20 @@ const BROADCAST_CAPACITY: usize = 256; /// /// Platform entities (GovernanceDecision, etc.) live in the temper-system /// tenant but have no os-app to ship policies with. This baseline policy -/// permits admin principals to manage governance decisions — needed so that -/// WASM modules can register callbacks via HTTP (system principal is blocked -/// from HTTP headers as a privilege-escalation safeguard). +/// permits only a credential-verified, tenant-scoped operator Agent to manage +/// platform entities. The deployment key has no implicit Admin authority. const SYSTEM_TENANT_POLICY: &str = r#" // Baseline policy for the temper-system tenant. -// Admin principals get full access to all platform entities (GovernanceDecision, -// Project, etc.). This matches the global fallback behavior — without it, loading -// any tenant-specific policy would shadow the global set and deny unlisted actions. +// Operators are ordinary tenant credentials. Both the registered AgentType and +// the credential-verification attribute must match. permit( - principal is Admin, + principal is Agent, action, resource -); +) when { + principal.agent_type == "operator" && + principal.agentTypeVerified == true +}; "#; impl PlatformState { @@ -90,7 +88,6 @@ impl PlatformState { api_key, api_token: None, spec_store, - identity_resolver: Arc::new(IdentityResolver::new()), }; state.server.bound_action_hook = Some(Arc::new( crate::genesis_install::GenesisInstallHook::new(state.clone()), @@ -126,7 +123,6 @@ impl PlatformState { api_key, api_token: None, spec_store, - identity_resolver: Arc::new(IdentityResolver::new()), }; state.server.bound_action_hook = Some(Arc::new( crate::genesis_install::GenesisInstallHook::new(state.clone()), diff --git a/crates/temper-platform/src/tenant_access.rs b/crates/temper-platform/src/tenant_access.rs index 71af26b12..618b94759 100644 --- a/crates/temper-platform/src/tenant_access.rs +++ b/crates/temper-platform/src/tenant_access.rs @@ -1,87 +1,42 @@ -//! Tenant access validation middleware. -//! -//! Defense-in-depth layer that verifies `github:*` principals have access -//! to the requested tenant. Agent principals and requests without identity -//! headers pass through (backward compatibility for local dev / MCP). +//! Tenant access validation for authenticated requests. use axum::extract::{Request, State}; use axum::http::StatusCode; use axum::middleware::Next; use axum::response::Response; +use temper_authz::{AuthenticatedRequestContext, PrincipalKind}; use crate::state::PlatformState; -/// Extract the tenant ID from the request. -/// -/// Checks `X-Tenant-Id` header first, then falls back to URL path inspection. -fn extract_tenant(req: &Request) -> Option { - // Check header first. - if let Some(val) = req.headers().get("x-tenant-id") { - return val.to_str().ok().map(|s| s.to_string()); - } - - // Check URL path for /api/tenants/:id patterns. - let path = req.uri().path(); - if let Some(rest) = path.strip_prefix("/api/tenants/") { - let tenant_id = rest.split('/').next().unwrap_or(""); - if !tenant_id.is_empty() { - return Some(tenant_id.to_string()); - } - } - - None -} - -/// Axum middleware that validates tenant access for `github:*` principals. -/// -/// Passthrough rules (no access check): -/// - No `X-Temper-Principal-Id` header (local dev / backward compat) -/// - `X-Temper-Principal-Kind` is `agent` (trusted backend-to-backend) -/// - Principal doesn't start with `github:` (non-human principal) -/// - No tenant could be extracted from the request -/// - Tenant is `temper-system` (always accessible) -/// - Not in TenantRouted mode (single-DB has no per-tenant access control) +/// Validate that the credential-bound tenant is the tenant addressed by the +/// request, then apply routed-storage membership checks for GitHub users. pub async fn tenant_access_check( State(state): State, req: Request, next: Next, ) -> Result { - // No identity header → passthrough (local dev). - let Some(principal_id) = req - .headers() - .get("x-temper-principal-id") - .and_then(|v| v.to_str().ok()) - .map(|s| s.to_string()) + let Some(authenticated) = req + .extensions() + .get::() + .cloned() else { + // The outer bearer edge permits only exact public routes without a + // context. It remains authoritative for that classification. return Ok(next.run(req).await); }; - // Agent principals pass through (trusted backend-to-backend). - let kind = req - .headers() - .get("x-temper-principal-kind") - .and_then(|v| v.to_str().ok()) - .unwrap_or(""); - if kind == "agent" { - return Ok(next.run(req).await); - } - - // Only validate github:* principals. - if !principal_id.starts_with("github:") { - return Ok(next.run(req).await); + let credential_tenant = authenticated.tenant().as_str(); + if let Some(path_tenant) = tenant_from_path(req.uri().path()) + && path_tenant != credential_tenant + { + return Err(StatusCode::FORBIDDEN); } - // Extract tenant from request. - let Some(tenant_id) = extract_tenant(&req) else { - return Ok(next.run(req).await); - }; - - // Always-accessible tenants. - if tenant_id == "temper-system" { + let principal = &authenticated.security_context().principal; + if principal.kind == PrincipalKind::Agent || !principal.id.starts_with("github:") { return Ok(next.run(req).await); } - // Check tenant access via the routed Turso capability. let Some(provider) = state .server .storage_stack @@ -91,26 +46,43 @@ pub async fn tenant_access_check( return Ok(next.run(req).await); }; if !provider.supports_tenant_admin() { - // Not in routed mode — no per-tenant access control. return Ok(next.run(req).await); } - match provider.tenants_for_user(&principal_id).await { - Ok(user_tenants) => { - if user_tenants.iter().any(|t| t.tenant_id == tenant_id) { - Ok(next.run(req).await) - } else { - Err(StatusCode::FORBIDDEN) - } + match provider.tenants_for_user(&principal.id).await { + Ok(user_tenants) + if user_tenants + .iter() + .any(|tenant| tenant.tenant_id == credential_tenant) => + { + Ok(next.run(req).await) } - Err(e) => { + Ok(_) => Err(StatusCode::FORBIDDEN), + Err(error) => { tracing::error!( - principal = %principal_id, - tenant = %tenant_id, - error = %e, + principal = %principal.id, + tenant = credential_tenant, + error = %error, "failed to check tenant access" ); Err(StatusCode::SERVICE_UNAVAILABLE) } } } + +fn tenant_from_path(path: &str) -> Option<&str> { + let rest = path.strip_prefix("/api/tenants/")?; + rest.split('/').next().filter(|tenant| !tenant.is_empty()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn extracts_only_explicit_tenant_path_segments() { + assert_eq!(tenant_from_path("/api/tenants/acme/specs"), Some("acme")); + assert_eq!(tenant_from_path("/api/tenants/"), None); + assert_eq!(tenant_from_path("/tdata/Orders"), None); + } +} diff --git a/crates/temper-platform/src/tenant_api.rs b/crates/temper-platform/src/tenant_api.rs index c6418abd2..b9365b937 100644 --- a/crates/temper-platform/src/tenant_api.rs +++ b/crates/temper-platform/src/tenant_api.rs @@ -9,18 +9,29 @@ //! - `GET /api/tenants/:id/users` — list users for a tenant //! - `GET /api/genesis/apps/follow-updates` — list staged follow-latest rollout status -use axum::extract::{Path, State}; -use axum::http::HeaderMap; +use axum::extract::{Extension, State}; use axum::http::StatusCode; use axum::response::IntoResponse; use axum::{Json, Router, routing}; use serde::{Deserialize, Serialize}; use std::sync::Arc; +use temper_authz::AuthenticatedRequestContext; use temper_server::storage::TursoStoreProvider; use crate::state::PlatformState; +mod apps; +mod auth; +pub(crate) use apps::{ + get_genesis_app_bundle, get_os_app_guide, install_genesis_app, list_genesis_follow_updates, + list_os_apps, +}; +use auth::{ + PlatformResourceAuthorization, require_authenticated, require_control_plane, + require_resource_authorization, require_same_tenant, validate_tenant_id, +}; + /// Request body for `POST /api/tenants`. #[derive(Debug, Deserialize)] pub struct CreateTenantRequest { @@ -75,6 +86,19 @@ fn turso_provider(state: &PlatformState) -> Option> .and_then(|stack| stack.turso.clone()) } +fn authorization_error(status: StatusCode) -> (StatusCode, Json) { + ( + status, + Json(serde_json::json!({ + "error": if status == StatusCode::UNAUTHORIZED { + "authentication required" + } else { + "authorization denied" + } + })), + ) +} + /// Build the tenant management API router. pub fn tenant_api_router() -> Router { Router::new() @@ -104,8 +128,33 @@ pub fn tenant_api_router() -> Router { /// `POST /api/tenants` — provision a new tenant database. async fn create_tenant( State(state): State, + authenticated: Option>, Json(req): Json, ) -> impl IntoResponse { + let authenticated = match require_authenticated(authenticated.as_deref()) { + Ok(authenticated) => authenticated, + Err(status) => return authorization_error(status), + }; + if let Err(status) = validate_tenant_id(&req.tenant_id) + .and_then(|_| require_control_plane(authenticated)) + .and_then(|_| { + require_resource_authorization( + &state, + authenticated, + PlatformResourceAuthorization { + action: "create_tenant", + resource_type: "Tenant", + resource_id: &req.tenant_id, + attrs: std::collections::BTreeMap::from([( + "targetTenant".to_string(), + serde_json::Value::String(req.tenant_id.clone()), + )]), + }, + ) + }) + { + return authorization_error(status); + } let Some(provider) = turso_provider(&state) else { return ( StatusCode::SERVICE_UNAVAILABLE, @@ -146,7 +195,28 @@ async fn create_tenant( } /// `GET /api/tenants` — list all registered tenants. -async fn list_tenants(State(state): State) -> impl IntoResponse { +async fn list_tenants( + State(state): State, + authenticated: Option>, +) -> impl IntoResponse { + let authenticated = match require_authenticated(authenticated.as_deref()) { + Ok(authenticated) => authenticated, + Err(status) => return authorization_error(status), + }; + if let Err(status) = require_control_plane(authenticated).and_then(|_| { + require_resource_authorization( + &state, + authenticated, + PlatformResourceAuthorization { + action: "list_tenants", + resource_type: "TenantCatalog", + resource_id: "all", + attrs: std::collections::BTreeMap::new(), + }, + ) + }) { + return authorization_error(status); + } let Some(provider) = turso_provider(&state) else { return ( StatusCode::SERVICE_UNAVAILABLE, @@ -185,8 +255,27 @@ async fn list_tenants(State(state): State) -> impl IntoResponse { /// `DELETE /api/tenants/:id` — remove a tenant and its data. pub(crate) async fn delete_tenant( State(state): State, + authenticated: Option>, axum::extract::Path(tenant_id): axum::extract::Path, ) -> impl IntoResponse { + let authenticated = match require_authenticated(authenticated.as_deref()) { + Ok(authenticated) => authenticated, + Err(status) => return authorization_error(status), + }; + if let Err(status) = require_same_tenant(authenticated, &tenant_id).and_then(|_| { + require_resource_authorization( + &state, + authenticated, + PlatformResourceAuthorization { + action: "delete_tenant", + resource_type: "Tenant", + resource_id: &tenant_id, + attrs: std::collections::BTreeMap::new(), + }, + ) + }) { + return authorization_error(status); + } let Some(provider) = turso_provider(&state) else { return ( StatusCode::SERVICE_UNAVAILABLE, @@ -232,9 +321,42 @@ pub(crate) async fn delete_tenant( /// `POST /api/tenants/:id/users` — add a user to a tenant. async fn add_user( State(state): State, + authenticated: Option>, axum::extract::Path(tenant_id): axum::extract::Path, Json(req): Json, ) -> impl IntoResponse { + let authenticated = match require_authenticated(authenticated.as_deref()) { + Ok(authenticated) => authenticated, + Err(status) => return authorization_error(status), + }; + let user_resource_id = format!("{tenant_id}/{}", req.user_id); + if let Err(status) = require_same_tenant(authenticated, &tenant_id).and_then(|_| { + require_resource_authorization( + &state, + authenticated, + PlatformResourceAuthorization { + action: "manage_tenant_users", + resource_type: "TenantUser", + resource_id: &user_resource_id, + attrs: std::collections::BTreeMap::from([ + ( + "targetTenant".to_string(), + serde_json::Value::String(tenant_id.clone()), + ), + ( + "userId".to_string(), + serde_json::Value::String(req.user_id.clone()), + ), + ( + "role".to_string(), + serde_json::Value::String(req.role.clone()), + ), + ]), + }, + ) + }) { + return authorization_error(status); + } let Some(provider) = turso_provider(&state) else { return ( StatusCode::SERVICE_UNAVAILABLE, @@ -271,8 +393,27 @@ async fn add_user( /// `GET /api/tenants/:id/users` — list users for a tenant. async fn list_users( State(state): State, + authenticated: Option>, axum::extract::Path(tenant_id): axum::extract::Path, ) -> impl IntoResponse { + let authenticated = match require_authenticated(authenticated.as_deref()) { + Ok(authenticated) => authenticated, + Err(status) => return authorization_error(status), + }; + if let Err(status) = require_same_tenant(authenticated, &tenant_id).and_then(|_| { + require_resource_authorization( + &state, + authenticated, + PlatformResourceAuthorization { + action: "read_tenant_users", + resource_type: "Tenant", + resource_id: &tenant_id, + attrs: std::collections::BTreeMap::new(), + }, + ) + }) { + return authorization_error(status); + } let Some(provider) = turso_provider(&state) else { return ( StatusCode::SERVICE_UNAVAILABLE, @@ -306,8 +447,37 @@ async fn list_users( /// `DELETE /api/tenants/:id/users/:user_id` — remove a user from a tenant. async fn remove_user( State(state): State, + authenticated: Option>, axum::extract::Path((tenant_id, user_id)): axum::extract::Path<(String, String)>, -) -> impl IntoResponse { +) -> StatusCode { + let authenticated = match require_authenticated(authenticated.as_deref()) { + Ok(authenticated) => authenticated, + Err(status) => return status, + }; + let resource_id = format!("{tenant_id}/{user_id}"); + if let Err(status) = require_same_tenant(authenticated, &tenant_id).and_then(|_| { + require_resource_authorization( + &state, + authenticated, + PlatformResourceAuthorization { + action: "manage_tenant_users", + resource_type: "TenantUser", + resource_id: &resource_id, + attrs: std::collections::BTreeMap::from([ + ( + "targetTenant".to_string(), + serde_json::Value::String(tenant_id.clone()), + ), + ( + "userId".to_string(), + serde_json::Value::String(user_id.clone()), + ), + ]), + }, + ) + }) { + return status; + } let Some(provider) = turso_provider(&state) else { return StatusCode::SERVICE_UNAVAILABLE; }; @@ -322,92 +492,5 @@ async fn remove_user( } } -// ── OS App Catalog Endpoints ────────────────────────────────────── - -/// `GET /api/os-apps` — list available OS apps. -pub(crate) async fn list_os_apps() -> impl IntoResponse { - let apps = crate::os_apps::list_os_apps(); - Json(serde_json::json!({ "apps": apps })) -} - -/// `GET /api/os-apps/:name` — get app guide markdown. -pub(crate) async fn get_os_app_guide( - axum::extract::Path(name): axum::extract::Path, -) -> impl IntoResponse { - match crate::os_apps::get_app_guide(&name) { - Some(guide) => ( - StatusCode::OK, - Json(serde_json::json!({ - "name": name, - "guide": guide, - })), - ), - None => ( - StatusCode::NOT_FOUND, - Json(serde_json::json!({ - "error": format!("No app guide found for '{name}'"), - })), - ), - } -} - -/// `GET /api/genesis/apps/follow-updates` — read staged follow-latest status. -pub(crate) async fn list_genesis_follow_updates( - State(state): State, -) -> impl IntoResponse { - let updates = crate::genesis_install::list_genesis_follow_latest_updates(&state).await; - ( - StatusCode::OK, - Json(serde_json::json!({ "value": updates })), - ) -} - -/// `POST /api/genesis/apps/install` — install a pinned Genesis app into this instance. -pub(crate) async fn install_genesis_app( - State(state): State, - Json(req): Json, -) -> impl IntoResponse { - match crate::genesis_install::install_genesis_app_from_registry(&state, req).await { - Ok(result) => (StatusCode::OK, Json(serde_json::json!(result))), - Err(error) if error.contains("not found") => ( - StatusCode::NOT_FOUND, - Json(serde_json::json!({ "error": error })), - ), - Err(error) => ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(serde_json::json!({ "error": error })), - ), - } -} - -/// `GET /api/genesis/apps/:owner/:name/versions/:hash/bundle` — export a pinned app closure. -pub(crate) async fn get_genesis_app_bundle( - State(state): State, - headers: HeaderMap, - Path((owner, name, hash)): Path<(String, String, String)>, -) -> impl IntoResponse { - let registry_tenant = headers - .get("x-tenant-id") - .and_then(|value| value.to_str().ok()) - .filter(|value| !value.trim().is_empty()) - .unwrap_or("default"); - match crate::genesis_install::export_genesis_registry_bundle( - &state, - registry_tenant, - &owner, - &name, - &hash, - ) - .await - { - Ok(bundle) => (StatusCode::OK, Json(serde_json::json!(bundle))), - Err(error) if error.contains("not found") => ( - StatusCode::NOT_FOUND, - Json(serde_json::json!({ "error": error })), - ), - Err(error) => ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(serde_json::json!({ "error": error })), - ), - } -} +#[cfg(test)] +mod tests; diff --git a/crates/temper-platform/src/tenant_api/apps.rs b/crates/temper-platform/src/tenant_api/apps.rs new file mode 100644 index 000000000..55c4569d2 --- /dev/null +++ b/crates/temper-platform/src/tenant_api/apps.rs @@ -0,0 +1,207 @@ +use axum::Json; +use axum::extract::{Extension, Path, State}; +use axum::http::StatusCode; +use axum::response::IntoResponse; +use temper_authz::AuthenticatedRequestContext; + +use super::auth::{ + PlatformResourceAuthorization, require_authenticated, require_resource_authorization, + require_same_tenant, +}; +use super::authorization_error; +use crate::state::PlatformState; + +/// List the credential tenant's authorized application catalog. +pub(crate) async fn list_os_apps( + State(state): State, + authenticated: Option>, +) -> impl IntoResponse { + let authenticated = match require_authenticated(authenticated.as_deref()) { + Ok(authenticated) => authenticated, + Err(status) => return authorization_error(status), + }; + if let Err(status) = require_resource_authorization( + &state, + authenticated, + PlatformResourceAuthorization { + action: "read_app_catalog", + resource_type: "AppCatalog", + resource_id: "all", + attrs: std::collections::BTreeMap::new(), + }, + ) { + return authorization_error(status); + } + let apps = crate::os_apps::list_os_apps(); + (StatusCode::OK, Json(serde_json::json!({ "apps": apps }))) +} + +/// Return one authorized application guide. +pub(crate) async fn get_os_app_guide( + State(state): State, + authenticated: Option>, + Path(name): Path, +) -> impl IntoResponse { + let authenticated = match require_authenticated(authenticated.as_deref()) { + Ok(authenticated) => authenticated, + Err(status) => return authorization_error(status), + }; + if let Err(status) = require_resource_authorization( + &state, + authenticated, + PlatformResourceAuthorization { + action: "read_app_catalog", + resource_type: "AppCatalogEntry", + resource_id: &name, + attrs: std::collections::BTreeMap::new(), + }, + ) { + return authorization_error(status); + } + match crate::os_apps::get_app_guide(&name) { + Some(guide) => ( + StatusCode::OK, + Json(serde_json::json!({"name": name, "guide": guide})), + ), + None => ( + StatusCode::NOT_FOUND, + Json(serde_json::json!({ + "error": format!("No app guide found for '{name}'"), + })), + ), + } +} + +/// Return follow-latest state only for the credential-bound tenant. +pub(crate) async fn list_genesis_follow_updates( + State(state): State, + authenticated: Option>, +) -> impl IntoResponse { + let authenticated = match require_authenticated(authenticated.as_deref()) { + Ok(authenticated) => authenticated, + Err(status) => return authorization_error(status), + }; + let tenant = authenticated.tenant().as_str(); + if let Err(status) = require_resource_authorization( + &state, + authenticated, + PlatformResourceAuthorization { + action: "read_app_installs", + resource_type: "AppInstall", + resource_id: tenant, + attrs: std::collections::BTreeMap::new(), + }, + ) { + return authorization_error(status); + } + let updates = crate::genesis_install::list_genesis_follow_latest_updates(&state) + .await + .into_iter() + .filter(|update| update.tenant == tenant) + .collect::>(); + ( + StatusCode::OK, + Json(serde_json::json!({ "value": updates })), + ) +} + +/// Install one pinned application into the credential-bound tenant. +pub(crate) async fn install_genesis_app( + State(state): State, + authenticated: Option>, + Json(req): Json, +) -> impl IntoResponse { + let authenticated = match require_authenticated(authenticated.as_deref()) { + Ok(authenticated) => authenticated, + Err(status) => return authorization_error(status), + }; + if let Err(status) = require_same_tenant(authenticated, &req.tenant).and_then(|_| { + require_resource_authorization( + &state, + authenticated, + PlatformResourceAuthorization { + action: "install_app", + resource_type: "App", + resource_id: &req.app_ref, + attrs: std::collections::BTreeMap::from([ + ( + "targetTenant".to_string(), + serde_json::Value::String(req.tenant.clone()), + ), + ( + "registryTenant".to_string(), + serde_json::Value::String(req.registry_tenant.clone()), + ), + ]), + }, + ) + }) { + return authorization_error(status); + } + match crate::genesis_install::install_genesis_app_from_registry(&state, req).await { + Ok(result) => (StatusCode::OK, Json(serde_json::json!(result))), + Err(error) if error.contains("not found") => ( + StatusCode::NOT_FOUND, + Json(serde_json::json!({ "error": error })), + ), + Err(error) => ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(serde_json::json!({ "error": error })), + ), + } +} + +/// Export one pinned application bundle from the credential-bound registry tenant. +pub(crate) async fn get_genesis_app_bundle( + State(state): State, + authenticated: Option>, + Path((owner, name, hash)): Path<(String, String, String)>, +) -> impl IntoResponse { + let authenticated = match require_authenticated(authenticated.as_deref()) { + Ok(authenticated) => authenticated, + Err(status) => return authorization_error(status), + }; + let registry_tenant = authenticated.tenant().as_str(); + let resource_id = format!("{owner}/{name}@{hash}"); + if let Err(status) = require_resource_authorization( + &state, + authenticated, + PlatformResourceAuthorization { + action: "read_app_bundle", + resource_type: "App", + resource_id: &resource_id, + attrs: std::collections::BTreeMap::from([ + ( + "owner".to_string(), + serde_json::Value::String(owner.clone()), + ), + ("name".to_string(), serde_json::Value::String(name.clone())), + ( + "versionHash".to_string(), + serde_json::Value::String(hash.clone()), + ), + ]), + }, + ) { + return authorization_error(status); + } + match crate::genesis_install::export_genesis_registry_bundle( + &state, + registry_tenant, + &owner, + &name, + &hash, + ) + .await + { + Ok(bundle) => (StatusCode::OK, Json(serde_json::json!(bundle))), + Err(error) if error.contains("not found") => ( + StatusCode::NOT_FOUND, + Json(serde_json::json!({ "error": error })), + ), + Err(error) => ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(serde_json::json!({ "error": error })), + ), + } +} diff --git a/crates/temper-platform/src/tenant_api/auth.rs b/crates/temper-platform/src/tenant_api/auth.rs new file mode 100644 index 000000000..3acf5c2f9 --- /dev/null +++ b/crates/temper-platform/src/tenant_api/auth.rs @@ -0,0 +1,80 @@ +use std::collections::BTreeMap; + +use axum::http::StatusCode; +use temper_authz::AuthenticatedRequestContext; +use temper_runtime::tenant::TenantId; + +use crate::state::PlatformState; + +pub(super) fn require_authenticated( + authenticated: Option<&AuthenticatedRequestContext>, +) -> Result<&AuthenticatedRequestContext, StatusCode> { + authenticated.ok_or(StatusCode::UNAUTHORIZED) +} + +pub(super) fn validate_tenant_id(tenant: &str) -> Result { + TenantId::try_new(tenant).map_err(|_| StatusCode::BAD_REQUEST) +} + +pub(super) fn require_same_tenant( + authenticated: &AuthenticatedRequestContext, + target_tenant: &str, +) -> Result<(), StatusCode> { + validate_tenant_id(target_tenant)?; + if authenticated.tenant().as_str() != target_tenant { + return Err(StatusCode::FORBIDDEN); + } + Ok(()) +} + +pub(super) fn require_control_plane( + authenticated: &AuthenticatedRequestContext, +) -> Result<(), StatusCode> { + if authenticated.tenant().as_str() == "default" { + return Ok(()); + } + Err(StatusCode::FORBIDDEN) +} + +pub(super) struct PlatformResourceAuthorization<'a> { + pub action: &'a str, + pub resource_type: &'a str, + pub resource_id: &'a str, + pub attrs: BTreeMap, +} + +pub(super) fn require_resource_authorization( + state: &PlatformState, + authenticated: &AuthenticatedRequestContext, + mut input: PlatformResourceAuthorization<'_>, +) -> Result<(), StatusCode> { + input.attrs.insert( + "id".to_string(), + serde_json::Value::String(input.resource_id.to_string()), + ); + input.attrs.insert( + "credentialTenant".to_string(), + serde_json::Value::String(authenticated.tenant().to_string()), + ); + state + .server + .authorize_with_context( + authenticated.security_context(), + input.action, + input.resource_type, + &input.attrs, + authenticated.tenant().as_str(), + ) + .map_err(|denial| { + tracing::warn!( + reason = %denial, + tenant = %authenticated.tenant(), + principal_id = %authenticated.security_context().principal.id, + action = input.action, + resource_type = input.resource_type, + resource_id = input.resource_id, + "platform management operation denied" + ); + StatusCode::FORBIDDEN + }) +} diff --git a/crates/temper-platform/src/tenant_api/tests.rs b/crates/temper-platform/src/tenant_api/tests.rs new file mode 100644 index 000000000..459085449 --- /dev/null +++ b/crates/temper-platform/src/tenant_api/tests.rs @@ -0,0 +1,219 @@ +use std::collections::BTreeMap; + +use axum::body::Body; +use axum::http::{Method, Request, StatusCode}; +use temper_authz::{AuthenticatedRequestContext, SecurityContext}; +use temper_runtime::tenant::TenantId; +use tower::ServiceExt; + +use super::tenant_api_router; +use crate::state::PlatformState; + +fn agent_context(tenant: &str) -> AuthenticatedRequestContext { + AuthenticatedRequestContext::new( + TenantId::new(tenant), + SecurityContext::from_resolved_identity("agent-1", "operator", None), + ) +} + +fn typed_request( + method: Method, + uri: &str, + body: serde_json::Value, + tenant: &str, +) -> Request { + let mut request = Request::builder() + .method(method) + .uri(uri) + .header("content-type", "application/json") + .body(Body::from(body.to_string())) + .expect("request should build"); + request.extensions_mut().insert(agent_context(tenant)); + request +} + +#[tokio::test] +async fn tenant_admin_routes_require_typed_authority() { + let app = tenant_api_router().with_state(PlatformState::new(None)); + let response = app + .oneshot( + Request::post("/tenants") + .header("content-type", "application/json") + .body(Body::from(r#"{"tenant_id":"victim"}"#)) + .expect("request should build"), + ) + .await + .expect("request should run"); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); +} + +#[tokio::test] +async fn typed_admin_kind_does_not_bypass_platform_cedar() { + let app = tenant_api_router().with_state(PlatformState::new(None)); + let mut request = Request::get("/tenants") + .body(Body::empty()) + .expect("request should build"); + request + .extensions_mut() + .insert(AuthenticatedRequestContext::new( + TenantId::new("default"), + SecurityContext { + principal: temper_authz::Principal { + id: "claimed-admin".to_string(), + kind: temper_authz::PrincipalKind::Admin, + role: None, + acting_for: None, + agent_type: None, + attributes: Default::default(), + }, + context_attrs: Default::default(), + correlation_id: "platform-admin-side-channel-test".to_string(), + }, + )); + let response = app.oneshot(request).await.expect("request should run"); + assert_eq!(response.status(), StatusCode::FORBIDDEN); +} + +#[tokio::test] +async fn credential_cannot_delete_or_manage_users_in_another_tenant() { + let app = tenant_api_router().with_state(PlatformState::new(None)); + let delete = app + .clone() + .oneshot(typed_request( + Method::DELETE, + "/tenants/victim", + serde_json::Value::Null, + "attacker", + )) + .await + .expect("request should run"); + assert_eq!(delete.status(), StatusCode::FORBIDDEN); + + let add_user = app + .oneshot(typed_request( + Method::POST, + "/tenants/victim/users", + serde_json::json!({"user_id": "attacker", "role": "owner"}), + "attacker", + )) + .await + .expect("request should run"); + assert_eq!(add_user.status(), StatusCode::FORBIDDEN); +} + +#[tokio::test] +async fn genesis_install_body_cannot_select_another_tenant() { + let app = tenant_api_router().with_state(PlatformState::new(None)); + let response = app + .oneshot(typed_request( + Method::POST, + "/genesis/apps/install", + serde_json::json!({ + "tenant": "victim", + "app_ref": "owner/app@0123456789abcdef", + "registry_url": "https://example.invalid", + }), + "attacker", + )) + .await + .expect("request should run"); + assert_eq!(response.status(), StatusCode::FORBIDDEN); +} + +#[tokio::test] +async fn app_catalog_authorization_is_resource_specific() { + let state = PlatformState::new(None); + state + .server + .authz + .reload_tenant_policies( + "tenant-a", + r#" +permit( + principal == Agent::"agent-1", + action == Action::"read_app_catalog", + resource == AppCatalog::"all" +); +"#, + ) + .expect("catalog policy should parse"); + let app = tenant_api_router().with_state(state); + + let list = app + .clone() + .oneshot(typed_request( + Method::GET, + "/os-apps", + serde_json::Value::Null, + "tenant-a", + )) + .await + .expect("request should run"); + assert_eq!(list.status(), StatusCode::OK); + + let guide = app + .oneshot(typed_request( + Method::GET, + "/os-apps/project-management", + serde_json::Value::Null, + "tenant-a", + )) + .await + .expect("request should run"); + assert_eq!(guide.status(), StatusCode::FORBIDDEN); +} + +#[tokio::test] +async fn control_plane_catalog_rejects_non_default_credentials() { + let state = PlatformState::new(None); + state + .server + .authz + .reload_tenant_policies("attacker", "permit(principal, action, resource);") + .expect("attacker policy should parse"); + let app = tenant_api_router().with_state(state); + let response = app + .oneshot(typed_request( + Method::GET, + "/tenants", + serde_json::Value::Null, + "attacker", + )) + .await + .expect("request should run"); + assert_eq!(response.status(), StatusCode::FORBIDDEN); +} + +#[test] +fn exact_platform_resource_policy_does_not_cover_sibling_resource() { + let state = PlatformState::new(None); + state + .server + .authz + .reload_tenant_policies( + "tenant-a", + r#" +permit( + principal == Agent::"agent-1", + action == Action::"install_app", + resource == App::"owner/allowed@hash" +); +"#, + ) + .expect("install policy should parse"); + let authenticated = agent_context("tenant-a"); + let authorize = |resource_id| { + super::auth::require_resource_authorization( + &state, + &authenticated, + super::auth::PlatformResourceAuthorization { + action: "install_app", + resource_type: "App", + resource_id, + attrs: BTreeMap::new(), + }, + ) + }; + assert!(authorize("owner/allowed@hash").is_ok()); + assert!(authorize("owner/victim@hash").is_err()); +} diff --git a/crates/temper-platform/tests/common/platform.rs b/crates/temper-platform/tests/common/platform.rs index 463ca1331..9914f1202 100644 --- a/crates/temper-platform/tests/common/platform.rs +++ b/crates/temper-platform/tests/common/platform.rs @@ -1,16 +1,24 @@ use std::collections::BTreeMap; use axum::Router; -use temper_platform::bootstrap::bootstrap_system_tenant; +use temper_platform::bootstrap::{ + SYSTEM_TENANT, bootstrap_agent_specs, bootstrap_operator_credential, bootstrap_system_tenant, +}; use temper_platform::router::build_platform_router; use temper_platform::state::PlatformState; +/// Tenant-scoped credential for the verified test operator. +pub const OPERATOR_KEY: &str = "test-operator-key"; + pub fn bootstrapped_state() -> PlatformState { let state = PlatformState::new(None); bootstrap_system_tenant(&state, &BTreeMap::new()); state } -pub fn bootstrapped_router() -> Router { - build_platform_router(bootstrapped_state()) +pub async fn bootstrapped_router() -> Router { + let state = bootstrapped_state(); + bootstrap_agent_specs(&state, SYSTEM_TENANT, true, &BTreeMap::new()); + bootstrap_operator_credential(&state, OPERATOR_KEY, SYSTEM_TENANT).await; + build_platform_router(state) } diff --git a/crates/temper-platform/tests/compile_first_e2e.rs b/crates/temper-platform/tests/compile_first_e2e.rs index 615051244..6169d39f3 100644 --- a/crates/temper-platform/tests/compile_first_e2e.rs +++ b/crates/temper-platform/tests/compile_first_e2e.rs @@ -18,7 +18,7 @@ mod common; use std::collections::BTreeMap; use common::http::{body_json, body_string}; -use temper_platform::bootstrap::bootstrap_system_tenant; +use temper_platform::bootstrap::{bootstrap_operator_credential, bootstrap_system_tenant}; use temper_platform::router::build_platform_router; use temper_platform::state::PlatformState; use temper_runtime::tenant::TenantId; @@ -29,6 +29,55 @@ use temper_spec::csdl::parse_csdl; const CSDL_XML: &str = include_str!("../../../test-fixtures/specs/model.csdl.xml"); const ORDER_IOA: &str = include_str!("../../../test-fixtures/specs/order.ioa.toml"); +const AGENT_TYPE_IOA: &str = include_str!("../src/specs/agent_type.ioa.toml"); +const AGENT_CREDENTIAL_IOA: &str = include_str!("../src/specs/agent_credential.ioa.toml"); + +const CREDENTIAL_CSDL_XML: &str = r#" + + + + + + + + + + + + + + + + + + + + + + + + + + + +"#; + +const ORDER_OPERATOR_POLICY: &str = r#" +permit(principal == Agent::"operator", action == Action::"create", resource is Order); +permit(principal == Agent::"operator", action == Action::"read", resource is Order); +permit(principal == Agent::"operator", action == Action::"list", resource is Order); +permit(principal == Agent::"operator", action == Action::"CancelOrder", resource is Order); +"#; + +const TASK_OPERATOR_POLICY: &str = r#" +permit(principal == Agent::"operator", action == Action::"create", resource is Task); +permit(principal == Agent::"operator", action == Action::"StartWork", resource is Task); +"#; + +const PROJECT_OPERATOR_POLICY: &str = r#" +permit(principal == Agent::"operator", action == Action::"create", resource is Project); +permit(principal == Agent::"operator", action == Action::"UpdateSpecs", resource is Project); +"#; /// Minimal Task IOA spec for multi-tenant tests. const TASK_IOA: &str = r#" @@ -111,6 +160,59 @@ fn build_user_registry(tenant: &str, ioa_specs: &[(&str, &str)]) -> SpecRegistry registry } +async fn register_test_operator( + state: &PlatformState, + tenant: &str, + credential: &str, + cedar_policy: &str, +) { + let credential_csdl = parse_csdl(CREDENTIAL_CSDL_XML).expect("credential CSDL should parse"); + let credential_specs = [ + ("AgentType", AGENT_TYPE_IOA), + ("AgentCredential", AGENT_CREDENTIAL_IOA), + ]; + let tenant_id = TenantId::new(tenant); + { + let mut registry = state + .registry + .write() + .expect("test registry lock should be available"); + registry + .try_register_tenant_with_reactions_and_constraints( + tenant_id.clone(), + credential_csdl, + CREDENTIAL_CSDL_XML.to_string(), + &credential_specs, + Vec::new(), + None, + true, + ) + .expect("credential specs should merge into the test tenant"); + for (entity_type, _) in credential_specs { + registry.set_verification_status( + &tenant_id, + entity_type, + VerificationStatus::Completed(EntityVerificationResult { + all_passed: true, + levels: vec![EntityLevelSummary { + level: "E2E fixture".to_string(), + passed: true, + summary: "Credential fixture pre-verified".to_string(), + details: None, + }], + verified_at: "2026-07-10T00:00:00Z".to_string(), + }), + ); + } + } + bootstrap_operator_credential(state, credential, tenant).await; + state + .server + .authz + .reload_tenant_policies(tenant, cedar_policy) + .expect("test operator policy should parse"); +} + // ========================================================================= // Test 1: Full Order lifecycle through compile-first path // ========================================================================= @@ -122,6 +224,7 @@ async fn e2e_compile_first_order_lifecycle() { let registry = build_user_registry("alpha", &[("Order", ORDER_IOA)]); let state = PlatformState::with_registry(registry, None); bootstrap_system_tenant(&state, &BTreeMap::new()); + register_test_operator(&state, "alpha", "alpha-operator-key", ORDER_OPERATOR_POLICY).await; let app = build_platform_router(state); // POST /tdata/Orders → 201, creates entity in Draft @@ -130,7 +233,7 @@ async fn e2e_compile_first_order_lifecycle() { .oneshot( Request::post("/tdata/Orders") .header("Content-Type", "application/json") - .header("X-Temper-Principal-Kind", "admin") + .header("Authorization", "Bearer alpha-operator-key") .header("X-Tenant-Id", "alpha") .body(Body::from("{}")) .unwrap(), @@ -157,7 +260,7 @@ async fn e2e_compile_first_order_lifecycle() { "/tdata/Orders('{entity_id}')/Temper.Example.CancelOrder" )) .header("Content-Type", "application/json") - .header("X-Temper-Principal-Kind", "admin") + .header("Authorization", "Bearer alpha-operator-key") .header("X-Tenant-Id", "alpha") .body(Body::from(r#"{"Reason": "changed mind"}"#)) .unwrap(), @@ -173,6 +276,7 @@ async fn e2e_compile_first_order_lifecycle() { .clone() .oneshot( Request::get(format!("/tdata/Orders('{entity_id}')")) + .header("Authorization", "Bearer alpha-operator-key") .header("X-Tenant-Id", "alpha") .body(Body::empty()) .unwrap(), @@ -188,6 +292,7 @@ async fn e2e_compile_first_order_lifecycle() { .clone() .oneshot( Request::get("/tdata/$metadata") + .header("Authorization", "Bearer alpha-operator-key") .header("X-Tenant-Id", "alpha") .body(Body::empty()) .unwrap(), @@ -206,6 +311,7 @@ async fn e2e_compile_first_order_lifecycle() { .clone() .oneshot( Request::get("/tdata") + .header("Authorization", "Bearer alpha-operator-key") .header("X-Tenant-Id", "alpha") .body(Body::empty()) .unwrap(), @@ -270,6 +376,8 @@ async fn e2e_compile_first_two_tenants() { let state = PlatformState::with_registry(registry, None); bootstrap_system_tenant(&state, &BTreeMap::new()); + register_test_operator(&state, "alpha", "alpha-operator-key", ORDER_OPERATOR_POLICY).await; + register_test_operator(&state, "beta", "beta-operator-key", TASK_OPERATOR_POLICY).await; let app = build_platform_router(state); // POST /tdata/Orders with X-Tenant-Id: alpha → 201 @@ -278,7 +386,7 @@ async fn e2e_compile_first_two_tenants() { .oneshot( Request::post("/tdata/Orders") .header("Content-Type", "application/json") - .header("X-Temper-Principal-Kind", "admin") + .header("Authorization", "Bearer alpha-operator-key") .header("X-Tenant-Id", "alpha") .body(Body::from("{}")) .unwrap(), @@ -305,7 +413,7 @@ async fn e2e_compile_first_two_tenants() { "/tdata/Orders('{alpha_id}')/Temper.Example.CancelOrder" )) .header("Content-Type", "application/json") - .header("X-Temper-Principal-Kind", "admin") + .header("Authorization", "Bearer alpha-operator-key") .header("X-Tenant-Id", "alpha") .body(Body::from("{}")) .unwrap(), @@ -322,7 +430,7 @@ async fn e2e_compile_first_two_tenants() { .oneshot( Request::post("/tdata/Tasks") .header("Content-Type", "application/json") - .header("X-Temper-Principal-Kind", "admin") + .header("Authorization", "Bearer beta-operator-key") .header("X-Tenant-Id", "beta") .body(Body::from("{}")) .unwrap(), @@ -349,7 +457,7 @@ async fn e2e_compile_first_two_tenants() { "/tdata/Tasks('{beta_id}')/Temper.Example.StartWork" )) .header("Content-Type", "application/json") - .header("X-Temper-Principal-Kind", "admin") + .header("Authorization", "Bearer beta-operator-key") .header("X-Tenant-Id", "beta") .body(Body::from("{}")) .unwrap(), @@ -365,6 +473,7 @@ async fn e2e_compile_first_two_tenants() { .clone() .oneshot( Request::get("/tdata") + .header("Authorization", "Bearer alpha-operator-key") .header("X-Tenant-Id", "alpha") .body(Body::empty()) .unwrap(), @@ -384,6 +493,7 @@ async fn e2e_compile_first_two_tenants() { .clone() .oneshot( Request::get("/tdata") + .header("Authorization", "Bearer beta-operator-key") .header("X-Tenant-Id", "beta") .body(Body::empty()) .unwrap(), @@ -411,6 +521,14 @@ async fn e2e_compile_first_system_and_user_coexist() { let registry = build_user_registry("alpha", &[("Order", ORDER_IOA)]); let state = PlatformState::with_registry(registry, None); bootstrap_system_tenant(&state, &BTreeMap::new()); + register_test_operator(&state, "alpha", "alpha-operator-key", ORDER_OPERATOR_POLICY).await; + register_test_operator( + &state, + "temper-system", + "system-operator-key", + PROJECT_OPERATOR_POLICY, + ) + .await; let app = build_platform_router(state); // GET /tdata/$metadata with X-Tenant-Id: alpha → sees user entities (Order) @@ -418,6 +536,7 @@ async fn e2e_compile_first_system_and_user_coexist() { .clone() .oneshot( Request::get("/tdata/$metadata") + .header("Authorization", "Bearer alpha-operator-key") .header("X-Tenant-Id", "alpha") .body(Body::empty()) .unwrap(), @@ -436,6 +555,7 @@ async fn e2e_compile_first_system_and_user_coexist() { .clone() .oneshot( Request::get("/tdata/$metadata") + .header("Authorization", "Bearer system-operator-key") .header("X-Tenant-Id", "temper-system") .body(Body::empty()) .unwrap(), @@ -457,7 +577,7 @@ async fn e2e_compile_first_system_and_user_coexist() { .oneshot( Request::post("/tdata/Orders") .header("Content-Type", "application/json") - .header("X-Temper-Principal-Kind", "admin") + .header("Authorization", "Bearer alpha-operator-key") .header("X-Tenant-Id", "alpha") .body(Body::from("{}")) .unwrap(), @@ -482,7 +602,7 @@ async fn e2e_compile_first_system_and_user_coexist() { "/tdata/Orders('{order_id}')/Temper.Example.CancelOrder" )) .header("Content-Type", "application/json") - .header("X-Temper-Principal-Kind", "admin") + .header("Authorization", "Bearer alpha-operator-key") .header("X-Tenant-Id", "alpha") .body(Body::from("{}")) .unwrap(), @@ -499,7 +619,7 @@ async fn e2e_compile_first_system_and_user_coexist() { .oneshot( Request::post("/tdata/Projects") .header("Content-Type", "application/json") - .header("X-Temper-Principal-Kind", "admin") + .header("Authorization", "Bearer system-operator-key") .header("X-Tenant-Id", "temper-system") .body(Body::from("{}")) .unwrap(), @@ -525,7 +645,7 @@ async fn e2e_compile_first_system_and_user_coexist() { "/tdata/Projects('{proj_id}')/Temper.System.UpdateSpecs" )) .header("Content-Type", "application/json") - .header("X-Temper-Principal-Kind", "admin") + .header("Authorization", "Bearer system-operator-key") .header("X-Tenant-Id", "temper-system") .body(Body::from("{}")) .unwrap(), diff --git a/crates/temper-platform/tests/identity_e2e.rs b/crates/temper-platform/tests/identity_e2e.rs index c3f90627b..383d56bd5 100644 --- a/crates/temper-platform/tests/identity_e2e.rs +++ b/crates/temper-platform/tests/identity_e2e.rs @@ -16,8 +16,10 @@ use std::collections::BTreeMap; use temper_platform::bootstrap::{bootstrap_agent_specs, bootstrap_system_tenant}; use temper_platform::state::PlatformState; use temper_runtime::tenant::TenantId; +use temper_server::StorageStack; use temper_server::identity::{IdentityResolver, hash_token}; use temper_server::request_context::AgentContext; +use temper_store_turso::TursoEventStore; use tower::ServiceExt; mod common; @@ -25,6 +27,45 @@ mod common; use common::http::body_json; const TEST_TENANT: &str = "identity-test"; +const TEST_OPERATOR_KEY: &str = "registered-operator-test-key"; + +const HTTP_IDENTITY_POLICY: &str = r#" +permit( + principal == Agent::"operator", + action == Action::"create", + resource is AgentType +); +permit( + principal == Agent::"operator", + action == Action::"Define", + resource is AgentType +); +permit( + principal == Agent::"operator", + action == Action::"create", + resource is AgentCredential +); +permit( + principal == Agent::"operator", + action == Action::"Issue", + resource is AgentCredential +); +permit( + principal == Agent::"operator", + action == Action::"Rotate", + resource is AgentCredential +); +permit( + principal == Agent::"operator", + action == Action::"delete", + resource is AgentCredential +); +permit( + principal == Agent::"http-inst-1", + action == Action::"list", + resource is AgentType +); +"#; /// Build a `PlatformState` with both system and agent specs bootstrapped /// on a dedicated test tenant. @@ -57,6 +98,52 @@ async fn dispatch( .unwrap_or_else(|e| panic!("dispatch {entity_type}.{action} failed: {e}")) } +async fn define_type_and_issue_credential( + state: &PlatformState, + agent_type_id: &str, + agent_type_name: &str, + plaintext: &str, + agent_instance_id: &str, +) -> String { + let response = dispatch( + state, + "AgentType", + agent_type_id, + "Define", + serde_json::json!({ + "name": agent_type_name, + "system_prompt": "test", + "tool_set": "local", + "model": "claude-sonnet-4-6", + "max_turns": "200", + "adapter_config": "{}", + "default_budget_cents": "0" + }), + ) + .await; + assert!(response.success, "Define: {:?}", response.error); + + let key_hash = hash_token(plaintext); + let response = dispatch( + state, + "AgentCredential", + &key_hash, + "Issue", + serde_json::json!({ + "agent_type_id": agent_type_id, + "agent_instance_id": agent_instance_id, + "key_hash": key_hash, + "key_prefix": "tmpr_test", + "description": "identity E2E credential", + "created_by": "test", + "expires_at": "" + }), + ) + .await; + assert!(response.success, "Issue: {:?}", response.error); + key_hash +} + // ========================================================================= // Dispatch-level identity tests // ========================================================================= @@ -312,11 +399,95 @@ async fn e2e_identity_resolution_rotated_credential() { .await; // Should no longer resolve (status is Rotated, not Active) - let resolver2 = IdentityResolver::new(); - let result = resolver2.resolve(&state.server, &tenant, plaintext).await; + let result = resolver.resolve(&state.server, &tenant, plaintext).await; assert!(result.is_none(), "rotated credential should not resolve"); } +/// Identity resolver: revocation takes effect for an already-used resolver. +#[tokio::test] +async fn e2e_identity_resolution_revocation_is_immediate() { + let state = identity_test_state(); + let tenant = TenantId::new(TEST_TENANT); + let plaintext = "tmpr_immediate-revocation-test"; + let key_hash = define_type_and_issue_credential( + &state, + "revoke-type", + "revocable-agent", + plaintext, + "revoke-inst", + ) + .await; + let resolver = IdentityResolver::new(); + + assert!( + resolver + .resolve(&state.server, &tenant, plaintext) + .await + .is_some() + ); + let response = dispatch( + &state, + "AgentCredential", + &key_hash, + "Revoke", + serde_json::json!({}), + ) + .await; + assert!(response.success, "Revoke: {:?}", response.error); + + assert!( + resolver + .resolve(&state.server, &tenant, plaintext) + .await + .is_none(), + "revocation must remove authority on the next resolution" + ); +} + +/// Identity resolution rejects registry rows whose stored hash no longer +/// matches the entity ID derived from the presented credential. +#[tokio::test] +async fn e2e_identity_resolution_rejects_mismatched_stored_hash() { + let state = identity_test_state(); + let tenant = TenantId::new(TEST_TENANT); + let plaintext = "tmpr_hash-binding-test"; + let key_hash = define_type_and_issue_credential( + &state, + "hash-binding-type", + "hash-binding-agent", + plaintext, + "hash-binding-inst", + ) + .await; + let resolver = IdentityResolver::new(); + assert!( + resolver + .resolve(&state.server, &tenant, plaintext) + .await + .is_some() + ); + + state + .server + .update_tenant_entity_fields( + &tenant, + "AgentCredential", + &key_hash, + serde_json::json!({"key_hash": "different-hash"}), + false, + ) + .await + .expect("generic test mutation should succeed"); + + assert!( + resolver + .resolve(&state.server, &tenant, plaintext) + .await + .is_none(), + "mismatched durable lookup ID and stored key hash must fail closed" + ); +} + /// Identity resolver: deprecated AgentType → None. #[tokio::test] async fn e2e_identity_resolution_deprecated_agent_type() { @@ -381,8 +552,7 @@ async fn e2e_identity_resolution_deprecated_agent_type() { .await; // Should no longer resolve (AgentType status is Deprecated, not Active) - let resolver2 = IdentityResolver::new(); - let result = resolver2.resolve(&state.server, &tenant, plaintext).await; + let result = resolver.resolve(&state.server, &tenant, plaintext).await; assert!( result.is_none(), "credential linked to deprecated AgentType should not resolve" @@ -393,17 +563,26 @@ async fn e2e_identity_resolution_deprecated_agent_type() { // HTTP-level identity tests // ========================================================================= -/// Build a router with agent specs and an API key configured for bearer auth. -fn identity_test_router() -> axum::Router { - let mut state = identity_test_state(); - state.api_token = Some("admin-test-key".to_string()); - temper_platform::router::build_platform_router(state) +/// Build state with an ordinary tenant operator and exact HTTP test policy. +async fn identity_http_state() -> PlatformState { + let state = identity_test_state(); + temper_platform::bootstrap_operator_credential(&state, TEST_OPERATOR_KEY, TEST_TENANT).await; + state + .server + .authz + .reload_tenant_policies(TEST_TENANT, HTTP_IDENTITY_POLICY) + .expect("HTTP identity test policy should parse"); + state +} + +async fn identity_test_router() -> axum::Router { + temper_platform::router::build_platform_router(identity_http_state().await) } /// Bearer auth: `/api/identity/resolve` is accessible without Authorization header. #[tokio::test] async fn e2e_http_identity_resolve_exempt_from_auth() { - let app = identity_test_router(); + let app = identity_test_router().await; // POST /api/identity/resolve without any Authorization header — should NOT 401 let response = app @@ -428,16 +607,15 @@ async fn e2e_http_identity_resolve_exempt_from_auth() { /// Bearer auth: valid agent credential resolves identity on HTTP requests. #[tokio::test] async fn e2e_http_agent_credential_auth() { - let app = identity_test_router(); + let app = identity_test_router().await; - // 1. Create AgentType (as admin) + // 1. Create AgentType as a registered, tenant-scoped operator. let response = app .clone() .oneshot( Request::post("/tdata/AgentTypes") .header("Content-Type", "application/json") - .header("Authorization", "Bearer admin-test-key") - .header("X-Temper-Principal-Kind", "admin") + .header("Authorization", format!("Bearer {TEST_OPERATOR_KEY}")) .header("X-Tenant-Id", TEST_TENANT) .body(Body::from(r#"{"id": "http-cc-type"}"#)) .unwrap(), @@ -452,8 +630,7 @@ async fn e2e_http_agent_credential_auth() { .oneshot( Request::post("/tdata/AgentTypes('http-cc-type')/Temper.Agent.Define") .header("Content-Type", "application/json") - .header("Authorization", "Bearer admin-test-key") - .header("X-Temper-Principal-Kind", "admin") + .header("Authorization", format!("Bearer {TEST_OPERATOR_KEY}")) .header("X-Tenant-Id", TEST_TENANT) .body(Body::from( r#"{"name": "claude-code", "system_prompt": "test", "tool_set": "local", "model": "claude-sonnet-4-6", "max_turns": "200", "adapter_config": "{}", "default_budget_cents": "0"}"#, @@ -475,8 +652,7 @@ async fn e2e_http_agent_credential_auth() { .oneshot( Request::post("/tdata/AgentCredentials") .header("Content-Type", "application/json") - .header("Authorization", "Bearer admin-test-key") - .header("X-Temper-Principal-Kind", "admin") + .header("Authorization", format!("Bearer {TEST_OPERATOR_KEY}")) .header("X-Tenant-Id", TEST_TENANT) .body(Body::from(format!(r#"{{"id": "{key_hash}"}}"#))) .unwrap(), @@ -493,8 +669,7 @@ async fn e2e_http_agent_credential_auth() { "/tdata/AgentCredentials('{key_hash}')/Temper.Agent.Issue" )) .header("Content-Type", "application/json") - .header("Authorization", "Bearer admin-test-key") - .header("X-Temper-Principal-Kind", "admin") + .header("Authorization", format!("Bearer {TEST_OPERATOR_KEY}")) .header("X-Tenant-Id", TEST_TENANT) .body(Body::from(format!( r#"{{"agent_type_id": "http-cc-type", "agent_instance_id": "http-inst-1", "key_hash": "{key_hash}", "key_prefix": "tmpr_http", "description": "HTTP auth test", "created_by": "test", "expires_at": ""}}"# @@ -547,7 +722,7 @@ async fn e2e_http_agent_credential_auth() { /// Bearer auth: no token → 401, wrong token → 401. #[tokio::test] async fn e2e_http_missing_and_wrong_token_rejected() { - let app = identity_test_router(); + let app = identity_test_router().await; // No auth header let response = app @@ -577,74 +752,45 @@ async fn e2e_http_missing_and_wrong_token_rejected() { assert_eq!(response.status(), StatusCode::UNAUTHORIZED); } -/// Bearer auth: global API key passes as admin. +/// Bearer auth: configuring a deployment key does not create runtime Admin authority. #[tokio::test] -async fn e2e_http_global_api_key_admin_access() { - let app = identity_test_router(); +async fn e2e_http_unregistered_deployment_key_has_no_admin_fallback() { + let mut state = identity_test_state(); + state.api_token = Some("unregistered-deployment-key".to_string()); + let app = temper_platform::router::build_platform_router(state); let response = app .oneshot( Request::get("/tdata/AgentTypes") - .header("Authorization", "Bearer admin-test-key") - .header("X-Temper-Principal-Kind", "admin") + .header("Authorization", "Bearer unregistered-deployment-key") .header("X-Tenant-Id", TEST_TENANT) .body(Body::empty()) .unwrap(), ) .await .unwrap(); - assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); } -/// Cache coherence: rotating a credential invalidates resolver cache immediately. +/// A generic OData deletion removes credential authority immediately. #[tokio::test] -async fn e2e_http_rotate_invalidates_identity_cache() { - let mut state = identity_test_state(); - state.api_token = Some("admin-test-key".to_string()); - let resolver = state.identity_resolver.clone(); +async fn e2e_http_generic_delete_removes_credential_authority() { + let state = identity_http_state().await; + let resolver = IdentityResolver::new(); let app = temper_platform::router::build_platform_router(state.clone()); let tenant = TenantId::new(TEST_TENANT); - // Create AgentType + credential. - let r = dispatch( + let plaintext = "tmpr_generic-delete-test"; + let key_hash = define_type_and_issue_credential( &state, - "AgentType", - "cache-type", - "Define", - serde_json::json!({ - "name": "cache-agent", - "system_prompt": "test", - "tool_set": "local", - "model": "claude-sonnet-4-6", - "max_turns": "200", - "adapter_config": "{}", - "default_budget_cents": "0" - }), + "delete-type", + "deletable-agent", + plaintext, + "delete-inst", ) .await; - assert!(r.success); - let plaintext = "tmpr_cache-invalidate-test"; - let key_hash = hash_token(plaintext); - let r = dispatch( - &state, - "AgentCredential", - &key_hash, - "Issue", - serde_json::json!({ - "agent_type_id": "cache-type", - "agent_instance_id": "cache-inst-1", - "key_hash": key_hash, - "key_prefix": "tmpr_cach", - "description": "cache test", - "created_by": "test", - "expires_at": "" - }), - ) - .await; - assert!(r.success); - - // Populate cache with a successful resolution. + // Establish that this resolver has already returned positive authority. assert!( resolver .resolve(&state.server, &tenant, plaintext) @@ -652,32 +798,85 @@ async fn e2e_http_rotate_invalidates_identity_cache() { .is_some() ); - // Rotate through HTTP route (this should trigger middleware invalidation). + // Use the generic entity mutation path, not an AgentCredential action. let response = app .oneshot( - Request::post(format!( - "/tdata/AgentCredentials('{key_hash}')/Temper.Agent.Rotate" - )) - .header("Content-Type", "application/json") - .header("Authorization", "Bearer admin-test-key") - .header("X-Temper-Principal-Kind", "admin") - .header("X-Tenant-Id", TEST_TENANT) - .body(Body::from( - r#"{"key_hash":"rotated-hash","key_prefix":"tmpr_rot","description":"rotated"}"#, - )) - .unwrap(), + Request::delete(format!("/tdata/AgentCredentials('{key_hash}')")) + .header("Authorization", format!("Bearer {TEST_OPERATOR_KEY}")) + .header("X-Tenant-Id", TEST_TENANT) + .body(Body::empty()) + .unwrap(), ) .await .unwrap(); - assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.status(), StatusCode::NO_CONTENT); - // Resolver should not return stale cached identity after rotate. assert!( resolver .resolve(&state.server, &tenant, plaintext) .await .is_none(), - "rotated credential must be invalidated in resolver cache immediately" + "generic deletion must remove authority on the next resolution" + ); +} + +/// A mutation on one replica is authoritative for identity checks on another. +#[tokio::test] +async fn e2e_replica_revocation_is_read_from_shared_durable_state() { + let directory = tempfile::tempdir().expect("create identity replica test directory"); + let database_url = format!("file:{}", directory.path().join("identity.db").display()); + let store = TursoEventStore::new(&database_url, None) + .await + .expect("create shared identity store"); + + let mut first = identity_test_state(); + let registry = first + .registry + .read() + .expect("identity registry lock should be healthy") + .clone(); + let mut second = PlatformState::with_registry(registry, None); + first + .server + .set_storage_stack(StorageStack::from_turso(store.clone())); + second + .server + .set_storage_stack(StorageStack::from_turso(store)); + + let plaintext = "tmpr_cross-replica-revocation"; + let key_hash = define_type_and_issue_credential( + &first, + "replica-type", + "replica-agent", + plaintext, + "replica-inst", + ) + .await; + let tenant = TenantId::new(TEST_TENANT); + let resolver = IdentityResolver::new(); + assert!( + resolver + .resolve(&first.server, &tenant, plaintext) + .await + .is_some() + ); + + let response = dispatch( + &second, + "AgentCredential", + &key_hash, + "Revoke", + serde_json::json!({}), + ) + .await; + assert!(response.success, "replica Revoke: {:?}", response.error); + + assert!( + resolver + .resolve(&first.server, &tenant, plaintext) + .await + .is_none(), + "the first replica must observe revocation from shared durable state" ); } @@ -729,10 +928,9 @@ async fn e2e_bootstrap_operator_credential_idempotent() { assert!(identity.verified); } -/// Resolver cache is tenant-scoped — a credential resolved in one tenant must -/// not leak into another tenant that has no matching AgentCredential entity. +/// Identity resolution is tenant-scoped. #[tokio::test] -async fn e2e_identity_resolution_cache_is_tenant_scoped() { +async fn e2e_identity_resolution_is_tenant_scoped() { let state = identity_test_state(); let api_key = "tmpr_tenant_scoped_cache_test"; @@ -750,7 +948,7 @@ async fn e2e_identity_resolution_cache_is_tenant_scoped() { .await; assert!( leaked.is_none(), - "resolver cache must not reuse identities across tenants" + "identity authority must not cross tenant boundaries" ); } diff --git a/crates/temper-platform/tests/identity_expiry_e2e.rs b/crates/temper-platform/tests/identity_expiry_e2e.rs new file mode 100644 index 000000000..f187be8d6 --- /dev/null +++ b/crates/temper-platform/tests/identity_expiry_e2e.rs @@ -0,0 +1,102 @@ +//! Credential-expiry coverage for the Class A authentication edge. + +use std::collections::BTreeMap; + +use temper_platform::bootstrap::{bootstrap_agent_specs, bootstrap_system_tenant}; +use temper_platform::state::PlatformState; +use temper_runtime::tenant::TenantId; +use temper_server::identity::{IdentityResolver, hash_token}; +use temper_server::request_context::AgentContext; + +const TENANT: &str = "identity-expiry-test"; + +async fn dispatch( + state: &PlatformState, + entity_type: &str, + entity_id: &str, + action: &str, + params: serde_json::Value, +) { + let response = state + .server + .dispatch_tenant_action( + &TenantId::new(TENANT), + entity_type, + entity_id, + action, + params, + &AgentContext::system(), + ) + .await + .unwrap_or_else(|error| panic!("dispatch {entity_type}.{action} failed: {error}")); + assert!(response.success, "dispatch failed: {:?}", response.error); +} + +async fn issue_credential(state: &PlatformState, token: &str, expires_at: &str) { + let key_hash = hash_token(token); + dispatch( + state, + "AgentCredential", + &key_hash, + "Issue", + serde_json::json!({ + "agent_type_id": "expiry-worker-type", + "agent_instance_id": format!("instance-{token}"), + "key_hash": key_hash, + "key_prefix": "expiry", + "description": "expiry boundary test", + "created_by": "test", + "expires_at": expires_at, + }), + ) + .await; +} + +#[tokio::test] +async fn resolver_denies_expired_and_malformed_credentials_but_accepts_future_expiry() { + let state = PlatformState::new(None); + bootstrap_system_tenant(&state, &BTreeMap::new()); + bootstrap_agent_specs(&state, TENANT, false, &BTreeMap::new()); + dispatch( + &state, + "AgentType", + "expiry-worker-type", + "Define", + serde_json::json!({ + "name": "expiry-worker", + "system_prompt": "test", + "tool_set": "local", + "model": "none", + "max_turns": "1", + "adapter_config": "{}", + "default_budget_cents": "0", + }), + ) + .await; + + issue_credential(&state, "expired-token", "2000-01-01T00:00:00Z").await; + issue_credential(&state, "malformed-token", "not-rfc3339").await; + issue_credential(&state, "future-token", "2999-01-01T00:00:00Z").await; + + let resolver = IdentityResolver::new(); + let tenant = TenantId::new(TENANT); + assert!( + resolver + .resolve(&state.server, &tenant, "expired-token") + .await + .is_none(), + "expired credential must not resolve" + ); + assert!( + resolver + .resolve(&state.server, &tenant, "malformed-token") + .await + .is_none(), + "malformed expiry must fail closed" + ); + let future = resolver + .resolve(&state.server, &tenant, "future-token") + .await + .expect("future-dated credential should resolve"); + assert_eq!(future.agent_type_name, "expiry-worker"); +} diff --git a/crates/temper-platform/tests/operator_bootstrap_cedar.rs b/crates/temper-platform/tests/operator_bootstrap_cedar.rs new file mode 100644 index 000000000..85ef4d3b8 --- /dev/null +++ b/crates/temper-platform/tests/operator_bootstrap_cedar.rs @@ -0,0 +1,479 @@ +//! ARN-389 / ADR-0172: operator bootstrap seeds `manage_policies`. +//! +//! Proves a virgin store's verified operator can manage Cedar, that +//! unverified / non-operator principals stay denied, that re-bootstrap is +//! idempotent, and that OS-app Cedar still loads. + +use std::collections::{BTreeMap, HashMap}; + +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use serde_json::json; +use temper_authz::{AuthzDecision, SecurityContext}; +use temper_platform::bootstrap::{bootstrap_agent_specs, bootstrap_operator_credential}; +use temper_platform::install_os_app; +use temper_platform::recovery::recover_cedar_policies; +use temper_platform::router::build_platform_router; +use temper_platform::state::PlatformState; +use temper_runtime::tenant::TenantId; +use temper_server::StorageStack; +use temper_server::identity::hash_token; +use temper_server::request_context::AgentContext; +use temper_server::state::PendingDecision; +use temper_store_turso::TursoEventStore; +use tower::ServiceExt; + +mod common; +use common::http::body_json; + +const OPERATOR_KEY: &str = "tmpr_operator-bootstrap-cedar"; +const DEVELOPER_KEY: &str = "tmpr_developer-bootstrap-cedar"; +const POLICY_ID: &str = "operator-bootstrap-manage-policies"; + +fn virgin_state(tenant: &str) -> PlatformState { + let state = PlatformState::new(None); + bootstrap_agent_specs(&state, tenant, false, &BTreeMap::new()); + state +} + +async fn virgin_state_with_store(tenant: &str) -> (PlatformState, tempfile::TempDir) { + let temp = tempfile::tempdir().expect("temp policy db"); + let db_url = format!("file:{}", temp.path().join("policy.db").display()); + let store = TursoEventStore::new(&db_url, None) + .await + .expect("create turso store"); + let mut state = virgin_state(tenant); + state + .server + .set_storage_stack(StorageStack::from_turso(store)); + (state, temp) +} + +fn manage_policies_attrs(tenant: &str) -> HashMap { + let mut attrs = HashMap::new(); + attrs.insert("id".to_string(), json!(tenant)); + attrs.insert("tenant".to_string(), json!(tenant)); + attrs +} + +fn authorize_manage_policies( + state: &PlatformState, + tenant: &str, + ctx: &SecurityContext, +) -> AuthzDecision { + state.server.authz.authorize_for_tenant( + tenant, + ctx, + "manage_policies", + "PolicySet", + &manage_policies_attrs(tenant), + ) +} + +fn unverified_operator_context() -> SecurityContext { + let mut ctx = SecurityContext::from_resolved_identity("operator", "operator", None); + ctx.principal + .attributes + .insert("agentTypeVerified".to_string(), json!(false)); + ctx.context_attrs + .insert("agentTypeVerified".to_string(), json!(false)); + ctx +} + +async fn issue_developer_credential(state: &PlatformState, tenant: &str, plaintext: &str) { + let tenant_id = TenantId::new(tenant); + let ctx = AgentContext::system(); + let key_hash = hash_token(plaintext); + let _ = state + .server + .dispatch_tenant_action( + &tenant_id, + "AgentType", + "developer-type", + "Define", + json!({ + "name": "developer", + "system_prompt": "test", + "tool_set": "local", + "model": "none", + "max_turns": "0", + "adapter_config": "{}", + "default_budget_cents": "0" + }), + &ctx, + ) + .await; + let _ = state + .server + .dispatch_tenant_action( + &tenant_id, + "AgentCredential", + &key_hash, + "Issue", + json!({ + "agent_type_id": "developer-type", + "agent_instance_id": "developer", + "key_hash": key_hash, + "key_prefix": plaintext.chars().take(8).collect::(), + "description": "non-operator test credential", + "created_by": "test", + "expires_at": "" + }), + &ctx, + ) + .await; +} + +fn statement_occurrences(haystack: &str, tenant: &str) -> usize { + let needle = format!(r#"resource == PolicySet::"{tenant}""#); + haystack.matches(&needle).count() +} + +#[tokio::test] +async fn virgin_store_verified_operator_can_manage_policies() { + let tenant = "acme"; + let state = virgin_state(tenant); + bootstrap_operator_credential(&state, OPERATOR_KEY, tenant).await; + + let operator = SecurityContext::from_resolved_identity("operator", "operator", None); + let decision = authorize_manage_policies(&state, tenant, &operator); + assert!( + decision.is_allowed(), + "verified operator must be allowed manage_policies on a virgin store, got {decision:?}" + ); + + let text = state + .server + .authz + .get_tenant_policy_text(tenant) + .expect("seeded policy text"); + assert!(text.contains(r#"Action::"manage_policies""#)); + assert!(text.contains(r#"PolicySet::"acme""#)); +} + +#[tokio::test] +async fn unverified_or_non_operator_cannot_manage_policies() { + let tenant = "acme"; + let state = virgin_state(tenant); + bootstrap_operator_credential(&state, OPERATOR_KEY, tenant).await; + + let unverified = authorize_manage_policies(&state, tenant, &unverified_operator_context()); + assert!( + !unverified.is_allowed(), + "unverified operator must stay denied, got {unverified:?}" + ); + + let developer = SecurityContext::from_resolved_identity("developer", "developer", None); + let denied = authorize_manage_policies(&state, tenant, &developer); + assert!( + !denied.is_allowed(), + "non-operator must stay denied, got {denied:?}" + ); +} + +#[tokio::test] +async fn rebootstrap_is_idempotent_and_persists_one_granular_row() { + let tenant = "acme"; + let (state, _temp) = virgin_state_with_store(tenant).await; + + bootstrap_operator_credential(&state, OPERATOR_KEY, tenant).await; + bootstrap_operator_credential(&state, OPERATOR_KEY, tenant).await; + + let text = state + .server + .authz + .get_tenant_policy_text(tenant) + .expect("seeded policy text"); + assert_eq!( + statement_occurrences(&text, tenant), + 1, + "re-bootstrap must not duplicate the live permit: {text}" + ); + + let rows = state + .server + .policy_store() + .expect("policy store") + .load_policies_for_tenant(tenant) + .await + .expect("load policies"); + let matches: Vec<_> = rows + .iter() + .filter(|row| row.policy_id == POLICY_ID) + .collect(); + assert_eq!( + matches.len(), + 1, + "re-bootstrap must not create duplicate granular rows: {rows:?}" + ); + assert!(matches[0].enabled); + assert!( + matches[0] + .cedar_text + .contains(r#"Action::"manage_policies""#) + ); +} + +#[tokio::test] +async fn existing_app_cedar_still_loads_after_operator_bootstrap() { + let tenant = "app-tenant"; + let (state, _temp) = virgin_state_with_store(tenant).await; + + bootstrap_operator_credential(&state, OPERATOR_KEY, tenant).await; + install_os_app(&state, tenant, "project-management") + .await + .expect("install project-management"); + + let operator = SecurityContext::from_resolved_identity("operator", "operator", None); + assert!( + authorize_manage_policies(&state, tenant, &operator).is_allowed(), + "operator manage_policies must survive app install" + ); + + let any = SecurityContext::from_resolved_identity("operator", "operator", None); + let mut issue_attrs = HashMap::new(); + issue_attrs.insert("id".to_string(), json!("issue-1")); + let create = + state + .server + .authz + .authorize_for_tenant(tenant, &any, "create", "Issue", &issue_attrs); + assert!( + create.is_allowed(), + "project-management Issue.create must remain allowed after operator seed: {create:?}" + ); + + let rows = state + .server + .policy_store() + .expect("policy store") + .load_policies_for_tenant(tenant) + .await + .expect("load policies"); + assert!( + rows.iter().any(|row| row.policy_id == POLICY_ID), + "operator bootstrap row must remain: {rows:?}" + ); + assert!( + rows.iter().any(|row| { + row.policy_id == "project-management-issue" + && row.cedar_text.contains("resource is Issue") + }), + "app Issue cedar row must persist: {rows:?}" + ); +} + +#[tokio::test] +async fn recovered_granular_row_still_allows_verified_operator() { + let tenant = "acme"; + let temp = tempfile::tempdir().expect("temp recovery db"); + let db_url = format!("file:{}", temp.path().join("policy.db").display()); + let store = TursoEventStore::new(&db_url, None) + .await + .expect("create turso store"); + + let mut seeded = virgin_state(tenant); + seeded + .server + .set_storage_stack(StorageStack::from_turso(store.clone())); + bootstrap_operator_credential(&seeded, OPERATOR_KEY, tenant).await; + + let recovered = PlatformState::new(None); + recover_cedar_policies(&recovered, &store).await; + + let operator = SecurityContext::from_resolved_identity("operator", "operator", None); + let decision = authorize_manage_policies(&recovered, tenant, &operator); + assert!( + decision.is_allowed(), + "recovered operator permit must allow manage_policies, got {decision:?}" + ); +} + +#[tokio::test] +async fn http_policy_api_allows_operator_and_denies_developer() { + let tenant = "acme"; + let (state, _temp) = virgin_state_with_store(tenant).await; + bootstrap_operator_credential(&state, OPERATOR_KEY, tenant).await; + issue_developer_credential(&state, tenant, DEVELOPER_KEY).await; + + let app = build_platform_router(state); + + let allowed = app + .clone() + .oneshot( + Request::get(format!("/api/tenants/{tenant}/policies")) + .header("Authorization", format!("Bearer {OPERATOR_KEY}")) + .header("X-Tenant-Id", tenant) + .body(Body::empty()) + .expect("operator GET policies"), + ) + .await + .expect("operator GET policies should run"); + assert_eq!( + allowed.status(), + StatusCode::OK, + "verified operator GET /policies must succeed" + ); + let body = body_json(allowed).await; + let text = body["policy_text"].as_str().unwrap_or_default(); + assert!(text.contains(r#"Action::"manage_policies""#), "{body}"); + + let listed = app + .clone() + .oneshot( + Request::get(format!("/api/tenants/{tenant}/policies/list")) + .header("Authorization", format!("Bearer {OPERATOR_KEY}")) + .header("X-Tenant-Id", tenant) + .body(Body::empty()) + .expect("operator list policies"), + ) + .await + .expect("operator list should run"); + assert_eq!(listed.status(), StatusCode::OK); + let listed_body = body_json(listed).await; + let empty_policies = Vec::new(); + let ids: Vec<&str> = listed_body["policies"] + .as_array() + .unwrap_or(&empty_policies) + .iter() + .filter_map(|row| row["policy_id"].as_str()) + .collect(); + assert!( + ids.contains(&POLICY_ID), + "list must include the bootstrap row: {listed_body}" + ); + + let denied = app + .oneshot( + Request::get(format!("/api/tenants/{tenant}/policies")) + .header("Authorization", format!("Bearer {DEVELOPER_KEY}")) + .header("X-Tenant-Id", tenant) + .body(Body::empty()) + .expect("developer GET policies"), + ) + .await + .expect("developer GET policies should run"); + assert_eq!( + denied.status(), + StatusCode::FORBIDDEN, + "non-operator GET /policies must be 403" + ); +} + +#[tokio::test] +async fn denied_developer_cannot_self_approve_operator_can() { + let tenant = "acme"; + let (state, _temp) = virgin_state_with_store(tenant).await; + bootstrap_operator_credential(&state, OPERATOR_KEY, tenant).await; + issue_developer_credential(&state, tenant, DEVELOPER_KEY).await; + + let developer_permit = format!( + r#"permit( + principal is Agent, + action == Action::"manage_policies", + resource == PolicySet::"{tenant}" +) when {{ + principal.agent_type == "developer" && + principal.agentTypeVerified == true +}};"# + ); + let pending = PendingDecision::from_denial( + tenant, + "developer", + "Assign", + "Issue", + "issue-1", + json!({"id": "issue-1"}), + "test denial", + None, + ); + let decision_id = pending.id.clone(); + state + .server + .persist_pending_decision(&pending) + .await + .expect("persist pending decision"); + + let app = build_platform_router(state); + let created = app + .clone() + .oneshot( + Request::post(format!("/api/tenants/{tenant}/policies/create")) + .header("Authorization", format!("Bearer {OPERATOR_KEY}")) + .header("X-Tenant-Id", tenant) + .header("content-type", "application/json") + .body(Body::from( + json!({ + "policy_id": "developer-manage-policies", + "cedar_text": developer_permit, + }) + .to_string(), + )) + .expect("create developer policy"), + ) + .await + .expect("create developer policy should run"); + assert_eq!( + created.status(), + StatusCode::CREATED, + "operator must be able to add Cedar" + ); + + let approve_body = json!({ + "scope": { + "principal": "this_agent", + "action": "this_action", + "resource": "this_resource", + "duration": "always" + } + }) + .to_string(); + + let self_approve = app + .clone() + .oneshot( + Request::post(format!( + "/api/tenants/{tenant}/decisions/{decision_id}/approve" + )) + .header("Authorization", format!("Bearer {DEVELOPER_KEY}")) + .header("X-Tenant-Id", tenant) + .header("content-type", "application/json") + .body(Body::from(approve_body.clone())) + .expect("developer self-approve"), + ) + .await + .expect("developer self-approve should run"); + assert_eq!( + self_approve.status(), + StatusCode::FORBIDDEN, + "developer with manage_policies must still get 403 on their own decision" + ); + + let operator_approve = app + .oneshot( + Request::post(format!( + "/api/tenants/{tenant}/decisions/{decision_id}/approve" + )) + .header("Authorization", format!("Bearer {OPERATOR_KEY}")) + .header("X-Tenant-Id", tenant) + .header("content-type", "application/json") + .body(Body::from(approve_body)) + .expect("operator approve"), + ) + .await + .expect("operator approve should run"); + assert_eq!( + operator_approve.status(), + StatusCode::OK, + "verified operator must approve another agent's decision" + ); + let approved = body_json(operator_approve).await; + assert_eq!(approved["status"], "approved"); + assert!( + approved["generated_policy"] + .as_str() + .unwrap_or_default() + .contains(r#"Action::"Assign""#), + "{approved}" + ); +} diff --git a/crates/temper-platform/tests/platform_e2e_dst.rs b/crates/temper-platform/tests/platform_e2e_dst.rs index f581be81c..d3f29e904 100644 --- a/crates/temper-platform/tests/platform_e2e_dst.rs +++ b/crates/temper-platform/tests/platform_e2e_dst.rs @@ -355,7 +355,7 @@ async fn e2e_full_platform_scenario() { /// Full Project lifecycle through HTTP: POST create → POST UpdateSpecs → POST Verify → GET state. #[tokio::test] async fn e2e_http_project_lifecycle() { - let app = bootstrapped_router(); + let app = bootstrapped_router().await; // POST /tdata/Projects → 201, creates a new Project entity let response = app @@ -363,7 +363,7 @@ async fn e2e_http_project_lifecycle() { .oneshot( Request::post("/tdata/Projects") .header("Content-Type", "application/json") - .header("X-Temper-Principal-Kind", "admin") + .header("Authorization", "Bearer test-operator-key") .header("X-Tenant-Id", SYSTEM_TENANT) .body(Body::from(r#"{"name": "test-project"}"#)) .unwrap(), @@ -391,7 +391,7 @@ async fn e2e_http_project_lifecycle() { "/tdata/Projects('{entity_id}')/Temper.System.UpdateSpecs" )) .header("Content-Type", "application/json") - .header("X-Temper-Principal-Kind", "admin") + .header("Authorization", "Bearer test-operator-key") .header("X-Tenant-Id", SYSTEM_TENANT) .body(Body::from("{}")) .unwrap(), @@ -410,7 +410,7 @@ async fn e2e_http_project_lifecycle() { "/tdata/Projects('{entity_id}')/Temper.System.Verify" )) .header("Content-Type", "application/json") - .header("X-Temper-Principal-Kind", "admin") + .header("Authorization", "Bearer test-operator-key") .header("X-Tenant-Id", SYSTEM_TENANT) .body(Body::from("{}")) .unwrap(), @@ -426,6 +426,7 @@ async fn e2e_http_project_lifecycle() { .clone() .oneshot( Request::get("/tdata/$metadata") + .header("Authorization", "Bearer test-operator-key") .header("X-Tenant-Id", SYSTEM_TENANT) .body(Body::empty()) .unwrap(), @@ -444,7 +445,7 @@ async fn e2e_http_project_lifecycle() { .clone() .oneshot( Request::get(format!("/tdata/Projects('{entity_id}')")) - .header("X-Temper-Principal-Kind", "admin") + .header("Authorization", "Bearer test-operator-key") .header("X-Tenant-Id", SYSTEM_TENANT) .body(Body::empty()) .unwrap(), @@ -459,13 +460,14 @@ async fn e2e_http_project_lifecycle() { /// Metadata and service document show all system entity types after bootstrap. #[tokio::test] async fn e2e_http_metadata_shows_system_entities() { - let app = bootstrapped_router(); + let app = bootstrapped_router().await; // GET /tdata/$metadata → body contains all 5 entity types let response = app .clone() .oneshot( Request::get("/tdata/$metadata") + .header("Authorization", "Bearer test-operator-key") .header("X-Tenant-Id", SYSTEM_TENANT) .body(Body::empty()) .unwrap(), @@ -492,6 +494,7 @@ async fn e2e_http_metadata_shows_system_entities() { .clone() .oneshot( Request::get("/tdata") + .header("Authorization", "Bearer test-operator-key") .header("X-Tenant-Id", SYSTEM_TENANT) .body(Body::empty()) .unwrap(), diff --git a/crates/temper-platform/tests/system_entity_actors.rs b/crates/temper-platform/tests/system_entity_actors.rs index 491245cee..8644f2676 100644 --- a/crates/temper-platform/tests/system_entity_actors.rs +++ b/crates/temper-platform/tests/system_entity_actors.rs @@ -51,6 +51,7 @@ async fn actor_project_full_lifecycle() { params: serde_json::json!({}), cross_entity_booleans: std::collections::BTreeMap::new(), idempotency_key: None, + expected_authorization_precondition: None, }, TIMEOUT, ) @@ -67,6 +68,7 @@ async fn actor_project_full_lifecycle() { params: serde_json::json!({}), cross_entity_booleans: std::collections::BTreeMap::new(), idempotency_key: None, + expected_authorization_precondition: None, }, TIMEOUT, ) @@ -83,6 +85,7 @@ async fn actor_project_full_lifecycle() { params: serde_json::json!({}), cross_entity_booleans: std::collections::BTreeMap::new(), idempotency_key: None, + expected_authorization_precondition: None, }, TIMEOUT, ) @@ -107,6 +110,7 @@ async fn actor_project_verify_requires_building_state() { params: serde_json::json!({}), cross_entity_booleans: std::collections::BTreeMap::new(), idempotency_key: None, + expected_authorization_precondition: None, }, TIMEOUT, ) @@ -137,6 +141,7 @@ async fn actor_tenant_full_lifecycle() { params: serde_json::json!({}), cross_entity_booleans: std::collections::BTreeMap::new(), idempotency_key: None, + expected_authorization_precondition: None, }, TIMEOUT, ) @@ -153,6 +158,7 @@ async fn actor_tenant_full_lifecycle() { params: serde_json::json!({}), cross_entity_booleans: std::collections::BTreeMap::new(), idempotency_key: None, + expected_authorization_precondition: None, }, TIMEOUT, ) @@ -169,6 +175,7 @@ async fn actor_tenant_full_lifecycle() { params: serde_json::json!({}), cross_entity_booleans: std::collections::BTreeMap::new(), idempotency_key: None, + expected_authorization_precondition: None, }, TIMEOUT, ) @@ -185,6 +192,7 @@ async fn actor_tenant_full_lifecycle() { params: serde_json::json!({}), cross_entity_booleans: std::collections::BTreeMap::new(), idempotency_key: None, + expected_authorization_precondition: None, }, TIMEOUT, ) @@ -208,6 +216,7 @@ async fn actor_tenant_cannot_deploy_archived() { params: serde_json::json!({}), cross_entity_booleans: std::collections::BTreeMap::new(), idempotency_key: None, + expected_authorization_precondition: None, }, TIMEOUT, ) @@ -220,6 +229,7 @@ async fn actor_tenant_cannot_deploy_archived() { params: serde_json::json!({}), cross_entity_booleans: std::collections::BTreeMap::new(), idempotency_key: None, + expected_authorization_precondition: None, }, TIMEOUT, ) @@ -234,6 +244,7 @@ async fn actor_tenant_cannot_deploy_archived() { params: serde_json::json!({}), cross_entity_booleans: std::collections::BTreeMap::new(), idempotency_key: None, + expected_authorization_precondition: None, }, TIMEOUT, ) @@ -269,6 +280,7 @@ async fn actor_catalog_publish_and_fork() { params: serde_json::json!({}), cross_entity_booleans: std::collections::BTreeMap::new(), idempotency_key: None, + expected_authorization_precondition: None, }, TIMEOUT, ) @@ -285,6 +297,7 @@ async fn actor_catalog_publish_and_fork() { params: serde_json::json!({}), cross_entity_booleans: std::collections::BTreeMap::new(), idempotency_key: None, + expected_authorization_precondition: None, }, TIMEOUT, ) @@ -301,6 +314,7 @@ async fn actor_catalog_publish_and_fork() { params: serde_json::json!({}), cross_entity_booleans: std::collections::BTreeMap::new(), idempotency_key: None, + expected_authorization_precondition: None, }, TIMEOUT, ) @@ -335,6 +349,7 @@ async fn actor_collaborator_invite_accept_remove() { params: serde_json::json!({}), cross_entity_booleans: std::collections::BTreeMap::new(), idempotency_key: None, + expected_authorization_precondition: None, }, TIMEOUT, ) @@ -351,6 +366,7 @@ async fn actor_collaborator_invite_accept_remove() { params: serde_json::json!({"role": "editor"}), cross_entity_booleans: std::collections::BTreeMap::new(), idempotency_key: None, + expected_authorization_precondition: None, }, TIMEOUT, ) @@ -366,6 +382,7 @@ async fn actor_collaborator_invite_accept_remove() { params: serde_json::json!({}), cross_entity_booleans: std::collections::BTreeMap::new(), idempotency_key: None, + expected_authorization_precondition: None, }, TIMEOUT, ) @@ -395,6 +412,7 @@ async fn actor_version_lifecycle() { params: serde_json::json!({}), cross_entity_booleans: std::collections::BTreeMap::new(), idempotency_key: None, + expected_authorization_precondition: None, }, TIMEOUT, ) @@ -410,6 +428,7 @@ async fn actor_version_lifecycle() { params: serde_json::json!({}), cross_entity_booleans: std::collections::BTreeMap::new(), idempotency_key: None, + expected_authorization_precondition: None, }, TIMEOUT, ) @@ -463,6 +482,7 @@ async fn actor_multiple_system_entities_independent() { params: serde_json::json!({}), cross_entity_booleans: std::collections::BTreeMap::new(), idempotency_key: None, + expected_authorization_precondition: None, }, TIMEOUT, ) @@ -475,6 +495,7 @@ async fn actor_multiple_system_entities_independent() { params: serde_json::json!({}), cross_entity_booleans: std::collections::BTreeMap::new(), idempotency_key: None, + expected_authorization_precondition: None, }, TIMEOUT, ) @@ -487,6 +508,7 @@ async fn actor_multiple_system_entities_independent() { params: serde_json::json!({}), cross_entity_booleans: std::collections::BTreeMap::new(), idempotency_key: None, + expected_authorization_precondition: None, }, TIMEOUT, ) diff --git a/crates/temper-runtime/src/scheduler/id_gen.rs b/crates/temper-runtime/src/scheduler/id_gen.rs index 836eab4e8..d93208c3f 100644 --- a/crates/temper-runtime/src/scheduler/id_gen.rs +++ b/crates/temper-runtime/src/scheduler/id_gen.rs @@ -85,7 +85,7 @@ mod tests { fn deterministic_ids_are_unique() { let id_gen = DeterministicIdGen::new(42); let ids: Vec = (0..100).map(|_| id_gen.next_uuid()).collect(); - let unique: std::collections::HashSet<_> = ids.iter().collect(); + let unique: std::collections::HashSet<_> = ids.iter().collect(); // determinism-ok: test-local uniqueness count, never iterated assert_eq!( ids.len(), unique.len(), diff --git a/crates/temper-runtime/src/tenant/mod.rs b/crates/temper-runtime/src/tenant/mod.rs index 3e69f6b9e..006263910 100644 --- a/crates/temper-runtime/src/tenant/mod.rs +++ b/crates/temper-runtime/src/tenant/mod.rs @@ -19,6 +19,18 @@ use serde::{Deserialize, Serialize}; pub struct TenantId(String); impl TenantId { + /// Parse a tenant ID without panicking on caller-controlled input. + pub fn try_new(id: impl Into) -> Result { + let id = id.into(); + if id.is_empty() { + return Err("tenant ID must not be empty".to_string()); + } + if id.contains(':') { + return Err(format!("tenant ID must not contain colons: {id}")); + } + Ok(Self(id)) + } + /// Create a new tenant ID. /// /// # Panics @@ -26,10 +38,7 @@ impl TenantId { /// Panics if the tenant ID is empty or contains colons (which are used as /// separators in persistence IDs). pub fn new(id: impl Into) -> Self { - let id = id.into(); - assert!(!id.is_empty(), "tenant ID must not be empty"); - assert!(!id.contains(':'), "tenant ID must not contain colons: {id}"); - Self(id) + Self::try_new(id).unwrap_or_else(|error| panic!("{error}")) } /// The default tenant for single-tenant deployments. @@ -182,6 +191,18 @@ mod tests { assert_eq!(t.to_string(), "alpha"); } + #[test] + fn tenant_id_try_new_rejects_untrusted_invalid_values() { + assert_eq!( + TenantId::try_new("").unwrap_err(), + "tenant ID must not be empty" + ); + assert_eq!( + TenantId::try_new("a:b").unwrap_err(), + "tenant ID must not contain colons: a:b" + ); + } + #[test] #[should_panic(expected = "must not be empty")] fn tenant_id_empty_panics() { diff --git a/crates/temper-sandbox/Cargo.toml b/crates/temper-sandbox/Cargo.toml index a053384ce..39f9e599e 100644 --- a/crates/temper-sandbox/Cargo.toml +++ b/crates/temper-sandbox/Cargo.toml @@ -11,6 +11,7 @@ anyhow = { workspace = true } serde_json = { workspace = true } tokio = { workspace = true, features = ["time", "process", "fs"] } reqwest = { workspace = true } +tracing = { workspace = true } uuid = { workspace = true } sha2 = { workspace = true } diff --git a/crates/temper-sandbox/src/dispatch.rs b/crates/temper-sandbox/src/dispatch.rs index 56b524316..995952c59 100644 --- a/crates/temper-sandbox/src/dispatch.rs +++ b/crates/temper-sandbox/src/dispatch.rs @@ -10,7 +10,10 @@ use sha2::{Digest, Sha256}; use crate::helpers::{ escape_odata_key, expect_json_object_arg, expect_string_arg, optional_string_arg, }; -use crate::http::{AgentIdentity, temper_governance_request, temper_request, temper_request_bytes}; +use crate::http::{ + AgentIdentity, InternalRequestCredentialIssuer, temper_governance_request, temper_request, + temper_request_bytes, +}; /// Shared context for dispatching temper methods. pub struct DispatchContext<'a> { @@ -22,22 +25,25 @@ pub struct DispatchContext<'a> { pub tenant: &'a str, /// Agent instance ID for Cedar authorization and trajectory attribution. pub agent_id: Option<&'a str>, - /// Agent software type (e.g. `claude-code`). - pub agent_type: Option<&'a str>, /// Session ID for grouping actions within a conversation. pub session_id: Option<&'a str>, - /// Optional passthrough principal id for server-hosted REPL loopback calls. - pub principal_id: Option<&'a str>, - /// Optional passthrough principal kind for server-hosted REPL loopback calls. - pub principal_kind: Option<&'a str>, - /// Optional passthrough agent role for server-hosted REPL loopback calls. - pub agent_role: Option<&'a str>, /// Optional closure to resolve entity type to entity set name. pub entity_set_resolver: Option<&'a (dyn Fn(&str) -> String + Send + Sync)>, /// Optional path to temper binary (for `compile_wasm` SDK resolution). pub binary_path: Option<&'a std::path::Path>, /// Optional API key for authentication. pub api_key: Option<&'a str>, + /// Per-request credential issuer for trusted server-side loopback calls. + pub internal_credential_issuer: Option<&'a InternalRequestCredentialIssuer>, + /// Whether this context may perform host-process operations — local + /// filesystem reads and spawning `cargo` (`upload_wasm`, `compile_wasm`). + /// + /// True only for a runner whose host process is the developer's own machine + /// (the local stdio MCP server). The server-hosted REPL sets this false: its + /// host process is the Temper server, so those ops read the server's + /// filesystem and run code as the server user — a host-compromise vector + /// (ARN-166: arbitrary file read + RCE), not a developer op. + pub allow_host_ops: bool, } impl<'a> DispatchContext<'a> { @@ -45,10 +51,7 @@ impl<'a> DispatchContext<'a> { fn identity(&self) -> AgentIdentity<'a> { AgentIdentity { session_id: self.session_id, - principal_id: self.principal_id, - principal_kind: self.principal_kind, - agent_role: self.agent_role, - agent_type: self.agent_type, + internal_credential_issuer: self.internal_credential_issuer, } } @@ -87,6 +90,31 @@ pub async fn dispatch_temper_method( dispatch_governance(ctx, method, args).await } // --- WASM --- + // Host ops (local file read, `cargo build`) are rejected before they + // touch the filesystem unless this context is host-trusted. The + // server-hosted REPL is not: running these in the server process is a + // host-compromise vector (ARN-166: arbitrary file read + RCE as the + // server user). + "upload_wasm" | "compile_wasm" if !ctx.allow_host_ops => { + // Surface the denial to the operator channel. The Monty runtime + // collapses a dispatch error into a `null` program result, so the + // HTTP caller sees no error field (a separate observability gap); + // this warn makes the refusal visible in logs/Datadog so a denied + // host op is not indistinguishable from a no-op (ARN-166). + tracing::warn!( + target: "temper.repl.host_op", + method = method, + tenant = ctx.tenant, + agent_id = ctx.agent_id.unwrap_or("-"), + "denied host operation in a non-host-trusted REPL context" + ); + Err(format!( + "temper.{method}() is not available in this context. Host operations \ + (local file read, cargo build) run only on the developer's own \ + machine via the local MCP server, never inside the Temper server \ + process." + )) + } "upload_wasm" | "compile_wasm" => dispatch_wasm(ctx, method, args).await, // --- Evolution / Observe --- "get_trajectories" | "get_insights" | "get_evolution_records" | "check_sentinel" => { @@ -417,10 +445,7 @@ async fn dispatch_governance( async move { let identity = AgentIdentity { session_id: session_id_owned.as_deref(), - principal_id: None, - principal_kind: None, - agent_role: None, - agent_type: None, + internal_credential_issuer: None, }; temper_governance_request( &http, @@ -736,3 +761,128 @@ fn resolve_sdk_path(binary_path: Option<&std::path::Path>) -> Result DispatchContext<'_> { + DispatchContext { + http: client, + base_url: "http://127.0.0.1:1", + tenant: "default", + agent_id: None, + session_id: None, + entity_set_resolver: None, + binary_path: None, + api_key: None, + internal_credential_issuer: None, + allow_host_ops, + } + } + + fn str_args(values: &[&str]) -> Vec { + values + .iter() + .map(|v| MontyObject::String((*v).to_string())) + .collect() + } + + /// Without host trust, `upload_wasm` is rejected before the filesystem is + /// touched. The path (`/etc/hosts`) exists and is readable: if the gate + /// failed to fire, dispatch would read it and then fail on the loopback + /// POST — a different error. The "not available in this context" message + /// proves the read never happened (ARN-166). + #[tokio::test] + async fn upload_wasm_rejected_without_host_ops() { + let client = reqwest::Client::new(); + let args = str_args(&["mod", "/etc/hosts"]); + let err = dispatch_temper_method(&ctx(&client, false), "upload_wasm", &args, &[]) + .await + .expect_err("upload_wasm must be rejected without host ops"); + assert!( + err.contains("not available in this context"), + "expected host-op rejection, got: {err}" + ); + assert!( + !err.contains("failed to read"), + "gate must fire before any filesystem read, got: {err}" + ); + } + + /// `compile_wasm` is gated the same way and must be rejected before it + /// spawns `rustup`/`cargo`. + #[tokio::test] + async fn compile_wasm_rejected_without_host_ops() { + let client = reqwest::Client::new(); + let args = str_args(&["mod", "pub fn main() {}"]); + let err = dispatch_temper_method(&ctx(&client, false), "compile_wasm", &args, &[]) + .await + .expect_err("compile_wasm must be rejected without host ops"); + assert!( + err.contains("not available in this context"), + "expected host-op rejection, got: {err}" + ); + // Prove the gate fires before any host side-effect: none of the + // downstream errors (rustup spawn, build-dir write) may appear, so a + // "spawn-then-deny" implementation could not pass this test. + assert!( + !err.contains("failed to run rustup") && !err.contains("failed to create build dir"), + "gate must fire before rustup spawn or build-dir write, got: {err}" + ); + } + + /// With host ops allowed (the local stdio MCP context), the gate does not + /// fire: dispatch proceeds into `upload_wasm` and fails at the filesystem + /// read of a nonexistent path — proving the capability flag, not a hardcoded + /// block, governs the two host methods. + #[tokio::test] + async fn upload_wasm_allowed_with_host_ops_reaches_filesystem() { + let client = reqwest::Client::new(); + let args = str_args(&["mod", "/nonexistent/temper-arn166-does-not-exist"]); + let err = dispatch_temper_method(&ctx(&client, true), "upload_wasm", &args, &[]) + .await + .expect_err("read of a nonexistent path must fail"); + assert!( + err.contains("failed to read"), + "gate must allow the read attempt when host ops are permitted, got: {err}" + ); + assert!( + !err.contains("not available in this context"), + "host-trusted context must not reject host ops, got: {err}" + ); + } + + /// Symmetric positive case for `compile_wasm`. `upload_wasm` and + /// `compile_wasm` share a single guarded match arm, so this proves the same + /// arm lets `compile_wasm` through when host ops are permitted — the gate is + /// the capability flag, not a hardcoded block on either method. + /// + /// This must not trigger a real `cargo build` (minutes long). The `ctx` + /// helper leaves `binary_path: None`, so `compile_wasm` fails at + /// `resolve_sdk_path`: the test binary's cwd is the package root + /// (`crates/temper-sandbox`, per Cargo), which has no + /// `crates/temper-wasm-sdk`. That check runs after the rustup probe and the + /// `/tmp` build-dir creation but *before* the crate files are written or + /// `cargo` is spawned, so no build happens. (Without the wasm32 target it + /// fails even earlier, at the rustup probe.) Either way the error is a + /// downstream environment failure, never the gate rejection, so the gate is + /// proven open for `compile_wasm` without a build. A real build would need + /// the test binary's cwd at the workspace root *and* wasm32 installed — + /// which is not how `cargo test [--workspace]` runs. + #[tokio::test] + async fn compile_wasm_allowed_with_host_ops_passes_the_gate() { + let client = reqwest::Client::new(); + let args = str_args(&["mod", "pub fn main() {}"]); + let err = dispatch_temper_method(&ctx(&client, true), "compile_wasm", &args, &[]) + .await + .expect_err("compile_wasm must fail downstream (no SDK/toolchain), not at the gate"); + assert!( + !err.contains("not available in this context"), + "host-trusted context must not reject compile_wasm, got: {err}" + ); + } +} diff --git a/crates/temper-sandbox/src/http.rs b/crates/temper-sandbox/src/http.rs index d6706b2d9..bfb6cb432 100644 --- a/crates/temper-sandbox/src/http.rs +++ b/crates/temper-sandbox/src/http.rs @@ -6,32 +6,49 @@ //! //! Identity is normally conveyed via `Authorization: Bearer` token. //! The platform resolves the token to a verified agent identity (ADR-0033). -//! The server-hosted REPL can also pass through an already-received -//! `X-Temper-*` identity for local/dev operator workflows. + +use std::sync::Arc; use reqwest::Method; use serde_json::Value; use crate::helpers::{format_authz_denied, format_http_error}; +/// One single-use credential for an exact internal HTTP request. +#[derive(Clone, Debug)] +pub struct InternalRequestCredential { + bearer_token: String, + tenant: String, +} + +impl InternalRequestCredential { + /// Construct a credential returned by a trusted server-side issuer. + pub fn new(bearer_token: String, tenant: String) -> Result { + if bearer_token.is_empty() || tenant.is_empty() { + return Err("internal request credential fields must not be empty".to_string()); + } + Ok(Self { + bearer_token, + tenant, + }) + } +} + +/// Callback that issues a fresh credential for one method and absolute URL. +pub type InternalRequestCredentialIssuer = + Arc Result + Send + Sync>; + /// Agent identity context attached to every Temper request. /// /// Identity is conveyed via `Authorization: Bearer` token. The platform /// resolves the token to a verified agent identity. Session grouping /// is sent via `X-Session-Id`. -#[derive(Debug, Clone, Default)] +#[derive(Clone, Default)] pub struct AgentIdentity<'a> { /// Session ID (becomes `X-Session-Id`). pub session_id: Option<&'a str>, - /// Caller principal id to pass through when the server-hosted REPL invokes - /// the local HTTP API on behalf of a request. - pub principal_id: Option<&'a str>, - /// Caller principal kind (`admin`, `agent`, `customer`). - pub principal_kind: Option<&'a str>, - /// Agent role used by Cedar policies (`lock_client`, `wasm_module`, ...). - pub agent_role: Option<&'a str>, - /// Agent software type / classification. - pub agent_type: Option<&'a str>, + /// Server-side issuer used by REPL loopback calls instead of raw identity. + pub internal_credential_issuer: Option<&'a InternalRequestCredentialIssuer>, } /// Process a Temper HTTP response into a `Result`. @@ -113,28 +130,27 @@ async fn send_json( body: Option<&Value>, ) -> Result { let url = format!("{base_url}{path}"); + let internal_credential = identity + .internal_credential_issuer + .map(|issuer| issuer(method.as_str(), &url)) + .transpose()?; + let request_tenant = internal_credential + .as_ref() + .map_or(tenant, |credential| credential.tenant.as_str()); let mut request = http .request(method, &url) - .header("X-Tenant-Id", tenant) + .header("X-Tenant-Id", request_tenant) .header("Accept", "application/json"); if let Some(sid) = identity.session_id { request = request.header("X-Session-Id", sid); } - if let Some(principal_id) = identity.principal_id { - request = request.header("X-Temper-Principal-Id", principal_id); - } - if let Some(principal_kind) = identity.principal_kind { - request = request.header("X-Temper-Principal-Kind", principal_kind); - } - if let Some(agent_role) = identity.agent_role { - request = request.header("X-Temper-Agent-Role", agent_role); - } - if let Some(agent_type) = identity.agent_type { - request = request.header("X-Temper-Agent-Type", agent_type); - } - - if let Some(key) = api_key { + if let Some(credential) = internal_credential.as_ref() { + request = request.header( + "Authorization", + format!("Bearer {}", credential.bearer_token), + ); + } else if let Some(key) = api_key { request = request.header("Authorization", format!("Bearer {key}")); } @@ -162,31 +178,30 @@ pub async fn temper_request_bytes( body: Vec, ) -> Result { let url = format!("{base_url}{path}"); + let internal_credential = identity + .internal_credential_issuer + .map(|issuer| issuer(method.as_str(), &url)) + .transpose()?; + let request_tenant = internal_credential + .as_ref() + .map_or(tenant, |credential| credential.tenant.as_str()); let mut request = http .request(method, &url) - .header("X-Tenant-Id", tenant) + .header("X-Tenant-Id", request_tenant) .header("Content-Type", "application/wasm"); - if let Some(key) = api_key { + if let Some(credential) = internal_credential.as_ref() { + request = request.header( + "Authorization", + format!("Bearer {}", credential.bearer_token), + ); + } else if let Some(key) = api_key { request = request.header("Authorization", format!("Bearer {key}")); } if let Some(sid) = identity.session_id { request = request.header("X-Session-Id", sid); } - if let Some(principal_id) = identity.principal_id { - request = request.header("X-Temper-Principal-Id", principal_id); - } - if let Some(principal_kind) = identity.principal_kind { - request = request.header("X-Temper-Principal-Kind", principal_kind); - } - if let Some(agent_role) = identity.agent_role { - request = request.header("X-Temper-Agent-Role", agent_role); - } - if let Some(agent_type) = identity.agent_type { - request = request.header("X-Temper-Agent-Type", agent_type); - } - request = request.body(body); let response = request diff --git a/crates/temper-sandbox/src/repl.rs b/crates/temper-sandbox/src/repl.rs index 3936bae0c..3926b1e96 100644 --- a/crates/temper-sandbox/src/repl.rs +++ b/crates/temper-sandbox/src/repl.rs @@ -11,7 +11,7 @@ use crate::dispatch::{DispatchContext, dispatch_temper_method}; use crate::runner::run_sandbox; /// Configuration for a REPL session. -#[derive(Clone, Debug)] +#[derive(Clone)] pub struct ReplConfig { /// Port of the running Temper HTTP server. pub server_port: u16, @@ -19,16 +19,16 @@ pub struct ReplConfig { pub tenant: String, /// Optional local label for the REPL session. pub agent_id: Option, - /// Caller principal id forwarded from the REPL request boundary. - pub principal_id: Option, - /// Caller principal kind forwarded from the REPL request boundary. - pub principal_kind: Option, - /// Agent role forwarded from the REPL request boundary. - pub agent_role: Option, - /// Agent type forwarded from the REPL request boundary. - pub agent_type: Option, /// Session id forwarded from the REPL request boundary. pub session_id: Option, + /// Per-request credential issuer for authenticated loopback calls. + pub internal_credential_issuer: crate::http::InternalRequestCredentialIssuer, + /// Whether host-process ops (`upload_wasm`/`compile_wasm`) are permitted. + /// + /// The server-hosted REPL sets this false: those ops would read the server's + /// filesystem and spawn `cargo` as the server user (ARN-166). The local + /// stdio MCP runner — the developer's own machine — sets it true. + pub allow_host_ops: bool, } /// Run Python code in the Temper Monty sandbox via the REPL endpoint. @@ -36,15 +36,15 @@ pub struct ReplConfig { /// Creates a lightweight HTTP client and dispatches `temper.*` methods /// back to the server at `127.0.0.1:{port}`. pub async fn run_repl(config: &ReplConfig, code: &str) -> Result { - let http = reqwest::Client::new(); + let http = reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build()?; let base_url = format!("http://127.0.0.1:{}", config.server_port); let tenant = config.tenant.clone(); let agent_id = config.agent_id.clone(); - let principal_id = config.principal_id.clone(); - let principal_kind = config.principal_kind.clone(); - let agent_role = config.agent_role.clone(); - let agent_type = config.agent_type.clone(); let session_id = config.session_id.clone(); + let internal_credential_issuer = config.internal_credential_issuer.clone(); + let allow_host_ops = config.allow_host_ops; run_sandbox( code, @@ -55,11 +55,8 @@ pub async fn run_repl(config: &ReplConfig, code: &str) -> Result { let base_url = base_url.clone(); let tenant = tenant.clone(); let agent_id = agent_id.clone(); - let principal_id = principal_id.clone(); - let principal_kind = principal_kind.clone(); - let agent_role = agent_role.clone(); - let agent_type = agent_type.clone(); let session_id = session_id.clone(); + let internal_credential_issuer = internal_credential_issuer.clone(); async move { // Strip self arg (dataclass method calls include self as args[0]) let args = if args.is_empty() { @@ -72,14 +69,12 @@ pub async fn run_repl(config: &ReplConfig, code: &str) -> Result { base_url: &base_url, tenant: &tenant, agent_id: agent_id.as_deref(), - agent_type: agent_type.as_deref(), session_id: session_id.as_deref(), - principal_id: principal_id.as_deref(), - principal_kind: principal_kind.as_deref(), - agent_role: agent_role.as_deref(), entity_set_resolver: None, binary_path: None, api_key: None, + internal_credential_issuer: Some(&internal_credential_issuer), + allow_host_ops, }; dispatch_temper_method(&ctx, &function_name, args, &kwargs).await } @@ -87,3 +82,90 @@ pub async fn run_repl(config: &ReplConfig, code: &str) -> Result { ) .await } + +#[cfg(test)] +mod host_op_propagation_tests { + use super::*; + use std::sync::Arc; + + fn config(port: u16, allow_host_ops: bool) -> ReplConfig { + // Return a valid credential so upload_wasm proceeds to the socket when + // the gate is open; when the gate is closed it never mints one. + let issuer: crate::http::InternalRequestCredentialIssuer = Arc::new(|_method, _url| { + crate::http::InternalRequestCredential::new( + "test-token".to_string(), + "default".to_string(), + ) + }); + ReplConfig { + server_port: port, + tenant: "default".to_string(), + agent_id: None, + session_id: None, + internal_credential_issuer: issuer, + allow_host_ops, + } + } + + /// Run `temper.upload_wasm` against a stand-in listener and report whether a + /// connection reached it. The acceptor accepts then immediately drops the + /// stream, so the REPL's HTTP client sees the peer close and errors out fast + /// rather than blocking on a response the test never sends (no client + /// timeout is configured). It signals each accepted connection on a channel, + /// so the observation is event-driven — no fixed sleep, no shared counter. + async fn upload_wasm_reaches_server(allow_host_ops: bool) -> bool { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let (tx, mut rx) = tokio::sync::mpsc::channel::<()>(4); + let acceptor = tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + drop(stream); + if tx.send(()).await.is_err() { + break; + } + } + }); + + // Read a real, self-created file so the read cannot fail for reasons + // unrelated to the gate; the file is portable (temp dir), unlike + // /etc/hosts. With the gate open, upload_wasm reads it then POSTs. + let wasm_path = + std::env::temp_dir().join(format!("temper-arn166-{}.wasm", uuid::Uuid::new_v4())); + tokio::fs::write(&wasm_path, b"\0asm\x01\0\0\0") + .await + .unwrap(); + let code = format!("temper.upload_wasm('m', '{}')", wasm_path.display()); + + let _ = run_repl(&config(port, allow_host_ops), &code).await; + let _ = tokio::fs::remove_file(&wasm_path).await; + + // A connection either already sits in the accept backlog (gate open) or + // never comes (gate closed). Bound the wait so the closed case returns. + let reached = tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv()) + .await + .ok() + .flatten() + .is_some(); + acceptor.abort(); + reached + } + + /// End-to-end proof that config.allow_host_ops propagates through run_repl to + /// the dispatch gate — closing the "flip the literal, tests still pass" gap. + /// A connection to the stand-in server is the observable: + /// - allow_host_ops=false → the gate suppresses the whole operation → no connection. + /// - allow_host_ops=true → upload_wasm reads its file and POSTs → a connection. + /// The only variable between the two runs is the flag, so a connection on the + /// `false` run (or none on `true`) means the flag failed to reach the gate. + #[tokio::test] + async fn allow_host_ops_propagates_from_config_to_the_dispatch_gate() { + assert!( + !upload_wasm_reaches_server(false).await, + "with allow_host_ops=false the gate must suppress the operation before any server call" + ); + assert!( + upload_wasm_reaches_server(true).await, + "with allow_host_ops=true the operation must proceed to the server — proving the flag, not a hardcoded block, governs the gate and that it propagates through run_repl" + ); + } +} diff --git a/crates/temper-server/Cargo.toml b/crates/temper-server/Cargo.toml index 6ce2378df..90fc36959 100644 --- a/crates/temper-server/Cargo.toml +++ b/crates/temper-server/Cargo.toml @@ -6,6 +6,10 @@ license.workspace = true rust-version.workspace = true description = "HTTP server assembly for Temper entity services" +[[test]] +name = "spec_validate_endpoint" +required-features = ["observe"] + [features] default = [] observe = ["temper-verify"] @@ -21,6 +25,7 @@ temper-store-postgres = { workspace = true } temper-store-turso = { workspace = true } temper-store-redis = { workspace = true } temper-observe = { workspace = true } +temper-ots = { workspace = true } temper-authz = { workspace = true } temper-evolution = { workspace = true } temper-wasm = { workspace = true } @@ -58,10 +63,15 @@ lru = { workspace = true } sha2 = { workspace = true } hmac = "0.12" libc = { workspace = true } +tempfile = "3" # ADR-0055: CPU profile capture. pprof-rs is used because crates.io # `datadog-profiling` is currently an empty placeholder (see ADR for # details). `flamegraph` feature brings in inferno for SVG rendering. -pprof = { version = "0.14", features = ["protobuf", "protobuf-codec"] } +# `prost-codec` rather than `protobuf-codec`: the latter pulls protobuf 2.x, +# which carries RUSTSEC-2024-0437 with no fix on the 2.x line. Both codecs emit +# the same pprof protobuf wire format, so the uploaded profile is unchanged +# (ARN-169). +pprof = { version = "0.14", features = ["prost-codec"] } [dev-dependencies] tokio-test = { workspace = true } @@ -69,12 +79,13 @@ criterion = { workspace = true } tokio = { workspace = true, features = ["full"] } wiremock = { workspace = true } tracing-subscriber = { workspace = true } -opentelemetry_sdk = { workspace = true } +# `testing` gives the in-memory metric exporter used by the ARN-189 replay-skip +# test. Dev-only: not built into any production artifact. +opentelemetry_sdk = { workspace = true, features = ["testing"] } temper-store-sim = { workspace = true } temper-server = { path = ".", features = ["sim"] } temper-wasm = { workspace = true, features = ["test-helpers"] } temper-platform = { workspace = true } -tempfile = "3" [[bench]] name = "actor_throughput" diff --git a/crates/temper-server/src/adapters/claude_code.rs b/crates/temper-server/src/adapters/claude_code.rs index 2d20ab7e9..4f96e96db 100644 --- a/crates/temper-server/src/adapters/claude_code.rs +++ b/crates/temper-server/src/adapters/claude_code.rs @@ -6,7 +6,10 @@ use async_trait::async_trait; use serde_json::Value; use tokio::process::Command; -use super::{AdapterContext, AdapterError, AdapterResult, AgentAdapter}; +use super::{ + AdapterContext, AdapterError, AdapterResult, AgentAdapter, CliAdapterEnvironment, + configure_cli_child_environment, execute_cli_command, +}; /// Adapter implementation for local `claude` CLI execution. #[derive(Debug, Default)] @@ -18,6 +21,10 @@ impl AgentAdapter for ClaudeCodeAdapter { "claude_code" } + fn requires_platform_credential(&self) -> bool { + true + } + async fn execute(&self, ctx: AdapterContext) -> Result { let checkpoint = checkpoint_from_state(&ctx); let run = run_claude(&ctx, checkpoint.as_deref()).await; @@ -92,12 +99,10 @@ async fn run_claude( command.current_dir(workdir); } - // Pass platform-minted credential for identity resolution (ADR-0033). - // The spawned agent uses this token to authenticate back to Temper, - // and the platform resolves it to a verified identity. - if let Some(ref api_key) = ctx.agent_ctx.agent_api_key { - command.env("TEMPER_API_KEY", api_key); - } + // Start from an empty environment, then install only the Claude provider's + // tenant-scoped auth/config and the platform invocation credential. + configure_cli_child_environment(&mut command, ctx, CliAdapterEnvironment::ClaudeCode); + command.kill_on_drop(true); command .env("TEMPER_RUN_ID", ctx.entity_id.clone()) .env("TEMPER_TASK_ID", ctx.entity_id.clone()) @@ -107,10 +112,7 @@ async fn run_claude( command.arg(prompt); } - let output = command - .output() - .await - .map_err(|e| AdapterError::Invocation(format!("failed to spawn '{command_name}': {e}")))?; + let output = execute_cli_command(&mut command, command_name).await?; let duration_ms = started.elapsed().as_millis() as u64; let stdout = String::from_utf8_lossy(&output.stdout).to_string(); diff --git a/crates/temper-server/src/adapters/codex.rs b/crates/temper-server/src/adapters/codex.rs index fef144371..8c9a39559 100644 --- a/crates/temper-server/src/adapters/codex.rs +++ b/crates/temper-server/src/adapters/codex.rs @@ -5,7 +5,10 @@ use std::time::Instant; use async_trait::async_trait; use tokio::process::Command; -use super::{AdapterContext, AdapterError, AdapterResult, AgentAdapter}; +use super::{ + AdapterContext, AdapterError, AdapterResult, AgentAdapter, CliAdapterEnvironment, + configure_cli_child_environment, execute_cli_command, +}; /// Adapter implementation for local `codex` CLI execution. #[derive(Debug, Default)] @@ -17,6 +20,10 @@ impl AgentAdapter for CodexAdapter { "codex" } + fn requires_platform_credential(&self) -> bool { + true + } + async fn execute(&self, ctx: AdapterContext) -> Result { let started = Instant::now(); // determinism-ok: wall-clock timing for external process @@ -35,16 +42,10 @@ impl AgentAdapter for CodexAdapter { command.current_dir(workdir); } - if let Some(codex_home) = ctx.integration_config.get("codex_home") - && !codex_home.trim().is_empty() - { - command.env("CODEX_HOME", codex_home); - } - - // Pass platform-minted credential for identity resolution (ADR-0033). - if let Some(ref api_key) = ctx.agent_ctx.agent_api_key { - command.env("TEMPER_API_KEY", api_key); - } + // Start from an empty environment, then install only the Codex + // provider's tenant-scoped auth/config and invocation credential. + configure_cli_child_environment(&mut command, &ctx, CliAdapterEnvironment::Codex); + command.kill_on_drop(true); command .env("TEMPER_RUN_ID", ctx.entity_id.clone()) .env("TEMPER_TASK_ID", ctx.entity_id.clone()) @@ -68,9 +69,7 @@ impl AgentAdapter for CodexAdapter { } } - let output = command.output().await.map_err(|e| { - AdapterError::Invocation(format!("failed to spawn '{command_name}': {e}")) - })?; + let output = execute_cli_command(&mut command, command_name).await?; let duration_ms = started.elapsed().as_millis() as u64; let stdout = String::from_utf8_lossy(&output.stdout).to_string(); diff --git a/crates/temper-server/src/adapters/mod.rs b/crates/temper-server/src/adapters/mod.rs index 3c6545bad..5e62dc161 100644 --- a/crates/temper-server/src/adapters/mod.rs +++ b/crates/temper-server/src/adapters/mod.rs @@ -10,9 +10,13 @@ mod http_webhook; mod openclaw; use std::collections::BTreeMap; +use std::env::var_os as read_process_environment; // determinism-ok: external CLI launch boundary +use std::fmt; +use std::process::{ExitStatus, Stdio}; use std::sync::Arc; use async_trait::async_trait; +use tokio::io::{AsyncRead, AsyncReadExt}; pub use claude_code::ClaudeCodeAdapter; pub use codex::CodexAdapter; @@ -20,7 +24,7 @@ pub use http_webhook::HttpWebhookAdapter; pub use openclaw::OpenClawAdapter; /// Agent identity context provided to adapter executions. -#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] +#[derive(Clone, Default, serde::Serialize, serde::Deserialize)] pub struct AdapterAgentContext { /// Calling principal ID. pub agent_id: Option, @@ -37,8 +41,20 @@ pub struct AdapterAgentContext { pub agent_api_key: Option, } +impl fmt::Debug for AdapterAgentContext { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("AdapterAgentContext") + .field("agent_id", &self.agent_id) + .field("session_id", &self.session_id) + .field("agent_type", &self.agent_type) + .field("agent_api_key_present", &self.agent_api_key.is_some()) + .finish() + } +} + /// Full adapter invocation context built from dispatch state. -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[derive(Clone, serde::Serialize, serde::Deserialize)] pub struct AdapterContext { /// Tenant identifier. pub tenant: String, @@ -53,13 +69,30 @@ pub struct AdapterContext { /// Serialized current entity state. pub entity_state: serde_json::Value, /// Integration config with secret templates resolved. + #[serde(skip_serializing, default)] pub integration_config: BTreeMap, /// Agent identity context. pub agent_ctx: AdapterAgentContext, /// Per-tenant secrets snapshot for adapter use. + #[serde(skip_serializing, default)] pub secrets: BTreeMap, } +impl fmt::Debug for AdapterContext { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("AdapterContext") + .field("tenant", &self.tenant) + .field("entity_type", &self.entity_type) + .field("entity_id", &self.entity_id) + .field("trigger_action", &self.trigger_action) + .field("agent_ctx", &self.agent_ctx) + .field("integration_config_count", &self.integration_config.len()) + .field("secret_count", &self.secrets.len()) + .finish() + } +} + impl AdapterContext { /// Retrieve a secret value by key from the invocation snapshot. pub fn get_secret(&self, key: &str) -> Option { @@ -126,10 +159,279 @@ pub trait AgentAdapter: Send + Sync { /// Stable adapter type key used for registry lookup. fn adapter_type(&self) -> &str; + /// Whether this adapter launches a client that must authenticate back to + /// Temper during the invocation. + fn requires_platform_credential(&self) -> bool { + false + } + /// Execute this adapter with the provided invocation context. async fn execute(&self, ctx: AdapterContext) -> Result; } +fn configure_temper_api_key(command: &mut tokio::process::Command, api_key: Option<&str>) { + // Defense in depth for callers that do not use the complete CLI boundary. + command.env_remove("TEMPER_API_KEY"); + if let Some(api_key) = api_key { + command.env("TEMPER_API_KEY", api_key); + } +} + +#[derive(Clone, Copy)] +pub(super) enum CliAdapterEnvironment { + ClaudeCode, + Codex, +} + +const CLI_BASE_ENV_ALLOWLIST: &[&str] = &[ + "PATH", + "TMPDIR", + "LANG", + "LC_ALL", + "LC_CTYPE", + "TERM", + "COLORTERM", + "NO_COLOR", + "SSL_CERT_FILE", + "SSL_CERT_DIR", + "NIX_SSL_CERT_FILE", +]; + +const CLI_OUTPUT_STREAM_BUDGET_BYTES: usize = 4 * 1024 * 1024; + +pub(super) struct CliProcessOutput { + pub(super) status: ExitStatus, + pub(super) stdout: Vec, + pub(super) stderr: Vec, +} + +struct BoundedStreamOutput { + bytes: Vec, + exceeded_budget: bool, +} + +async fn read_bounded_cli_stream( + mut stream: impl AsyncRead + Unpin, +) -> std::io::Result { + let mut bytes = Vec::with_capacity(16 * 1024); + let mut exceeded_budget = false; + let mut chunk = [0_u8; 16 * 1024]; + loop { + let read = stream.read(&mut chunk).await?; + if read == 0 { + break; + } + if !exceeded_budget + && bytes + .len() + .checked_add(read) + .is_some_and(|total| total <= CLI_OUTPUT_STREAM_BUDGET_BYTES) + { + bytes.extend_from_slice(&chunk[..read]); + } else { + exceeded_budget = true; + break; + } + } + Ok(BoundedStreamOutput { + bytes, + exceeded_budget, + }) +} + +/// Execute one CLI adapter child while bounding captured stdout and stderr. +/// +/// A reader signals as soon as its retention budget is exhausted; the parent is +/// then killed and reaped. Concurrent reads prevent either pipe from blocking a +/// well-behaved child while it exits. +pub(super) async fn execute_cli_command( + command: &mut tokio::process::Command, + command_name: &str, +) -> Result { + command + .kill_on_drop(true) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let mut child = command.spawn().map_err(|error| { + AdapterError::Invocation(format!("failed to spawn '{command_name}': {error}")) + })?; + let stdout = child.stdout.take().ok_or_else(|| { + AdapterError::Invocation(format!("failed to capture '{command_name}' stdout")) + })?; + let stderr = child.stderr.take().ok_or_else(|| { + AdapterError::Invocation(format!("failed to capture '{command_name}' stderr")) + })?; + + enum Completion { + Complete(CliProcessOutput), + Failed(AdapterError), + } + + let completion = { + let mut wait = Box::pin(child.wait()); + let mut read_stdout = Box::pin(read_bounded_cli_stream(stdout)); + let mut read_stderr = Box::pin(read_bounded_cli_stream(stderr)); + let mut status = None; + let mut captured_stdout = None; + let mut captured_stderr = None; + + loop { + tokio::select! { + result = &mut wait, if status.is_none() => { + match result { + Ok(value) => status = Some(value), + Err(error) => break Completion::Failed(AdapterError::Execution(format!( + "failed waiting for '{command_name}': {error}" + ))), + } + } + result = &mut read_stdout, if captured_stdout.is_none() => { + match result { + Ok(output) if output.exceeded_budget => { + break Completion::Failed(AdapterError::Execution(format!( + "'{command_name}' stdout exceeded the {CLI_OUTPUT_STREAM_BUDGET_BYTES}-byte budget" + ))); + } + Ok(output) => captured_stdout = Some(output.bytes), + Err(error) => break Completion::Failed(AdapterError::Execution(format!( + "failed reading '{command_name}' stdout: {error}" + ))), + } + } + result = &mut read_stderr, if captured_stderr.is_none() => { + match result { + Ok(output) if output.exceeded_budget => { + break Completion::Failed(AdapterError::Execution(format!( + "'{command_name}' stderr exceeded the {CLI_OUTPUT_STREAM_BUDGET_BYTES}-byte budget" + ))); + } + Ok(output) => captured_stderr = Some(output.bytes), + Err(error) => break Completion::Failed(AdapterError::Execution(format!( + "failed reading '{command_name}' stderr: {error}" + ))), + } + } + } + + if status.is_some() && captured_stdout.is_some() && captured_stderr.is_some() { + match ( + status.take(), + captured_stdout.take(), + captured_stderr.take(), + ) { + (Some(status), Some(stdout), Some(stderr)) => { + break Completion::Complete(CliProcessOutput { + status, + stdout, + stderr, + }); + } + _ => continue, + } + } + } + }; + + match completion { + Completion::Complete(output) => Ok(output), + Completion::Failed(error) => { + let _ = child.kill().await; + let _ = child.wait().await; + Err(error) + } + } +} + +/// Install the complete environment for a tool-executing CLI child. +/// +/// The child starts from an empty environment. Only non-authority runtime +/// variables, an explicitly configured home/config path, the selected +/// provider's tenant-scoped credentials, and the invocation bearer are added. +/// Database, Turso, deployment, webhook, unrelated provider, and proxy +/// credentials can therefore never cross this boundary by ambient inheritance. +pub(super) fn configure_cli_child_environment( + command: &mut tokio::process::Command, + ctx: &AdapterContext, + adapter: CliAdapterEnvironment, +) { + command.env_clear(); + for key in CLI_BASE_ENV_ALLOWLIST { + if let Some(value) = read_process_environment(key) { + command.env(key, value); + } + } + + install_config_env(command, ctx, "home", "HOME"); + match adapter { + CliAdapterEnvironment::ClaudeCode => { + install_config_env(command, ctx, "claude_config_dir", "CLAUDE_CONFIG_DIR"); + install_config_env(command, ctx, "anthropic_base_url", "ANTHROPIC_BASE_URL"); + install_tenant_secret( + command, + ctx, + "ANTHROPIC_API_KEY", + &["ANTHROPIC_API_KEY", "anthropic_api_key"], + ); + install_tenant_secret( + command, + ctx, + "ANTHROPIC_AUTH_TOKEN", + &["ANTHROPIC_AUTH_TOKEN", "anthropic_auth_token"], + ); + install_tenant_secret( + command, + ctx, + "CLAUDE_CODE_OAUTH_TOKEN", + &["CLAUDE_CODE_OAUTH_TOKEN", "claude_code_oauth_token"], + ); + } + CliAdapterEnvironment::Codex => { + install_config_env(command, ctx, "codex_home", "CODEX_HOME"); + install_config_env(command, ctx, "openai_base_url", "OPENAI_BASE_URL"); + install_config_env(command, ctx, "openai_organization", "OPENAI_ORGANIZATION"); + install_config_env(command, ctx, "openai_project", "OPENAI_PROJECT"); + install_tenant_secret( + command, + ctx, + "OPENAI_API_KEY", + &["OPENAI_API_KEY", "openai_api_key"], + ); + } + } + configure_temper_api_key(command, ctx.agent_ctx.agent_api_key.as_deref()); +} + +fn install_config_env( + command: &mut tokio::process::Command, + ctx: &AdapterContext, + config_key: &str, + env_key: &str, +) { + if let Some(value) = ctx + .integration_config + .get(config_key) + .filter(|value| !value.trim().is_empty()) + { + command.env(env_key, value); + } +} + +fn install_tenant_secret( + command: &mut tokio::process::Command, + ctx: &AdapterContext, + env_key: &str, + secret_keys: &[&str], +) { + if let Some(value) = secret_keys + .iter() + .find_map(|key| ctx.secrets.get(*key)) + .filter(|value| !value.is_empty()) + { + command.env(env_key, value); + } +} + /// Registry of available adapter implementations keyed by adapter type. #[derive(Clone, Default)] pub struct AdapterRegistry { @@ -174,7 +476,12 @@ impl AdapterRegistry { #[cfg(test)] mod tests { - use super::AdapterRegistry; + use super::{ + AdapterAgentContext, AdapterContext, AdapterRegistry, CLI_OUTPUT_STREAM_BUDGET_BYTES, + CliAdapterEnvironment, configure_cli_child_environment, configure_temper_api_key, + execute_cli_command, + }; + use std::collections::BTreeMap; #[test] fn builtins_are_registered() { @@ -195,4 +502,224 @@ mod tests { assert!(registry.get("http").is_some()); assert!(registry.get("missing").is_none()); } + + #[test] + fn only_cli_adapters_request_platform_credentials() { + let registry = AdapterRegistry::with_builtins(); + assert!( + registry + .get("claude_code") + .expect("Claude adapter should exist") + .requires_platform_credential() + ); + assert!( + registry + .get("codex") + .expect("Codex adapter should exist") + .requires_platform_credential() + ); + assert!( + !registry + .get("http") + .expect("HTTP adapter should exist") + .requires_platform_credential() + ); + assert!( + !registry + .get("openclaw") + .expect("OpenClaw adapter should exist") + .requires_platform_credential() + ); + } + + #[test] + fn adapter_context_debug_redacts_plaintext_credential() { + let context = AdapterContext { + tenant: "tenant-a".to_string(), + entity_type: "Run".to_string(), + entity_id: "run-1".to_string(), + trigger_action: "Execute".to_string(), + trigger_params: serde_json::json!({"prompt": "trigger-secret"}), + entity_state: serde_json::json!({"fields": {"secret": "state-secret"}}), + integration_config: BTreeMap::from([( + "authorization".to_string(), + "integration-secret".to_string(), + )]), + agent_ctx: AdapterAgentContext { + agent_api_key: Some("tmpr_super-secret".to_string()), + ..AdapterAgentContext::default() + }, + secrets: BTreeMap::from([( + "ANTHROPIC_API_KEY".to_string(), + "provider-secret".to_string(), + )]), + }; + let debug = format!("{context:?}"); + for secret in [ + "tmpr_super-secret", + "trigger-secret", + "state-secret", + "integration-secret", + "provider-secret", + ] { + assert!(!debug.contains(secret)); + } + assert!(debug.contains("agent_api_key_present: true")); + + let serialized = serde_json::to_string(&context).expect("serialize redacted context"); + assert!(!serialized.contains("tmpr_super-secret")); + assert!(!serialized.contains("integration-secret")); + assert!(!serialized.contains("provider-secret")); + } + + #[test] + fn child_command_never_inherits_deployment_key() { + let mut command = tokio::process::Command::new("unused"); + command.env("TEMPER_API_KEY", "deployment-root"); + configure_temper_api_key(&mut command, None); + let without_lease = command + .as_std() + .get_envs() + .find(|(key, _)| *key == "TEMPER_API_KEY") + .expect("TEMPER_API_KEY should have an explicit removal entry"); + assert!(without_lease.1.is_none()); + + configure_temper_api_key(&mut command, Some("tmpr_invocation-only")); + let with_lease = command + .as_std() + .get_envs() + .find(|(key, _)| *key == "TEMPER_API_KEY") + .and_then(|(_, value)| value) + .expect("invocation credential should be installed"); + assert_eq!(with_lease, "tmpr_invocation-only"); + } + + fn cli_environment_context() -> AdapterContext { + AdapterContext { + tenant: "tenant-a".to_string(), + entity_type: "Run".to_string(), + entity_id: "run-1".to_string(), + trigger_action: "Execute".to_string(), + trigger_params: serde_json::json!({}), + entity_state: serde_json::json!({}), + integration_config: BTreeMap::from([ + ("home".to_string(), "/tenant/home".to_string()), + ( + "claude_config_dir".to_string(), + "/tenant/claude".to_string(), + ), + ("codex_home".to_string(), "/tenant/codex".to_string()), + ]), + agent_ctx: AdapterAgentContext { + agent_api_key: Some("tmpr_invocation-only".to_string()), + ..AdapterAgentContext::default() + }, + secrets: BTreeMap::from([ + ( + "ANTHROPIC_API_KEY".to_string(), + "tenant-anthropic".to_string(), + ), + ("OPENAI_API_KEY".to_string(), "tenant-openai".to_string()), + ( + "unrelated_tenant_secret".to_string(), + "must-not-cross".to_string(), + ), + ]), + } + } + + fn plant_server_authority_sentinels(command: &mut tokio::process::Command) { + for (key, value) in [ + ("DATABASE_URL", "database-root"), + ("TURSO_AUTH_TOKEN", "turso-root"), + ("TURSO_PLATFORM_AUTH_TOKEN", "turso-platform-root"), + ("WEBHOOK_HMAC_SECRET", "webhook-root"), + ("AWS_SECRET_ACCESS_KEY", "aws-root"), + ("TEMPER_API_KEY", "deployment-root"), + ] { + command.env(key, value); + } + } + + #[cfg(unix)] + #[tokio::test] + async fn claude_child_gets_only_explicit_selected_environment() { + let mut command = tokio::process::Command::new("/bin/sh"); + plant_server_authority_sentinels(&mut command); + configure_cli_child_environment( + &mut command, + &cli_environment_context(), + CliAdapterEnvironment::ClaudeCode, + ); + command.arg("-c").arg( + r#" + test -z "${DATABASE_URL-}" + test -z "${TURSO_AUTH_TOKEN-}" + test -z "${TURSO_PLATFORM_AUTH_TOKEN-}" + test -z "${WEBHOOK_HMAC_SECRET-}" + test -z "${AWS_SECRET_ACCESS_KEY-}" + test -z "${OPENAI_API_KEY-}" + test -z "${unrelated_tenant_secret-}" + test "${TEMPER_API_KEY-}" = "tmpr_invocation-only" + test "${ANTHROPIC_API_KEY-}" = "tenant-anthropic" + test "${HOME-}" = "/tenant/home" + test "${CLAUDE_CONFIG_DIR-}" = "/tenant/claude" + test -n "${PATH-}" + "#, + ); + let output = command.output().await.expect("run isolated Claude child"); + assert!( + output.status.success(), + "isolated Claude environment assertion failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + + #[cfg(unix)] + #[tokio::test] + async fn codex_child_does_not_receive_claude_or_server_authority() { + let mut command = tokio::process::Command::new("/bin/sh"); + plant_server_authority_sentinels(&mut command); + configure_cli_child_environment( + &mut command, + &cli_environment_context(), + CliAdapterEnvironment::Codex, + ); + command.arg("-c").arg( + r#" + test -z "${DATABASE_URL-}" + test -z "${TURSO_AUTH_TOKEN-}" + test -z "${ANTHROPIC_API_KEY-}" + test -z "${unrelated_tenant_secret-}" + test "${TEMPER_API_KEY-}" = "tmpr_invocation-only" + test "${OPENAI_API_KEY-}" = "tenant-openai" + test "${HOME-}" = "/tenant/home" + test "${CODEX_HOME-}" = "/tenant/codex" + test -n "${PATH-}" + "#, + ); + let output = command.output().await.expect("run isolated Codex child"); + assert!( + output.status.success(), + "isolated Codex environment assertion failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + + #[cfg(unix)] + #[tokio::test] + async fn cli_output_capture_rejects_streams_over_budget() { + let mut command = tokio::process::Command::new("/usr/bin/head"); + command + .arg("-c") + .arg((CLI_OUTPUT_STREAM_BUDGET_BYTES + 1).to_string()) + .arg("/dev/zero") + .kill_on_drop(true); + + let error = match execute_cli_command(&mut command, "bounded-output-test").await { + Ok(_) => panic!("oversized stdout must fail closed"), + Err(error) => error, + }; + assert!(error.to_string().contains("stdout exceeded"), "{error}"); + } } diff --git a/crates/temper-server/src/admin/mod.rs b/crates/temper-server/src/admin/mod.rs index 62c3da751..79c481b2b 100644 --- a/crates/temper-server/src/admin/mod.rs +++ b/crates/temper-server/src/admin/mod.rs @@ -4,16 +4,21 @@ //! ADR-0051 sub-decision 5. SRE can retune caps without a redeploy when a //! customer saturates unexpectedly. -use axum::extract::{Path, State}; +use std::collections::BTreeMap; + +use axum::extract::{Extension, Path, Query, State}; use axum::http::StatusCode; use axum::response::IntoResponse; +use axum::response::Response; use axum::routing::{get, patch}; use axum::{Json, Router}; +use temper_authz::AuthenticatedRequestContext; use temper_runtime::tenant::TenantId; use temper_spec::automaton::Admission; -use crate::profiling::{cpu_profile_handler, wall_profile_handler}; +use crate::authz::require_tenant_match; +use crate::profiling::{CpuProfileQuery, cpu_profile_handler, wall_profile_handler}; use crate::state::ServerState; /// Build the `/admin` sub-router. @@ -25,8 +30,48 @@ pub fn build_admin_router() -> Router { ) // ADR-0055: on-demand CPU and wall-clock profile capture. // Gated by TEMPER_PROFILING_ENABLED at request time. - .route("/profile/cpu", get(cpu_profile_handler)) - .route("/profile/wall", get(wall_profile_handler)) + .route("/profile/cpu", get(capture_cpu_profile)) + .route("/profile/wall", get(capture_wall_profile)) +} + +fn require_admin_operation( + state: &ServerState, + authenticated: &AuthenticatedRequestContext, + action: &str, + resource_type: &str, + resource_id: &str, + mut resource_attrs: BTreeMap, +) -> Result<(), StatusCode> { + let security_context = authenticated.security_context(); + resource_attrs.insert( + "id".to_string(), + serde_json::Value::String(resource_id.to_string()), + ); + resource_attrs.insert( + "tenant".to_string(), + serde_json::Value::String(authenticated.tenant().to_string()), + ); + state + .authorize_with_context( + security_context, + action, + resource_type, + &resource_attrs, + authenticated.tenant().as_str(), + ) + .map_err(|denial| { + tracing::warn!( + reason = %denial, + tenant = %authenticated.tenant(), + principal_id = %security_context.principal.id, + action, + resource_type, + resource_id, + "admin operation denied" + ); + StatusCode::FORBIDDEN + })?; + Ok(()) } /// PATCH /admin/admission/{tenant}/{entity_type} @@ -35,10 +80,34 @@ pub fn build_admin_router() -> Router { /// the runtime override and falls back to the spec-declared caps. async fn override_admission( State(state): State, + Extension(authenticated): Extension, Path((tenant, entity_type)): Path<(String, String)>, Json(admission): Json>, ) -> impl IntoResponse { - let _tenant_id = TenantId::from(tenant.clone()); + if let Err(status) = require_tenant_match(&authenticated, &tenant) { + return status.into_response(); + } + let tenant_id = TenantId::from(tenant.clone()); + let resource_id = format!("{tenant}/{entity_type}"); + if let Err(status) = require_admin_operation( + &state, + &authenticated, + "manage_admission", + "AdmissionControl", + &resource_id, + BTreeMap::from([ + ( + "targetTenant".to_string(), + serde_json::Value::String(tenant_id.to_string()), + ), + ( + "entityType".to_string(), + serde_json::Value::String(entity_type.clone()), + ), + ]), + ) { + return status.into_response(); + } state .admission .override_caps(&entity_type, admission.clone()) @@ -58,6 +127,43 @@ async fn override_admission( "admission": admission, })), ) + .into_response() +} + +async fn capture_cpu_profile( + State(state): State, + Extension(authenticated): Extension, + query: Query, +) -> Response { + if let Err(status) = require_admin_operation( + &state, + &authenticated, + "capture_profile", + "Profiler", + "cpu", + BTreeMap::new(), + ) { + return status.into_response(); + } + cpu_profile_handler(query).await +} + +async fn capture_wall_profile( + State(state): State, + Extension(authenticated): Extension, + query: Query, +) -> Response { + if let Err(status) = require_admin_operation( + &state, + &authenticated, + "capture_profile", + "Profiler", + "wall", + BTreeMap::new(), + ) { + return status.into_response(); + } + wall_profile_handler(query).await } #[cfg(test)] @@ -65,6 +171,7 @@ mod tests { use super::*; use axum::body::Body; use axum::http::Request; + use temper_authz::{AuthenticatedRequestContext, SecurityContext}; use temper_runtime::ActorSystem; use temper_spec::csdl::parse_csdl; use tower::ServiceExt; @@ -76,6 +183,65 @@ mod tests { ServerState::new(system, csdl, csdl_xml.to_string()) } + fn authenticated_request(mut request: Request) -> Request { + request + .extensions_mut() + .insert(AuthenticatedRequestContext::new( + TenantId::new("acme"), + SecurityContext::system(), + )); + request + } + + fn operator_request(mut request: Request, tenant: &str) -> Request { + request + .extensions_mut() + .insert(AuthenticatedRequestContext::new( + TenantId::new(tenant), + SecurityContext::from_resolved_identity("operator", "operator", None), + )); + request + } + + fn claimed_admin_request(mut request: Request, tenant: &str) -> Request { + request + .extensions_mut() + .insert(AuthenticatedRequestContext::new( + TenantId::new(tenant), + SecurityContext { + principal: temper_authz::Principal { + id: "claimed-admin".to_string(), + kind: temper_authz::PrincipalKind::Admin, + role: None, + acting_for: None, + agent_type: None, + attributes: Default::default(), + }, + context_attrs: Default::default(), + correlation_id: "admin-side-channel-test".to_string(), + }, + )); + request + } + + #[tokio::test] + async fn typed_admin_kind_does_not_bypass_cedar() { + let app = crate::router::build_router(test_server_state()); + let response = app + .oneshot(claimed_admin_request( + Request::builder() + .method("PATCH") + .uri("/_admin/admission/acme/Session") + .header("Content-Type", "application/json") + .body(Body::from("null")) + .expect("request should build"), + "acme", + )) + .await + .expect("request should run"); + assert_eq!(response.status(), StatusCode::FORBIDDEN); + } + #[tokio::test] async fn admin_override_applies_and_clears_caps() { let state = test_server_state(); @@ -90,21 +256,21 @@ mod tests { }); let resp = app .clone() - .oneshot( + .oneshot(authenticated_request( Request::builder() .method("PATCH") .uri("/_admin/admission/acme/Session") .header("Content-Type", "application/json") .body(Body::from(serde_json::to_vec(&body).unwrap())) .unwrap(), - ) + )) .await .unwrap(); assert_eq!(resp.status(), StatusCode::OK); // Clear: null body unsets. let resp = app - .oneshot( + .oneshot(authenticated_request( Request::builder() .method("PATCH") .uri("/_admin/admission/acme/Session") @@ -113,9 +279,109 @@ mod tests { serde_json::to_vec(&serde_json::Value::Null).unwrap(), )) .unwrap(), - ) + )) .await .unwrap(); assert_eq!(resp.status(), StatusCode::OK); } + + #[tokio::test] + async fn operator_uses_resource_specific_cedar_authority() { + let state = test_server_state(); + state + .authz + .reload_tenant_policies( + "acme", + r#" +permit( + principal == Agent::"operator", + action == Action::"manage_admission", + resource == AdmissionControl::"acme/Session" +); +permit( + principal == Agent::"operator", + action == Action::"capture_profile", + resource == Profiler::"cpu" +); +"#, + ) + .expect("operator policy should parse"); + let app = crate::router::build_router(state); + + let allowed = app + .clone() + .oneshot(operator_request( + Request::builder() + .method("PATCH") + .uri("/_admin/admission/acme/Session") + .header("Content-Type", "application/json") + .body(Body::from("null")) + .expect("request should build"), + "acme", + )) + .await + .expect("request should run"); + assert_eq!(allowed.status(), StatusCode::OK); + + let wrong_resource = app + .clone() + .oneshot(operator_request( + Request::builder() + .method("PATCH") + .uri("/_admin/admission/acme/Other") + .header("Content-Type", "application/json") + .body(Body::from("null")) + .expect("request should build"), + "acme", + )) + .await + .expect("request should run"); + assert_eq!(wrong_resource.status(), StatusCode::FORBIDDEN); + + let wrong_tenant = app + .clone() + .oneshot(operator_request( + Request::builder() + .method("PATCH") + .uri("/_admin/admission/acme/Session") + .header("Content-Type", "application/json") + .body(Body::from("null")) + .expect("request should build"), + "other", + )) + .await + .expect("request should run"); + assert_eq!(wrong_tenant.status(), StatusCode::UNAUTHORIZED); + + let allowed_profile_but_disabled = app + .clone() + .oneshot(operator_request( + Request::builder() + .uri("/_admin/profile/cpu?seconds=1") + .body(Body::empty()) + .expect("request should build"), + "acme", + )) + .await + .expect("request should run"); + assert_eq!( + allowed_profile_but_disabled.status(), + StatusCode::SERVICE_UNAVAILABLE + ); + + let unauthorized_profile_resource = app + .oneshot(operator_request( + Request::builder() + .uri("/_admin/profile/wall?seconds=1") + .body(Body::empty()) + .expect("request should build"), + "acme", + )) + .await + .expect("request should run"); + assert_eq!( + unauthorized_profile_resource.status(), + StatusCode::FORBIDDEN + ); + } } diff --git a/crates/temper-server/src/api/authorize.rs b/crates/temper-server/src/api/authorize.rs index a06e56ff5..5b3327802 100644 --- a/crates/temper-server/src/api/authorize.rs +++ b/crates/temper-server/src/api/authorize.rs @@ -3,14 +3,14 @@ //! Provides lightweight Cedar authorization checks for agent tool calls and //! records tool invocations in the trajectory log for observability. -use axum::extract::State; -use axum::http::{HeaderMap, StatusCode}; +use axum::extract::{Extension, State}; +use axum::http::StatusCode; use axum::response::IntoResponse; +use temper_authz::AuthenticatedRequestContext; use temper_runtime::scheduler::sim_now; use tracing::instrument; -use crate::authz::{DenialInput, record_authz_denial, security_context_from_headers}; -use crate::odata::extract_tenant; +use crate::authz::{DenialInput, record_authz_denial, require_authenticated_context}; use crate::state::{ServerState, TrajectoryEntry, TrajectorySource}; /// Request body for POST /api/authorize. @@ -31,18 +31,46 @@ pub(crate) struct AuthorizeRequest { #[instrument(skip_all, fields(otel.name = "POST /api/authorize"))] pub(crate) async fn handle_authorize( State(state): State, - headers: HeaderMap, + authenticated: Option>, axum::Json(body): axum::Json, ) -> impl IntoResponse { - let security_ctx = security_context_from_headers(&headers, Some(&body.agent_id), None, None); - let resource_attrs = std::collections::BTreeMap::new(); - let tenant = match extract_tenant(&headers, &state) { - Ok(t) => t, - Err(e) => return e.into_response(), + let authenticated = match require_authenticated_context(authenticated.as_deref()) { + Ok(authenticated) => authenticated, + Err(status) => return status.into_response(), }; + let security_ctx = authenticated.security_context(); + if body.agent_id != security_ctx.principal.id { + return ( + StatusCode::FORBIDDEN, + axum::Json(serde_json::json!({ + "error": "agent_id must match the authenticated principal" + })), + ) + .into_response(); + } + let mut resource_attrs = match body.context { + serde_json::Value::Null => std::collections::BTreeMap::new(), + serde_json::Value::Object(context) => context.into_iter().collect(), + _ => { + return ( + StatusCode::BAD_REQUEST, + axum::Json(serde_json::json!({ + "error": "context must be a JSON object" + })), + ) + .into_response(); + } + }; + // The requested resource identity is canonical. Caller-supplied context + // may enrich the resource, but can never replace the UID Cedar evaluates. + resource_attrs.insert( + "id".to_string(), + serde_json::Value::String(body.resource_id.clone()), + ); + let tenant = authenticated.tenant(); match state.authorize_with_context( - &security_ctx, + security_ctx, &body.action, &body.resource_type, &resource_attrs, @@ -62,18 +90,22 @@ pub(crate) async fn handle_authorize( &state, DenialInput { tenant: tenant.as_str(), - security_ctx: &security_ctx, - agent_id_override: Some(&body.agent_id), + security_ctx, + agent_id_override: None, action: &body.action, resource_type: &body.resource_type, resource_id: &body.resource_id, - resource_attrs: serde_json::json!({ - "agent_id": body.agent_id, - "context": body.context, - }), + resource_attrs: serde_json::Value::Object( + resource_attrs.clone().into_iter().collect(), + ), reason: &reason, module_name: None, from_status: None, + intent: authenticated.intent().map(str::to_string), + session_id: authenticated.session_id().map(str::to_string), + // Pre-flight probe: action, resource type, and session are + // caller-chosen, so this row must never enter a conformance verdict. + spec_governed: Some(false), }, ) .await; @@ -121,13 +153,24 @@ pub(crate) struct AuditRequest { #[instrument(skip_all, fields(otel.name = "POST /api/audit"))] pub(crate) async fn handle_audit( State(state): State, - headers: HeaderMap, + authenticated: Option>, axum::Json(body): axum::Json, ) -> impl IntoResponse { - let tenant = match extract_tenant(&headers, &state) { - Ok(t) => t, - Err(e) => return e.into_response(), + let authenticated = match require_authenticated_context(authenticated.as_deref()) { + Ok(authenticated) => authenticated, + Err(status) => return status.into_response(), }; + let security_ctx = authenticated.security_context(); + if body.agent_id != security_ctx.principal.id { + return ( + StatusCode::FORBIDDEN, + axum::Json(serde_json::json!({ + "error": "agent_id must match the authenticated principal" + })), + ) + .into_response(); + } + let tenant = authenticated.tenant(); let entry = TrajectoryEntry { timestamp: sim_now().to_rfc3339(), @@ -139,17 +182,18 @@ pub(crate) async fn handle_audit( from_status: None, to_status: None, error: body.error, - agent_id: Some(body.agent_id), + agent_id: Some(security_ctx.principal.id.clone()), session_id: body.session_id, authz_denied: None, denied_resource: None, denied_module: None, source: Some(TrajectorySource::Entity), spec_governed: Some(false), - agent_type: None, + agent_type: security_ctx.principal.agent_type.clone(), request_body: body.request_body, intent: body.intent, matched_policy_ids: None, + capture_seq: None, }; if !state.enqueue_trajectory_entry(entry) { diff --git a/crates/temper-server/src/api/decisions.rs b/crates/temper-server/src/api/decisions.rs index 113deef8e..f834cdb89 100644 --- a/crates/temper-server/src/api/decisions.rs +++ b/crates/temper-server/src/api/decisions.rs @@ -2,27 +2,31 @@ //! //! Handles listing, approving, and denying evolution decisions, plus SSE //! streaming for real-time decision notifications (both per-tenant and -//! cross-tenant). +//! credential-tenant views). -use std::convert::Infallible; - -use axum::extract::{Path, Query, State}; -use axum::http::{HeaderMap, StatusCode}; +use axum::extract::{Extension, Path, Query, State}; +use axum::http::StatusCode; use axum::response::IntoResponse; -use axum::response::sse::{Event, KeepAlive, Sse}; +use temper_authz::AuthenticatedRequestContext; use temper_evolution::records::{Decision, DecisionRecord, RecordHeader, RecordType}; use temper_runtime::scheduler::sim_now; -use tokio_stream::StreamExt; -use tokio_stream::wrappers::BroadcastStream; use tracing::instrument; use temper_runtime::tenant::TenantId; use super::{PolicyAuthed, decisions_access, empty_decision_list, format_decision_list}; -use crate::authz::{persist_and_activate_policy, require_observe_auth}; +use crate::authz::{ + record_policy_change, require_authenticated_context, require_observe_auth, require_tenant_match, +}; use crate::request_context::AgentContext; use crate::state::{DecisionStatus, PendingDecision, ServerState}; +mod approval; +mod streams; +pub(crate) use streams::{ + handle_agent_progress_stream, handle_all_decisions_stream, handle_decision_stream, +}; + /// Query parameters for listing decisions. #[derive(serde::Deserialize)] pub(crate) struct DecisionListParams { @@ -35,8 +39,6 @@ pub(crate) struct DecisionListParams { pub(crate) struct ApproveBody { /// Policy scope matrix for Cedar generation. scope: temper_authz::PolicyScopeMatrix, - /// Optional: who approved. - decided_by: Option, } /// GET /api/tenants/{tenant}/decisions — list decisions with optional status filter. @@ -45,17 +47,25 @@ pub(crate) struct ApproveBody { #[instrument(skip_all, fields(tenant, otel.name = "GET /api/tenants/{tenant}/decisions"))] pub(crate) async fn handle_list_decisions( State(state): State, - Path(tenant): Path, - headers: HeaderMap, + Path(path_tenant): Path, + authenticated: Option>, Query(params): Query, ) -> impl IntoResponse { - let access = match decisions_access::decision_list_access(&state, &headers, &tenant).await { + let authenticated = match require_authenticated_context(authenticated.as_deref()) { + Ok(authenticated) => authenticated, + Err(status) => return status.into_response(), + }; + if let Err(status) = require_tenant_match(authenticated, &path_tenant) { + return status.into_response(); + } + let tenant = authenticated.tenant().as_str(); + let access = match decisions_access::decision_list_access(&state, authenticated).await { Ok(access) => access, Err(resp) => return resp, }; - if let Some(store) = state.metadata_store_for_tenant(&tenant).await { + if let Some(store) = state.metadata_store_for_tenant(tenant).await { match store - .query_decisions(&tenant, params.status.as_deref()) + .query_decisions(tenant, params.status.as_deref()) .await { Ok(data_strings) => return format_decision_list(access.filter(data_strings)), @@ -71,10 +81,13 @@ pub(crate) async fn handle_list_decisions( #[instrument(skip_all, fields(tenant, id, otel.name = "POST /api/tenants/{tenant}/decisions/{id}/approve"))] pub(crate) async fn handle_approve_decision( State(state): State, - Path((tenant, id)): Path<(String, String)>, - _auth: PolicyAuthed, + Path((_tenant, id)): Path<(String, String)>, + auth: PolicyAuthed, axum::Json(body): axum::Json, ) -> impl IntoResponse { + let tenant = auth.tenant().as_str().to_string(); + let decided_by = auth.security_context().principal.id.clone(); + let approval_guard = state.policy_approval_lock.lock().await; let scope = body.scope; if let Err(e) = temper_authz::validate_policy_scope_matrix(&scope) { return ( @@ -94,7 +107,7 @@ pub(crate) async fn handle_approve_decision( ) .into_response(); }; - match store.get_pending_decision(&id).await { + match store.get_pending_decision(&tenant, &id).await { Ok(Some(data_str)) => match serde_json::from_str::(&data_str) { Ok(d) if d.tenant == tenant => d, _ => { @@ -126,8 +139,23 @@ pub(crate) async fn handle_approve_decision( .into_response(); } + if let Some(resp) = decisions_access::reject_self_resolution(&decided_by, &decision) { + return resp; + } + let generated_policy = decision.generate_policy_from_matrix(&scope); let evolution_record_id = decision.evolution_record_id.clone(); + let pending_decision_json = match serde_json::to_string(&decision) { + Ok(json) => json, + Err(error) => { + tracing::error!(%error, "failed to serialize pending decision before approval"); + return ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to serialize pending decision: {error}"), + ) + .into_response(); + } + }; // Validate the generated policy combined with existing enabled policies. let prospective = { @@ -139,39 +167,96 @@ pub(crate) async fn handle_approve_decision( format!("{existing}\n{generated_policy}") } }; - if let Err(resp) = super::validate_and_reload_policies(&state, &tenant, &prospective) { - return resp; - } - - // Persist the individual policy to the granular `policies` table. - let decided_by_ref = body.decided_by.as_deref().unwrap_or("unknown"); - persist_and_activate_policy( - &state, - &tenant, - &format!("decision:{id}"), - &generated_policy, - decided_by_ref, - ) - .await; - - // Update in-memory map to reflect the new policy. - { - let mut policies = state.tenant_policies.write().unwrap(); // ci-ok: infallible lock - policies.insert(tenant.clone(), prospective); + if let Err(error) = state.authz.validate_tenant_policies(&prospective) { + tracing::warn!(%error, "policy validation failed"); + return ( + StatusCode::BAD_REQUEST, + format!("Policy validation failed: {error}"), + ) + .into_response(); } - // Mark decision approved only after policy reload succeeds. + // Construct the durable approved form before starting the transaction. decision.status = DecisionStatus::Approved; decision.approved_scope = Some(scope.clone()); decision.generated_policy = Some(generated_policy.clone()); - decision.decided_by = body.decided_by.clone(); + decision.decided_by = Some(decided_by.clone()); decision.decided_at = Some(sim_now().to_rfc3339()); let approved_decision = decision.clone(); - - // Persist updated decision synchronously. - if let Err(e) = state.persist_pending_decision(&approved_decision).await { - tracing::warn!(id = %id, error = %e, "failed to persist approved decision"); + let approved_decision_json = match serde_json::to_string(&approved_decision) { + Ok(json) => json, + Err(error) => { + tracing::error!(%error, "failed to serialize approved decision"); + return ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to serialize approved decision: {error}"), + ) + .into_response(); + } + }; + let policy_id = format!("decision:{id}"); + let Some(store) = state.metadata_store_for_tenant(&tenant).await else { + return ( + StatusCode::SERVICE_UNAVAILABLE, + "durable metadata backend not configured", + ) + .into_response(); + }; + let existing_policy_text = { + let policies = state.tenant_policies.read().unwrap(); // ci-ok: infallible lock + policies.get(&tenant).cloned().unwrap_or_default() + }; + let result = approval::commit_then_activate( + || async { + store + .commit_policy_approval(crate::storage::PolicyApprovalCommit { + tenant: &tenant, + decision_id: &id, + approved_decision_json: &approved_decision_json, + policy_id: &policy_id, + cedar_text: &generated_policy, + created_by: &decided_by, + }) + .await + .map_err(|error| error.to_string()) + }, + || { + state + .authz + .reload_tenant_policies(&tenant, &prospective) + .map_err(|error| error.to_string())?; + match state.tenant_policies.write() { + Ok(mut policies) => { + policies.insert(tenant.clone(), prospective.clone()); + Ok(()) + } + Err(error) => { + let rollback_runtime = state + .authz + .reload_tenant_policies(&tenant, &existing_policy_text) + .err() + .map(|rollback| format!("; runtime rollback failed: {rollback}")) + .unwrap_or_default(); + Err(format!( + "tenant policy cache lock poisoned: {error}{rollback_runtime}" + )) + } + } + }, + || async { + store + .rollback_policy_approval(&tenant, &id, &pending_decision_json, &policy_id) + .await + .map_err(|error| error.to_string()) + }, + ) + .await; + if let Err(error) = result { + tracing::error!(decision_id = %id, %error, "policy approval failed"); + return (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()).into_response(); } + drop(approval_guard); + record_policy_change(&state, &tenant, &policy_id, &decided_by); let _ = state .observe_refresh_tx .send(crate::state::ObserveRefreshHint::Decisions); @@ -186,10 +271,7 @@ pub(crate) async fn handle_approve_decision( let d_record = DecisionRecord { header: d_header, decision: Decision::Approved, - decided_by: body - .decided_by - .clone() - .unwrap_or_else(|| "unknown".to_string()), + decided_by: decided_by.clone(), rationale: format!( "Approved with scope: {:?}. Policy: {}", scope, generated_policy @@ -198,17 +280,18 @@ pub(crate) async fn handle_approve_decision( implementation: None, }; // Persist D-Record to the platform metadata backend. - if let Some(store) = state.platform_metadata_store() { + if let Some(store) = state.metadata_store_for_tenant(&tenant).await { let data_json = serde_json::to_string(&d_record).unwrap_or_default(); if let Err(e) = store - .insert_evolution_record( - &d_record.header.id, - "Decision", - &format!("{:?}", d_record.header.status), - &d_record.header.created_by, - d_record.header.derived_from.as_deref(), - &data_json, - ) + .insert_evolution_record(crate::storage::EvolutionRecordWrite { + tenant: &tenant, + id: &d_record.header.id, + record_type: "Decision", + status: &format!("{:?}", d_record.header.status), + created_by: &d_record.header.created_by, + derived_from: d_record.header.derived_from.as_deref(), + data_json: &data_json, + }) .await { tracing::warn!(error = %e, backend = store.backend_name(), "failed to persist D-Record"); @@ -220,7 +303,7 @@ pub(crate) async fn handle_approve_decision( if let Some(ref gd_id) = approved_decision.governance_decision_id { let state_c = state.clone(); let gd_id = gd_id.clone(); - let decided_by = body.decided_by.clone().unwrap_or_else(|| "unknown".into()); + let decided_by = decided_by.clone(); let generated_policy = generated_policy.clone(); let resolution_task = async move { // determinism-ok: async callback dispatch for governance decision resolution @@ -263,15 +346,12 @@ pub(crate) async fn handle_approve_decision( #[instrument(skip_all, fields(tenant, id, otel.name = "POST /api/tenants/{tenant}/decisions/{id}/deny"))] pub(crate) async fn handle_deny_decision( State(state): State, - Path((tenant, id)): Path<(String, String)>, - _auth: PolicyAuthed, - body: Option>, + Path((_tenant, id)): Path<(String, String)>, + auth: PolicyAuthed, + _body: Option>, ) -> impl IntoResponse { - let decided_by = body - .as_ref() - .and_then(|b| b.get("decided_by")) - .and_then(|v| v.as_str()) - .map(String::from); + let tenant = auth.tenant().as_str().to_string(); + let principal_id = auth.security_context().principal.id.clone(); // Read decision from the durable metadata backend. let mut decision: PendingDecision = { @@ -283,7 +363,7 @@ pub(crate) async fn handle_deny_decision( ) .into_response(); }; - match store.get_pending_decision(&id).await { + match store.get_pending_decision(&tenant, &id).await { Ok(Some(data_str)) => match serde_json::from_str::(&data_str) { Ok(d) if d.tenant == tenant => d, _ => { @@ -315,8 +395,12 @@ pub(crate) async fn handle_deny_decision( .into_response(); } + if let Some(resp) = decisions_access::reject_self_resolution(&principal_id, &decision) { + return resp; + } + decision.status = DecisionStatus::Denied; - decision.decided_by = decided_by; + decision.decided_by = Some(principal_id); decision.decided_at = Some(sim_now().to_rfc3339()); let denied_decision = decision.clone(); @@ -369,56 +453,31 @@ pub(crate) async fn handle_deny_decision( .into_response() } -/// GET /api/tenants/{tenant}/decisions/stream — SSE for new pending decisions. -/// -/// Cedar-gated: requires `manage_policies` action on `PolicySet` resource. -#[instrument(skip_all, fields(tenant, otel.name = "GET /api/tenants/{tenant}/decisions/stream"))] -pub(crate) async fn handle_decision_stream( - State(state): State, - Path(tenant): Path, - _auth: PolicyAuthed, -) -> impl IntoResponse { - let rx = state.pending_decision_tx.subscribe(); - let stream = BroadcastStream::new(rx).filter_map(move |result| { - match result { - Ok(pd) => { - if pd.tenant != tenant { - return None; - } - let data = serde_json::to_string(&pd).unwrap_or_default(); - Some(Ok::( - Event::default().event("pending_decision").data(data), - )) - } - // Lagged receiver: skip missed events and continue. - Err(_) => None, - } - }); - - Sse::new(stream) - .keep_alive(KeepAlive::default()) - .into_response() -} - -/// GET /api/decisions — list all decisions across all tenants. -/// -/// Requires admin-level authorization for cross-tenant visibility. +/// GET /api/decisions — list decisions in the credential-bound tenant. #[instrument(skip_all, fields(otel.name = "GET /api/decisions"))] pub(crate) async fn handle_list_all_decisions( State(state): State, - headers: HeaderMap, + authenticated: Option>, Query(params): Query, ) -> impl IntoResponse { - if let Err(status) = require_observe_auth(&state, &headers, "manage_policies", "PolicySet") { - return (status, "Authorization required for cross-tenant access").into_response(); + let authenticated = match require_authenticated_context(authenticated.as_deref()) { + Ok(authenticated) => authenticated, + Err(status) => return status.into_response(), + }; + if let Err(status) = require_observe_auth(&state, authenticated, "manage_policies", "PolicySet") + { + return (status, "Authorization required").into_response(); } - let stores = state.collect_all_metadata_stores().await; + let tenant = authenticated.tenant().as_str(); let mut all_data = Vec::new(); - for store in &stores { - match store.query_all_decisions(params.status.as_deref()).await { + if let Some(store) = state.metadata_store_for_tenant(tenant).await { + match store + .query_decisions(tenant, params.status.as_deref()) + .await + { Ok(data_strings) => all_data.extend(data_strings), Err(e) => { - tracing::warn!(error = %e, backend = store.backend_name(), "failed to query decisions from metadata store"); + tracing::warn!(error = %e, backend = store.backend_name(), tenant, "failed to query decisions from metadata store"); } } } @@ -427,64 +486,3 @@ pub(crate) async fn handle_list_all_decisions( } empty_decision_list() } - -/// GET /api/decisions/stream — SSE for all pending decisions across all tenants. -/// -/// Requires admin-level authorization for cross-tenant visibility. -#[instrument(skip_all, fields(otel.name = "GET /api/decisions/stream"))] -pub(crate) async fn handle_all_decisions_stream( - State(state): State, - headers: HeaderMap, -) -> impl IntoResponse { - if let Err(status) = require_observe_auth(&state, &headers, "manage_policies", "PolicySet") { - return (status, "Authorization required for cross-tenant access").into_response(); - } - let rx = state.pending_decision_tx.subscribe(); - let stream = BroadcastStream::new(rx).filter_map(move |result| match result { - Ok(pd) => { - let data = serde_json::to_string(&pd).unwrap_or_default(); - Some(Ok::( - Event::default().event("pending_decision").data(data), - )) - } - Err(_) => None, - }); - - Sse::new(stream) - .keep_alive(KeepAlive::default()) - .into_response() -} - -/// GET /api/agents/{agent_id}/stream — SSE for agent progress events. -/// -/// Requires admin-level authorization. -#[instrument(skip_all, fields(agent_id, otel.name = "GET /api/agents/{agent_id}/stream"))] -pub(crate) async fn handle_agent_progress_stream( - State(state): State, - Path(agent_id): Path, - headers: HeaderMap, -) -> impl IntoResponse { - if let Err(status) = require_observe_auth(&state, &headers, "read_agents", "AgentAudit") { - return (status, "Authorization required").into_response(); - } - let rx = state.agent_progress_tx.subscribe(); - let stream = BroadcastStream::new(rx).filter_map(move |result| { - match result { - Ok(event) => { - if event.agent_id != agent_id { - return None; - } - let data = serde_json::to_string(&event).unwrap_or_default(); - Some(Ok::( - Event::default().event(&event.kind).data(data), - )) - } - // Lagged receiver: skip missed events and continue. - Err(_) => None, - } - }); - - Sse::new(stream) - .keep_alive(KeepAlive::default()) - .into_response() -} diff --git a/crates/temper-server/src/api/decisions/approval.rs b/crates/temper-server/src/api/decisions/approval.rs new file mode 100644 index 000000000..15022f8b4 --- /dev/null +++ b/crates/temper-server/src/api/decisions/approval.rs @@ -0,0 +1,216 @@ +//! Ordering and compensation for policy approval activation. + +use std::future::Future; + +#[derive(Debug, PartialEq, Eq)] +pub(super) enum ApprovalError { + Persistence(String), + Activation { + activation: String, + rollback: Option, + }, +} + +impl std::fmt::Display for ApprovalError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Persistence(error) => write!(formatter, "approval persistence failed: {error}"), + Self::Activation { + activation, + rollback: None, + } => write!( + formatter, + "policy activation failed and durable approval was rolled back: {activation}" + ), + Self::Activation { + activation, + rollback: Some(rollback), + } => write!( + formatter, + "policy activation failed ({activation}); durable rollback also failed ({rollback})" + ), + } + } +} + +/// Commit durable state first, activate only after commit, and compensate on +/// activation failure. +pub(super) async fn commit_then_activate( + commit: C, + activate: A, + rollback: R, +) -> Result<(), ApprovalError> +where + C: FnOnce() -> CFut, + CFut: Future>, + A: FnOnce() -> Result<(), String>, + R: FnOnce() -> RFut, + RFut: Future>, +{ + commit().await.map_err(ApprovalError::Persistence)?; + if let Err(activation) = activate() { + let rollback = rollback().await.err(); + return Err(ApprovalError::Activation { + activation, + rollback, + }); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::sync::{Arc, Mutex}; + + use super::{ApprovalError, commit_then_activate}; + + #[tokio::test] + async fn persistence_failure_never_activates_or_rolls_back() { + let calls = Arc::new(Mutex::new(Vec::new())); + let result = commit_then_activate( + { + let calls = Arc::clone(&calls); + move || async move { + calls.lock().unwrap().push("commit"); + Err("disk unavailable".to_string()) + } + }, + { + let calls = Arc::clone(&calls); + move || { + calls.lock().unwrap().push("activate"); + Ok(()) + } + }, + { + let calls = Arc::clone(&calls); + move || async move { + calls.lock().unwrap().push("rollback"); + Ok(()) + } + }, + ) + .await; + + assert_eq!( + result, + Err(ApprovalError::Persistence("disk unavailable".to_string())) + ); + assert_eq!(*calls.lock().unwrap(), vec!["commit"]); + } + + #[tokio::test] + async fn activation_failure_compensates_durable_state() { + let calls = Arc::new(Mutex::new(Vec::new())); + let result = commit_then_activate( + { + let calls = Arc::clone(&calls); + move || async move { + calls.lock().unwrap().push("commit"); + Ok(()) + } + }, + { + let calls = Arc::clone(&calls); + move || { + calls.lock().unwrap().push("activate"); + Err("engine unavailable".to_string()) + } + }, + { + let calls = Arc::clone(&calls); + move || async move { + calls.lock().unwrap().push("rollback"); + Ok(()) + } + }, + ) + .await; + + assert_eq!( + result, + Err(ApprovalError::Activation { + activation: "engine unavailable".to_string(), + rollback: None, + }) + ); + assert_eq!( + *calls.lock().unwrap(), + vec!["commit", "activate", "rollback"] + ); + } + + #[tokio::test] + async fn injected_activation_failure_restores_real_durable_rows() { + let url = format!( + "file:{}/temper-policy-approval-{}.db", + std::env::temp_dir().display(), + uuid::Uuid::new_v4() // determinism-ok: unique filename in a real-Turso unit test + ); + let store = temper_store_turso::TursoEventStore::new(&url, None) + .await + .unwrap(); + let pending = serde_json::json!({ + "id": "decision-1", + "tenant": "tenant-a", + "status": "pending" + }) + .to_string(); + let approved = serde_json::json!({ + "id": "decision-1", + "tenant": "tenant-a", + "status": "approved" + }) + .to_string(); + store + .upsert_pending_decision("decision-1", "tenant-a", "pending", &pending) + .await + .unwrap(); + + let result = commit_then_activate( + || async { + store + .commit_policy_approval(temper_store_turso::TursoPolicyApprovalCommit { + tenant: "tenant-a", + decision_id: "decision-1", + approved_decision_json: &approved, + policy_id: "decision:decision-1", + cedar_text: "permit(principal, action, resource);", + created_by: "reviewer", + }) + .await + .map_err(|error| error.to_string()) + }, + || Err("injected activation failure".to_string()), + || async { + store + .rollback_policy_approval( + "tenant-a", + "decision-1", + &pending, + "decision:decision-1", + ) + .await + .map_err(|error| error.to_string()) + }, + ) + .await; + + assert!(matches!(result, Err(ApprovalError::Activation { .. }))); + assert!( + store + .load_policies_for_tenant("tenant-a") + .await + .unwrap() + .is_empty() + ); + assert_eq!( + store + .get_pending_decision("tenant-a", "decision-1") + .await + .unwrap() + .unwrap(), + pending + ); + } +} diff --git a/crates/temper-server/src/api/decisions/streams.rs b/crates/temper-server/src/api/decisions/streams.rs new file mode 100644 index 000000000..11c15110e --- /dev/null +++ b/crates/temper-server/src/api/decisions/streams.rs @@ -0,0 +1,99 @@ +use std::convert::Infallible; + +use axum::extract::{Extension, Path, State}; +use axum::response::IntoResponse; +use axum::response::sse::{Event, KeepAlive, Sse}; +use temper_authz::AuthenticatedRequestContext; +use tokio_stream::StreamExt; +use tokio_stream::wrappers::BroadcastStream; +use tracing::instrument; + +use super::super::PolicyAuthed; +use crate::authz::{require_authenticated_context, require_observe_auth}; +use crate::state::ServerState; + +/// GET /api/tenants/{tenant}/decisions/stream — SSE for pending decisions. +#[instrument(skip_all, fields(tenant, otel.name = "GET /api/tenants/{tenant}/decisions/stream"))] +pub(crate) async fn handle_decision_stream( + State(state): State, + Path(_tenant): Path, + auth: PolicyAuthed, +) -> impl IntoResponse { + let tenant = auth.tenant().as_str().to_string(); + let rx = state.pending_decision_tx.subscribe(); + let stream = BroadcastStream::new(rx).filter_map(move |result| match result { + Ok(pd) if pd.tenant == tenant => { + let data = serde_json::to_string(&pd).unwrap_or_default(); + Some(Ok::( + Event::default().event("pending_decision").data(data), + )) + } + Ok(_) | Err(_) => None, + }); + + Sse::new(stream) + .keep_alive(KeepAlive::default()) + .into_response() +} + +/// GET /api/decisions/stream — tenant-scoped pending decision stream. +#[instrument(skip_all, fields(otel.name = "GET /api/decisions/stream"))] +pub(crate) async fn handle_all_decisions_stream( + State(state): State, + authenticated: Option>, +) -> impl IntoResponse { + let authenticated = match require_authenticated_context(authenticated.as_deref()) { + Ok(authenticated) => authenticated, + Err(status) => return status.into_response(), + }; + if let Err(status) = require_observe_auth(&state, authenticated, "manage_policies", "PolicySet") + { + return (status, "Authorization required").into_response(); + } + let tenant = authenticated.tenant().as_str().to_string(); + let rx = state.pending_decision_tx.subscribe(); + let stream = BroadcastStream::new(rx).filter_map(move |result| match result { + Ok(pd) if pd.tenant == tenant => { + let data = serde_json::to_string(&pd).unwrap_or_default(); + Some(Ok::( + Event::default().event("pending_decision").data(data), + )) + } + Ok(_) | Err(_) => None, + }); + + Sse::new(stream) + .keep_alive(KeepAlive::default()) + .into_response() +} + +/// GET /api/agents/{agent_id}/stream — SSE for agent progress events. +#[instrument(skip_all, fields(agent_id, otel.name = "GET /api/agents/{agent_id}/stream"))] +pub(crate) async fn handle_agent_progress_stream( + State(state): State, + Path(agent_id): Path, + authenticated: Option>, +) -> impl IntoResponse { + let authenticated = match require_authenticated_context(authenticated.as_deref()) { + Ok(authenticated) => authenticated, + Err(status) => return status.into_response(), + }; + if let Err(status) = require_observe_auth(&state, authenticated, "read_agents", "AgentAudit") { + return (status, "Authorization required").into_response(); + } + let tenant = authenticated.tenant().as_str().to_string(); + let rx = state.agent_progress_tx.subscribe(); + let stream = BroadcastStream::new(rx).filter_map(move |result| match result { + Ok(event) if event.tenant == tenant && event.agent_id == agent_id => { + let data = serde_json::to_string(&event).unwrap_or_default(); + Some(Ok::( + Event::default().event(&event.kind).data(data), + )) + } + Ok(_) | Err(_) => None, + }); + + Sse::new(stream) + .keep_alive(KeepAlive::default()) + .into_response() +} diff --git a/crates/temper-server/src/api/decisions_access.rs b/crates/temper-server/src/api/decisions_access.rs index 2fa105aa5..e91202164 100644 --- a/crates/temper-server/src/api/decisions_access.rs +++ b/crates/temper-server/src/api/decisions_access.rs @@ -1,38 +1,61 @@ //! Decision read access helpers. -use axum::http::HeaderMap; -use axum::response::Response; -use temper_authz::PrincipalKind; +use axum::http::StatusCode; +use axum::response::{IntoResponse, Response}; +use temper_authz::{AuthenticatedRequestContext, PrincipalKind}; use super::require_policy_auth; -use crate::authz::security_context_from_headers; use crate::state::{PendingDecision, ServerState}; +/// True when the caller is the principal who was denied. +pub(crate) fn is_self_resolution(principal_id: &str, denied_agent_id: &str) -> bool { + principal_id == denied_agent_id +} + +/// Forbid the denied principal from approving or denying their own decision. +/// +/// Independent of Cedar (ADR-0172). A caller with `manage_policies` still +/// cannot resolve a decision whose `agent_id` is their own principal id. +pub(crate) fn reject_self_resolution( + principal_id: &str, + decision: &PendingDecision, +) -> Option { + if !is_self_resolution(principal_id, &decision.agent_id) { + return None; + } + tracing::warn!( + decision_id = %decision.id, + agent_id = %decision.agent_id, + "denied principal cannot approve or deny their own decision" + ); + Some( + ( + StatusCode::FORBIDDEN, + axum::Json(serde_json::json!({ + "error": { + "code": "AuthorizationDenied", + "message": "The denied principal cannot approve or deny this decision", + } + })), + ) + .into_response(), + ) +} + pub(crate) enum DecisionListAccess { Full, - Owned { - agent_id: String, - session_id: Option, - }, + Owned { agent_id: String }, } impl DecisionListAccess { pub(crate) fn filter(&self, data_strings: Vec) -> Vec { match self { Self::Full => data_strings, - Self::Owned { - agent_id, - session_id, - } => data_strings + Self::Owned { agent_id } => data_strings .into_iter() .filter(|data| { serde_json::from_str::(data) - .map(|decision| { - let same_session = session_id - .as_deref() - .is_some_and(|id| decision.session_id.as_deref() == Some(id)); - decision.agent_id == *agent_id || same_session - }) + .map(|decision| decision.agent_id == *agent_id) .unwrap_or(false) }) .collect(), @@ -42,27 +65,45 @@ impl DecisionListAccess { pub(crate) async fn decision_list_access( state: &ServerState, - headers: &HeaderMap, - tenant: &str, + authenticated: &AuthenticatedRequestContext, ) -> Result { - let security_ctx = security_context_from_headers(headers, None, None, None); - if matches!(security_ctx.principal.kind, PrincipalKind::Admin) { - return Ok(DecisionListAccess::Full); - } - + let security_ctx = authenticated.security_context(); if matches!(security_ctx.principal.kind, PrincipalKind::Agent) { return Ok(DecisionListAccess::Owned { - agent_id: security_ctx.principal.id, - session_id: security_ctx - .context_attrs - .get("sessionId") - .and_then(|value| value.as_str()) - .map(ToOwned::to_owned), + agent_id: security_ctx.principal.id.clone(), }); } - match require_policy_auth(state, headers, tenant).await { + match require_policy_auth(state, authenticated).await { Some(response) => Err(response), None => Ok(DecisionListAccess::Full), } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn self_resolution_matches_denied_principal_only() { + assert!(is_self_resolution("developer", "developer")); + assert!(!is_self_resolution("operator", "developer")); + assert!(!is_self_resolution("developer", "operator")); + } + + #[test] + fn reject_self_resolution_blocks_denied_principal() { + let decision = PendingDecision::from_denial( + "acme", + "developer", + "Assign", + "Issue", + "issue-1", + serde_json::json!({"id": "issue-1"}), + "denied", + None, + ); + assert!(reject_self_resolution("developer", &decision).is_some()); + assert!(reject_self_resolution("operator", &decision).is_none()); + } +} diff --git a/crates/temper-server/src/api/decisions_get.rs b/crates/temper-server/src/api/decisions_get.rs index 2ff2d0a43..55a55369e 100644 --- a/crates/temper-server/src/api/decisions_get.rs +++ b/crates/temper-server/src/api/decisions_get.rs @@ -1,23 +1,31 @@ //! Tenant-scoped decision lookup. -use axum::extract::{Path, State}; -use axum::http::{HeaderMap, StatusCode}; +use axum::extract::{Extension, Path, State}; +use axum::http::StatusCode; use axum::response::IntoResponse; -use temper_authz::PrincipalKind; +use temper_authz::{AuthenticatedRequestContext, PrincipalKind}; use tracing::instrument; use super::require_policy_auth; -use crate::authz::security_context_from_headers; +use crate::authz::{require_authenticated_context, require_tenant_match}; use crate::state::{PendingDecision, ServerState}; /// GET /api/tenants/{tenant}/decisions/{id} — fetch one decision by ID. #[instrument(skip_all, fields(tenant, id, otel.name = "GET /api/tenants/{tenant}/decisions/{id}"))] pub(crate) async fn handle_get_decision( State(state): State, - Path((tenant, id)): Path<(String, String)>, - headers: HeaderMap, + Path((path_tenant, id)): Path<(String, String)>, + authenticated: Option>, ) -> impl IntoResponse { - let Some(store) = state.metadata_store_for_tenant(&tenant).await else { + let authenticated = match require_authenticated_context(authenticated.as_deref()) { + Ok(authenticated) => authenticated, + Err(status) => return status.into_response(), + }; + if let Err(status) = require_tenant_match(authenticated, &path_tenant) { + return status.into_response(); + } + let tenant = authenticated.tenant().as_str(); + let Some(store) = state.metadata_store_for_tenant(tenant).await else { tracing::error!("durable metadata backend not configured for get decision"); return ( StatusCode::SERVICE_UNAVAILABLE, @@ -25,7 +33,7 @@ pub(crate) async fn handle_get_decision( ) .into_response(); }; - let decision: PendingDecision = match store.get_pending_decision(&id).await { + let decision: PendingDecision = match store.get_pending_decision(tenant, &id).await { Ok(Some(data_str)) => match serde_json::from_str::(&data_str) { Ok(d) if d.tenant == tenant => d, _ => return (StatusCode::NOT_FOUND, "Decision not found").into_response(), @@ -41,18 +49,10 @@ pub(crate) async fn handle_get_decision( } }; - let security_ctx = security_context_from_headers(&headers, None, None, None); - let session_id = security_ctx - .context_attrs - .get("sessionId") - .and_then(|v| v.as_str()); + let security_ctx = authenticated.security_context(); let owner_agent = matches!(security_ctx.principal.kind, PrincipalKind::Agent) - && (security_ctx.principal.id == decision.agent_id - || session_id.is_some() && session_id == decision.session_id.as_deref()); - if !matches!(security_ctx.principal.kind, PrincipalKind::Admin) - && !owner_agent - && let Some(resp) = require_policy_auth(&state, &headers, &tenant).await - { + && security_ctx.principal.id == decision.agent_id; + if !owner_agent && let Some(resp) = require_policy_auth(&state, authenticated).await { return resp; } diff --git a/crates/temper-server/src/api/files.rs b/crates/temper-server/src/api/files.rs index 985594d97..bc7f66441 100644 --- a/crates/temper-server/src/api/files.rs +++ b/crates/temper-server/src/api/files.rs @@ -1,10 +1,17 @@ -use axum::extract::State; -use axum::http::HeaderMap; +use std::collections::{BTreeMap, BTreeSet}; + +use axum::extract::{Extension, State}; use axum::response::IntoResponse; +use temper_authz::AuthenticatedRequestContext; use tracing::instrument; -use crate::odata::extract_tenant; -use crate::state::{PublishFileArtifactRequest, ServerState}; +use crate::authz::{ + ResourceAuthorization, require_authenticated_context, require_resource_authorization, +}; +use crate::state::{ + BatchTextReadError, PUBLISH_ARTIFACT_STALE_AUTHORIZATION, PublishArtifactAuthorization, + PublishFileArtifactRequest, ServerState, validate_batch_text_ids, +}; #[derive(Debug, serde::Deserialize)] pub(crate) struct BatchTextFileReadRequest { @@ -47,35 +54,118 @@ pub(crate) struct PublishArtifactResponse { artifact: crate::storage::PublishedArtifactStoreRow, } +async fn authoritative_file_resource_attrs( + state: &ServerState, + tenant: &temper_runtime::tenant::TenantId, + resource_type: &str, + resource_id: &str, +) -> Result<(BTreeMap, Option), axum::http::StatusCode> { + let governed = state + .has_registered_spec(tenant, resource_type) + .map_err(|_| axum::http::StatusCode::INTERNAL_SERVER_ERROR)?; + if !governed + || !state + .ensure_entity_loaded(tenant, resource_type, resource_id) + .await + { + return Ok((BTreeMap::new(), None)); + } + state + .load_authz_resource_snapshot(tenant, resource_type, resource_id) + .await + .map(|snapshot| { + let precondition = crate::entity_actor::effects::entity_authorization_precondition( + &snapshot.current_state.state, + ); + (snapshot.resource_attrs, Some(precondition)) + }) + .map_err(|_| axum::http::StatusCode::INTERNAL_SERVER_ERROR) +} + +async fn require_file_resources( + state: &ServerState, + authenticated: &AuthenticatedRequestContext, + resource_type: &str, + resource_ids: &[String], +) -> Result<(), axum::http::StatusCode> { + let mut checked = BTreeSet::new(); + for resource_id in resource_ids { + if checked.insert(resource_id.as_str()) { + let (resource_attrs, _) = authoritative_file_resource_attrs( + state, + authenticated.tenant(), + resource_type, + resource_id, + ) + .await?; + require_resource_authorization( + state, + authenticated, + ResourceAuthorization { + action: "read", + resource_type, + resource_id, + resource_attrs, + }, + )?; + } + } + Ok(()) +} + +fn batch_read_error_response(error: BatchTextReadError) -> axum::response::Response { + let (status, code) = match &error { + BatchTextReadError::InvalidRequest(_) => { + (axum::http::StatusCode::BAD_REQUEST, "InvalidBatchFileRead") + } + BatchTextReadError::TooManyItems { .. } + | BatchTextReadError::ItemTooLarge { .. } + | BatchTextReadError::ResponseTooLarge { .. } => ( + axum::http::StatusCode::PAYLOAD_TOO_LARGE, + "BatchFileReadBudgetExceeded", + ), + BatchTextReadError::Storage(_) => ( + axum::http::StatusCode::INTERNAL_SERVER_ERROR, + "BatchFileReadFailed", + ), + }; + ( + status, + axum::Json(serde_json::json!({ + "error": { "code": code, "message": error.to_string() } + })), + ) + .into_response() +} + /// POST /api/files/read-text-batch — read many text file bodies via the /// projection-backed immutable content path. #[instrument(skip_all, fields(otel.name = "POST /api/files/read-text-batch"))] pub(crate) async fn handle_read_text_batch( State(state): State, - headers: HeaderMap, + authenticated: Option>, axum::Json(body): axum::Json, ) -> impl IntoResponse { - let tenant = match extract_tenant(&headers, &state) { - Ok(t) => t, - Err(e) => return e.into_response(), + let authenticated = match require_authenticated_context(authenticated.as_deref()) { + Ok(authenticated) => authenticated, + Err(status) => return status.into_response(), }; + let tenant = authenticated.tenant(); + if let Err(error) = validate_batch_text_ids(&body.file_ids) { + return batch_read_error_response(error); + } + if let Err(status) = require_file_resources(&state, authenticated, "File", &body.file_ids).await + { + return status.into_response(); + } - match state.read_file_texts_batch(&tenant, &body.file_ids).await { + match state.read_file_texts_batch(tenant, &body.file_ids).await { Ok(files) => ( axum::http::StatusCode::OK, axum::Json(BatchTextFileReadResponse { files }), ) .into_response(), - Err(error) => ( - axum::http::StatusCode::INTERNAL_SERVER_ERROR, - axum::Json(serde_json::json!({ - "error": { - "code": "BatchFileReadFailed", - "message": error, - } - })), - ) - .into_response(), + Err(error) => batch_read_error_response(error), } } @@ -84,16 +174,25 @@ pub(crate) async fn handle_read_text_batch( #[instrument(skip_all, fields(otel.name = "POST /api/files/read-version-text-batch"))] pub(crate) async fn handle_read_version_text_batch( State(state): State, - headers: HeaderMap, + authenticated: Option>, axum::Json(body): axum::Json, ) -> impl IntoResponse { - let tenant = match extract_tenant(&headers, &state) { - Ok(t) => t, - Err(e) => return e.into_response(), + let authenticated = match require_authenticated_context(authenticated.as_deref()) { + Ok(authenticated) => authenticated, + Err(status) => return status.into_response(), }; + let tenant = authenticated.tenant(); + if let Err(error) = validate_batch_text_ids(&body.file_version_ids) { + return batch_read_error_response(error); + } + if let Err(status) = + require_file_resources(&state, authenticated, "FileVersion", &body.file_version_ids).await + { + return status.into_response(); + } match state - .read_file_version_texts_batch(&tenant, &body.file_version_ids) + .read_file_version_texts_batch(tenant, &body.file_version_ids) .await { Ok(files) => ( @@ -101,16 +200,7 @@ pub(crate) async fn handle_read_version_text_batch( axum::Json(BatchTextFileVersionReadResponse { files }), ) .into_response(), - Err(error) => ( - axum::http::StatusCode::INTERNAL_SERVER_ERROR, - axum::Json(serde_json::json!({ - "error": { - "code": "BatchFileVersionReadFailed", - "message": error, - } - })), - ) - .into_response(), + Err(error) => batch_read_error_response(error), } } @@ -132,13 +222,14 @@ pub(crate) async fn handle_read_version_text_batch( ))] pub(crate) async fn handle_publish_artifact( State(state): State, - headers: HeaderMap, + authenticated: Option>, axum::Json(body): axum::Json, ) -> impl IntoResponse { - let tenant = match extract_tenant(&headers, &state) { - Ok(t) => t, - Err(e) => return e.into_response(), + let authenticated = match require_authenticated_context(authenticated.as_deref()) { + Ok(authenticated) => authenticated, + Err(status) => return status.into_response(), }; + let tenant = authenticated.tenant(); let span = tracing::Span::current(); span.record("tenant", tenant.as_str()); span.record("file_id", body.file_id.as_str()); @@ -163,18 +254,175 @@ pub(crate) async fn handle_publish_artifact( .into_response(); } - match state - .publish_file_artifact( - &tenant, - PublishFileArtifactRequest { - file_id: body.file_id, - label: body.label, - owner_ref_type: body.owner_ref_type, - owner_ref_id: body.owner_ref_id, - source_file_version_id: body.source_file_version_id, - namespace: body.namespace, + let request_attrs = BTreeMap::from([ + ( + "requestedFileId".to_string(), + serde_json::Value::String(body.file_id.clone()), + ), + ( + "requestedSourceFileVersionId".to_string(), + serde_json::Value::String(body.source_file_version_id.clone()), + ), + ( + "requestedLabel".to_string(), + serde_json::Value::String(body.label.clone()), + ), + ( + "requestedOwnerRefType".to_string(), + serde_json::Value::String(body.owner_ref_type.clone()), + ), + ( + "requestedOwnerRefId".to_string(), + serde_json::Value::String(body.owner_ref_id.clone()), + ), + ( + "requestedNamespace".to_string(), + serde_json::Value::String(body.namespace.clone().unwrap_or_default()), + ), + ]); + let (mut file_resource_attrs, file_state_precondition) = + match authoritative_file_resource_attrs(&state, tenant, "File", &body.file_id).await { + Ok(attrs) => attrs, + Err(status) => return status.into_response(), + }; + let file_authorization = + file_state_precondition.map(|state_precondition| PublishArtifactAuthorization { + source_entity_type: "File".to_string(), + source_entity_id: body.file_id.clone(), + state_precondition, + resource_attrs: file_resource_attrs.clone(), + }); + file_resource_attrs.extend(request_attrs.clone()); + let publish_authorizations = if body.source_file_version_id.trim().is_empty() { + if let Err(status) = require_resource_authorization( + &state, + authenticated, + ResourceAuthorization { + action: "publish_artifact", + resource_type: "File", + resource_id: &body.file_id, + resource_attrs: file_resource_attrs, + }, + ) { + return status.into_response(); + } + file_authorization.into_iter().collect() + } else { + let (mut version_resource_attrs, version_state_precondition) = + match authoritative_file_resource_attrs( + &state, + tenant, + "FileVersion", + &body.source_file_version_id, + ) + .await + { + Ok(attrs) => attrs, + Err(status) => return status.into_response(), + }; + let authoritative_source_file_id = version_resource_attrs + .get("file_id") + .and_then(serde_json::Value::as_str) + .filter(|file_id| !file_id.is_empty()) + .map(str::to_string); + let version_authorization = + version_state_precondition.map(|state_precondition| PublishArtifactAuthorization { + source_entity_type: "FileVersion".to_string(), + source_entity_id: body.source_file_version_id.clone(), + state_precondition, + resource_attrs: version_resource_attrs.clone(), + }); + version_resource_attrs.extend(request_attrs); + if let Err(status) = require_resource_authorization( + &state, + authenticated, + ResourceAuthorization { + action: "publish_artifact", + resource_type: "FileVersion", + resource_id: &body.source_file_version_id, + resource_attrs: version_resource_attrs, + }, + ) { + return status.into_response(); + } + let stored_file_id = if let Some(file_id) = authoritative_source_file_id { + file_id + } else { + match state + .file_version_source_file_id(tenant, &body.source_file_version_id) + .await + { + Ok(file_id) => file_id, + Err(error) => { + return ( + axum::http::StatusCode::INTERNAL_SERVER_ERROR, + axum::Json(serde_json::json!({ + "error": { + "code": "FileVersionRelationshipReadFailed", + "message": error, + } + })), + ) + .into_response(); + } + } + }; + if stored_file_id != body.file_id { + return ( + axum::http::StatusCode::BAD_REQUEST, + axum::Json(serde_json::json!({ + "error": { + "code": "FileVersionFileMismatch", + "message": format!( + "FileVersion '{}' belongs to File '{}', not '{}'", + body.source_file_version_id, stored_file_id, body.file_id, + ), + } + })), + ) + .into_response(); + } + if let Err(status) = require_resource_authorization( + &state, + authenticated, + ResourceAuthorization { + action: "publish_artifact", + resource_type: "File", + resource_id: &body.file_id, + resource_attrs: file_resource_attrs, }, + ) { + return status.into_response(); + } + file_authorization + .into_iter() + .chain(version_authorization) + .collect() + }; + + let publish_request = PublishFileArtifactRequest { + file_id: body.file_id, + label: body.label, + owner_ref_type: body.owner_ref_type, + owner_ref_id: body.owner_ref_id, + source_file_version_id: body.source_file_version_id, + namespace: body.namespace, + }; + if let Err(error) = publish_request.validate() { + return ( + axum::http::StatusCode::BAD_REQUEST, + axum::Json(serde_json::json!({ + "error": { + "code": "InvalidPublishArtifactRequest", + "message": error, + } + })), ) + .into_response(); + } + + match state + .publish_file_artifact_authorized(tenant, publish_request, publish_authorizations) .await { Ok(artifact) => { @@ -203,15 +451,22 @@ pub(crate) async fn handle_publish_artifact( ) .into_response() } - Err(error) => ( - axum::http::StatusCode::INTERNAL_SERVER_ERROR, - axum::Json(serde_json::json!({ - "error": { - "code": "PublishArtifactFailed", - "message": error, - } - })), - ) - .into_response(), + Err(error) => { + let (status, code) = if error == PUBLISH_ARTIFACT_STALE_AUTHORIZATION { + (axum::http::StatusCode::CONFLICT, "ConcurrentModification") + } else { + ( + axum::http::StatusCode::INTERNAL_SERVER_ERROR, + "PublishArtifactFailed", + ) + }; + ( + status, + axum::Json(serde_json::json!({ + "error": { "code": code, "message": error } + })), + ) + .into_response() + } } } diff --git a/crates/temper-server/src/api/mod.rs b/crates/temper-server/src/api/mod.rs index 6f9499e61..80079cc2f 100644 --- a/crates/temper-server/src/api/mod.rs +++ b/crates/temper-server/src/api/mod.rs @@ -12,16 +12,21 @@ mod files; mod policies; mod repl; mod secrets; +mod spec_pin; +mod trajectory_analysis; use axum::Router; use axum::extract::{FromRequestParts, Path, State}; +use axum::http::StatusCode; use axum::http::request::Parts; -use axum::http::{HeaderMap, StatusCode}; use axum::response::IntoResponse; use axum::routing::{get, patch, post, put}; -use temper_authz::PrincipalKind; +use temper_authz::{AuthenticatedRequestContext, SecurityContext}; +use temper_runtime::tenant::TenantId; -use crate::authz::{DenialInput, record_authz_denial, security_context_from_headers}; +use crate::authz::{ + DenialInput, record_authz_denial, require_authenticated_context, require_tenant_match, +}; use crate::state::ServerState; /// Build the management API router (mounted at /api). @@ -40,6 +45,8 @@ use crate::state::ServerState; /// - POST /api/files/read-text-batch -> batch current-file text reads via projections + blobs /// - POST /api/files/read-version-text-batch -> batch immutable file-version text reads /// - POST /api/files/publish-artifact -> promote a governed file to a public immutable artifact +/// - GET /api/ots/trajectories/{id}/atif -> export an OTS trajectory as ATIF v1.7 +/// - POST /api/conformance/check -> check a session against its actor spec pub fn build_api_router() -> Router { Router::new() .route( @@ -97,6 +104,15 @@ pub fn build_api_router() -> Router { post(crate::observe::evolution::handle_post_ots_trajectory) .get(crate::observe::evolution::handle_get_ots_trajectories), ) + .route( + "/ots/trajectories/{trajectory_id}/atif", + get(trajectory_analysis::handle_get_ots_trajectory_atif), + ) + // Deterministic conformance checking of a recorded run against its spec + .route( + "/conformance/check", + post(trajectory_analysis::handle_conformance_check), + ) .route( "/tenants/{tenant}/secrets/{key_name}", put(secrets::handle_put_secret).delete(secrets::handle_delete_secret), @@ -174,24 +190,27 @@ pub fn build_api_router() -> Router { /// Authorize a policy management request against Cedar policies. /// /// Returns `Some(response)` if authorization is denied, `None` if allowed. -/// Admin principals always bypass Cedar for policy management. pub(crate) async fn require_policy_auth( state: &ServerState, - headers: &HeaderMap, - tenant: &str, + authenticated: &AuthenticatedRequestContext, ) -> Option { - let security_ctx = security_context_from_headers(headers, None, None, None); - if matches!(security_ctx.principal.kind, PrincipalKind::Admin) { - // Admin principals (e.g. Observe UI) always bypass Cedar for policy - // management. Without this, approving the first policy would lock out - // the admin from managing subsequent decisions. - return None; - } + let security_ctx = authenticated.security_context(); + let tenant = authenticated.tenant().as_str(); + let resource_attrs = std::collections::BTreeMap::from([ + ( + "id".to_string(), + serde_json::Value::String(tenant.to_string()), + ), + ( + "tenant".to_string(), + serde_json::Value::String(tenant.to_string()), + ), + ]); if let Err(denial) = state.authorize_with_context( - &security_ctx, + security_ctx, "manage_policies", "PolicySet", - &std::collections::BTreeMap::new(), + &resource_attrs, tenant, ) { let reason = denial.to_string(); @@ -199,7 +218,7 @@ pub(crate) async fn require_policy_auth( state, DenialInput { tenant, - security_ctx: &security_ctx, + security_ctx, agent_id_override: None, action: "manage_policies", resource_type: "PolicySet", @@ -208,6 +227,10 @@ pub(crate) async fn require_policy_auth( reason: &reason, module_name: None, from_status: None, + intent: authenticated.intent().map(str::to_string), + session_id: authenticated.session_id().map(str::to_string), + // Management-plane denial, not a spec-governed dispatch. + spec_governed: Some(false), }, ) .await; @@ -234,7 +257,19 @@ pub(crate) async fn require_policy_auth( /// produces (403 + `AuthorizationDenied` JSON including the decision id). /// The tenant is read from the request parts by name, so handlers keep their /// own `Path` / `Path<(String, String)>` extractors untouched. -pub(crate) struct PolicyAuthed; +pub(crate) struct PolicyAuthed(AuthenticatedRequestContext); + +impl PolicyAuthed { + /// Credential-bound tenant authorized for this policy operation. + pub(crate) fn tenant(&self) -> &TenantId { + self.0.tenant() + } + + /// Credential-derived Cedar principal authorized for this operation. + pub(crate) fn security_context(&self) -> &SecurityContext { + self.0.security_context() + } +} impl FromRequestParts for PolicyAuthed { type Rejection = axum::response::Response; @@ -252,9 +287,13 @@ impl FromRequestParts for PolicyAuthed { // {tenant} path parameter; reaching this branch is a routing bug. return Err(StatusCode::INTERNAL_SERVER_ERROR.into_response()); }; - match require_policy_auth(state, &parts.headers, tenant).await { + let authenticated = + require_authenticated_context(parts.extensions.get::()) + .map_err(IntoResponse::into_response)?; + require_tenant_match(authenticated, tenant).map_err(IntoResponse::into_response)?; + match require_policy_auth(state, authenticated).await { Some(resp) => Err(resp), - None => Ok(Self), + None => Ok(Self(authenticated.clone())), } } } @@ -262,11 +301,12 @@ impl FromRequestParts for PolicyAuthed { /// GET /api/tenants/{tenant}/policies/suggestions — suggested policies from denial patterns. async fn handle_policy_suggestions( State(state): State, - Path(tenant): Path, - _auth: PolicyAuthed, + Path(_tenant): Path, + auth: PolicyAuthed, ) -> impl IntoResponse { - let suggestions = if let Some(store) = state.metadata_store_for_tenant(&tenant).await { - match store.load_policy_denial_patterns(&tenant).await { + let tenant = auth.tenant().as_str(); + let suggestions = if let Some(store) = state.metadata_store_for_tenant(tenant).await { + match store.load_policy_denial_patterns(tenant).await { Ok(rows) if !rows.is_empty() => { let mut engine = crate::state::policy_suggestions::PolicySuggestionEngine::new(); for row in rows { diff --git a/crates/temper-server/src/api/policies.rs b/crates/temper-server/src/api/policies.rs index 1bbd8a165..fe0f665b4 100644 --- a/crates/temper-server/src/api/policies.rs +++ b/crates/temper-server/src/api/policies.rs @@ -2,44 +2,23 @@ //! //! Handles Cedar policy CRUD operations for tenants, including full replacement, //! incremental rule addition, individual policy listing/toggling/editing/deletion, -//! and cross-tenant policy views. +//! and credential-tenant policy views. -use axum::extract::{Path, State}; -use axum::http::{HeaderMap, StatusCode}; +use axum::extract::{Extension, Path, State}; +use axum::http::StatusCode; use axum::response::IntoResponse; +use temper_authz::AuthenticatedRequestContext; use tracing::instrument; use super::PolicyAuthed; -use crate::authz::{load_and_activate_tenant_policies, persist_and_activate_policy}; +use crate::authz::persist_and_activate_policy; use crate::state::ServerState; -use crate::storage::PolicyStoreRow; - -/// Derive a human-readable source label from a `policy_id`. -fn policy_source(policy_id: &str) -> &'static str { - if policy_id.starts_with("os-app:") { - "os-app" - } else if policy_id.starts_with("decision:") { - "decision" - } else if policy_id == "migrated-legacy" { - "migrated-legacy" - } else { - "manual" - } -} -/// Serialize a [`PolicyRow`] to a JSON value for API responses. -fn policy_row_to_json(row: &PolicyStoreRow) -> serde_json::Value { - serde_json::json!({ - "tenant": row.tenant, - "policy_id": row.policy_id, - "cedar_text": row.cedar_text, - "enabled": row.enabled, - "policy_hash": row.policy_hash, - "created_at": row.created_at, - "created_by": row.created_by, - "source": policy_source(&row.policy_id), - }) -} +mod support; +use support::{ + build_prospective_enabled_text, build_prospective_enabled_text_with_override, + policy_row_to_json, reload_tenant_from_store, +}; // --------------------------------------------------------------------------- // Existing endpoints (unchanged interface, kept for backward compatibility) @@ -51,9 +30,10 @@ fn policy_row_to_json(row: &PolicyStoreRow) -> serde_json::Value { #[instrument(skip_all, fields(tenant, otel.name = "GET /api/tenants/{tenant}/policies"))] pub(crate) async fn handle_get_policies( State(state): State, - Path(tenant): Path, - _auth: PolicyAuthed, + Path(_tenant): Path, + auth: PolicyAuthed, ) -> impl IntoResponse { + let tenant = auth.tenant().as_str().to_string(); let policies = state.tenant_policies.read().unwrap(); // ci-ok: infallible lock let text = policies.get(&tenant).cloned().unwrap_or_default(); ( @@ -69,10 +49,11 @@ pub(crate) async fn handle_get_policies( #[instrument(skip_all, fields(tenant, otel.name = "PUT /api/tenants/{tenant}/policies"))] pub(crate) async fn handle_put_policies( State(state): State, - Path(tenant): Path, - _auth: PolicyAuthed, + Path(_tenant): Path, + auth: PolicyAuthed, body: axum::body::Bytes, ) -> impl IntoResponse { + let tenant = auth.tenant().as_str().to_string(); let body_json: serde_json::Value = match serde_json::from_slice(&body) { Ok(v) => v, Err(e) => { @@ -102,7 +83,14 @@ pub(crate) async fn handle_put_policies( policies.insert(tenant.clone(), policy_text.clone()); } - persist_and_activate_policy(&state, &tenant, "primary", &policy_text, "api").await; + persist_and_activate_policy( + &state, + &tenant, + "primary", + &policy_text, + &auth.security_context().principal.id, + ) + .await; let _ = state .observe_refresh_tx @@ -121,10 +109,11 @@ pub(crate) async fn handle_put_policies( #[instrument(skip_all, fields(tenant, otel.name = "POST /api/tenants/{tenant}/policies/rules"))] pub(crate) async fn handle_add_policy_rule( State(state): State, - Path(tenant): Path, - _auth: PolicyAuthed, + Path(_tenant): Path, + auth: PolicyAuthed, body: axum::body::Bytes, ) -> impl IntoResponse { + let tenant = auth.tenant().as_str().to_string(); let body_json: serde_json::Value = match serde_json::from_slice(&body) { Ok(v) => v, Err(e) => { @@ -164,7 +153,14 @@ pub(crate) async fn handle_add_policy_rule( policies.insert(tenant.clone(), new_tenant_text.clone()); } - persist_and_activate_policy(&state, &tenant, "primary", &new_tenant_text, "api").await; + persist_and_activate_policy( + &state, + &tenant, + "primary", + &new_tenant_text, + &auth.security_context().principal.id, + ) + .await; let _ = state .observe_refresh_tx @@ -188,9 +184,10 @@ pub(crate) async fn handle_add_policy_rule( #[instrument(skip_all, fields(tenant, otel.name = "GET /api/tenants/{tenant}/policies/list"))] pub(crate) async fn handle_list_policies( State(state): State, - Path(tenant): Path, - _auth: PolicyAuthed, + Path(_tenant): Path, + auth: PolicyAuthed, ) -> impl IntoResponse { + let tenant = auth.tenant().as_str().to_string(); let Some(store) = state.policy_store() else { return ( StatusCode::SERVICE_UNAVAILABLE, @@ -227,16 +224,21 @@ pub(crate) async fn handle_list_policies( } } -/// GET /api/policies — list policies across all tenants (admin only). +/// GET /api/policies — list policies in the credential-bound tenant. #[instrument(skip_all, fields(otel.name = "GET /api/policies"))] pub(crate) async fn handle_list_all_policies( State(state): State, - headers: HeaderMap, + authenticated: Option>, ) -> impl IntoResponse { + let authenticated = match crate::authz::require_authenticated_context(authenticated.as_deref()) + { + Ok(authenticated) => authenticated, + Err(status) => return status.into_response(), + }; if let Err(status) = - crate::authz::require_observe_auth(&state, &headers, "manage_policies", "PolicySet") + crate::authz::require_observe_auth(&state, authenticated, "manage_policies", "PolicySet") { - return (status, "Authorization required for cross-tenant access").into_response(); + return (status, "Authorization required").into_response(); } let Some(store) = state.policy_store() else { @@ -247,7 +249,8 @@ pub(crate) async fn handle_list_all_policies( .into_response(); }; - let mut rows = match store.load_all_policies().await { + let tenant = authenticated.tenant().as_str(); + let mut rows = match store.load_policies_for_tenant(tenant).await { Ok(rows) => rows, Err(e) => { tracing::warn!(error = %e, "failed to list policies from durable store"); @@ -290,10 +293,11 @@ pub(crate) async fn handle_list_all_policies( #[instrument(skip_all, fields(tenant, otel.name = "POST /api/tenants/{tenant}/policies/create"))] pub(crate) async fn handle_create_policy( State(state): State, - Path(tenant): Path, - _auth: PolicyAuthed, + Path(_tenant): Path, + auth: PolicyAuthed, axum::Json(body): axum::Json, ) -> impl IntoResponse { + let tenant = auth.tenant().as_str().to_string(); let policy_id = match body.get("policy_id").and_then(|v| v.as_str()) { Some(v) if !v.is_empty() => v.to_string(), _ => { @@ -323,10 +327,7 @@ pub(crate) async fn handle_create_policy( } // Persist the new policy entry. - let created_by = body - .get("created_by") - .and_then(|v| v.as_str()) - .unwrap_or("api"); + let created_by = auth.security_context().principal.id.as_str(); persist_and_activate_policy(&state, &tenant, &policy_id, &cedar_text, created_by).await; // Update in-memory map to match the prospective text. @@ -358,10 +359,11 @@ pub(crate) async fn handle_create_policy( #[instrument(skip_all, fields(tenant, policy_id, otel.name = "PATCH /api/tenants/{tenant}/policies/entry/{policy_id}"))] pub(crate) async fn handle_patch_policy( State(state): State, - Path((tenant, policy_id)): Path<(String, String)>, - _auth: PolicyAuthed, + Path((_tenant, policy_id)): Path<(String, String)>, + auth: PolicyAuthed, axum::Json(body): axum::Json, ) -> impl IntoResponse { + let tenant = auth.tenant().as_str().to_string(); let Some(store) = state.policy_store() else { return ( StatusCode::SERVICE_UNAVAILABLE, @@ -396,10 +398,7 @@ pub(crate) async fn handle_patch_policy( return resp; } - let created_by = body - .get("created_by") - .and_then(|v| v.as_str()) - .unwrap_or("api"); + let created_by = auth.security_context().principal.id.as_str(); if let Err(e) = store .update_policy_text(&tenant, &policy_id, cedar_text, created_by) .await @@ -458,9 +457,10 @@ pub(crate) async fn handle_patch_policy( #[instrument(skip_all, fields(tenant, policy_id, otel.name = "DELETE /api/tenants/{tenant}/policies/entry/{policy_id}"))] pub(crate) async fn handle_delete_policy_entry( State(state): State, - Path((tenant, policy_id)): Path<(String, String)>, - _auth: PolicyAuthed, + Path((_tenant, policy_id)): Path<(String, String)>, + auth: PolicyAuthed, ) -> impl IntoResponse { + let tenant = auth.tenant().as_str().to_string(); let Some(store) = state.policy_store() else { return ( StatusCode::SERVICE_UNAVAILABLE, @@ -495,78 +495,3 @@ pub(crate) async fn handle_delete_policy_entry( ) .into_response() } - -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -/// Reload a tenant's in-memory policy state from durable storage. -/// -/// Reads all enabled policies, concatenates them, updates `tenant_policies`, -/// and reloads the Cedar engine. -async fn reload_tenant_from_store(state: &ServerState, tenant: &str) { - load_and_activate_tenant_policies(state, tenant).await; -} - -/// Build the prospective enabled policy text for a tenant, optionally including -/// a new policy entry that isn't persisted yet. -async fn build_prospective_enabled_text( - state: &ServerState, - tenant: &str, - additional: Option<(&str, &str)>, -) -> String { - let mut text = { - let policies = state.tenant_policies.read().unwrap(); // ci-ok: infallible lock - policies.get(tenant).cloned().unwrap_or_default() - }; - if let Some((_id, cedar_text)) = additional { - if !text.is_empty() { - text.push('\n'); - } - text.push_str(cedar_text); - } - text -} - -/// Build the prospective enabled policy text for a tenant, replacing one -/// specific policy entry's text and/or enabled state. -async fn build_prospective_enabled_text_with_override( - state: &ServerState, - tenant: &str, - override_policy_id: &str, - override_cedar_text: &str, - override_enabled: Option, -) -> String { - // Load all current policies from durable storage to get accurate per-entry data. - let rows = if let Some(store) = state.policy_store() { - store - .load_policies_for_tenant(tenant) - .await - .unwrap_or_default() - } else { - vec![] - }; - - let mut combined = String::new(); - for row in &rows { - let is_target = row.policy_id == override_policy_id; - let cedar_text = if is_target { - override_cedar_text - } else { - &row.cedar_text - }; - let enabled = if is_target { - override_enabled.unwrap_or(row.enabled) - } else { - row.enabled - }; - if !enabled { - continue; - } - if !combined.is_empty() { - combined.push('\n'); - } - combined.push_str(cedar_text); - } - combined -} diff --git a/crates/temper-server/src/api/policies/support.rs b/crates/temper-server/src/api/policies/support.rs new file mode 100644 index 000000000..d60b56311 --- /dev/null +++ b/crates/temper-server/src/api/policies/support.rs @@ -0,0 +1,90 @@ +use crate::authz::load_and_activate_tenant_policies; +use crate::state::ServerState; +use crate::storage::PolicyStoreRow; + +pub(super) fn policy_row_to_json(row: &PolicyStoreRow) -> serde_json::Value { + serde_json::json!({ + "tenant": row.tenant, + "policy_id": row.policy_id, + "cedar_text": row.cedar_text, + "enabled": row.enabled, + "policy_hash": row.policy_hash, + "created_at": row.created_at, + "created_by": row.created_by, + "source": policy_source(&row.policy_id), + }) +} + +fn policy_source(policy_id: &str) -> &'static str { + if policy_id.starts_with("os-app:") { + "os-app" + } else if policy_id.starts_with("decision:") { + "decision" + } else if policy_id == "migrated-legacy" { + "migrated-legacy" + } else { + "manual" + } +} + +pub(super) async fn reload_tenant_from_store(state: &ServerState, tenant: &str) { + load_and_activate_tenant_policies(state, tenant).await; +} + +pub(super) async fn build_prospective_enabled_text( + state: &ServerState, + tenant: &str, + additional: Option<(&str, &str)>, +) -> String { + let mut text = { + let policies = state.tenant_policies.read().unwrap(); // ci-ok: infallible lock + policies.get(tenant).cloned().unwrap_or_default() + }; + if let Some((_id, cedar_text)) = additional { + if !text.is_empty() { + text.push('\n'); + } + text.push_str(cedar_text); + } + text +} + +pub(super) async fn build_prospective_enabled_text_with_override( + state: &ServerState, + tenant: &str, + override_policy_id: &str, + override_cedar_text: &str, + override_enabled: Option, +) -> String { + let rows = if let Some(store) = state.policy_store() { + store + .load_policies_for_tenant(tenant) + .await + .unwrap_or_default() + } else { + vec![] + }; + + let mut combined = String::new(); + for row in &rows { + let is_target = row.policy_id == override_policy_id; + let cedar_text = if is_target { + override_cedar_text + } else { + &row.cedar_text + }; + let enabled = if is_target { + override_enabled.unwrap_or(row.enabled) + } else { + row.enabled + }; + if !enabled { + continue; + } + if !combined.is_empty() { + combined.push('\n'); + } + combined.push_str(cedar_text); + } + combined +} diff --git a/crates/temper-server/src/api/repl.rs b/crates/temper-server/src/api/repl.rs index 0518647a6..56310a85f 100644 --- a/crates/temper-server/src/api/repl.rs +++ b/crates/temper-server/src/api/repl.rs @@ -3,66 +3,138 @@ //! Exposes the Temper Monty sandbox over HTTP, allowing agents to execute //! Python code with `temper.*` methods that loop back to the server. -use axum::extract::State; +use std::sync::Arc; + +use axum::extract::{Extension, State}; use axum::http::{HeaderMap, StatusCode}; use axum::response::IntoResponse; +use temper_authz::AuthenticatedRequestContext; use temper_runtime::scheduler::sim_now; use tracing::instrument; -use crate::odata::extract_tenant; +use crate::authz::require_authenticated_context; use crate::state::{ServerState, TrajectoryEntry, TrajectorySource}; +fn require_repl_authorization( + state: &ServerState, + authenticated: &AuthenticatedRequestContext, +) -> Result<(), StatusCode> { + state + .authorize_with_context( + authenticated.security_context(), + "execute_repl", + "Repl", + &std::collections::BTreeMap::from([( + "id".to_string(), + serde_json::Value::String(authenticated.tenant().to_string()), + )]), + authenticated.tenant().as_str(), + ) + .map_err(|denial| { + tracing::warn!( + reason = %denial, + tenant = %authenticated.tenant(), + principal_id = %authenticated.security_context().principal.id, + "REPL execution denied" + ); + StatusCode::FORBIDDEN + }) +} + /// Request body for POST /api/repl. #[derive(serde::Deserialize)] pub(crate) struct ReplRequest { code: String, } +/// Build the `ReplConfig` for the server-hosted REPL. +/// +/// `allow_host_ops` is hardcoded false here and only here: the server-hosted +/// REPL's host process is the Temper server, so `upload_wasm`/`compile_wasm` +/// would read server files and spawn `cargo` as the server user (ARN-166, +/// ADR-0158). Host ops belong to the local MCP server on the developer's own +/// machine. Centralised in one constructor so the invariant has a single tested +/// home rather than an inline literal that a refactor could flip unnoticed. +fn server_repl_config( + server_port: u16, + tenant: String, + agent_id: String, + session_id: Option, + internal_credential_issuer: temper_sandbox::http::InternalRequestCredentialIssuer, +) -> temper_sandbox::repl::ReplConfig { + temper_sandbox::repl::ReplConfig { + server_port, + tenant, + agent_id: Some(agent_id), + session_id, + internal_credential_issuer, + allow_host_ops: false, + } +} + /// POST /api/repl — execute Python code in the Temper Monty sandbox. /// /// The sandbox provides `temper.*` methods (create, action, submit_specs, etc.) -/// that loop back to this server via HTTP. Agent identity is extracted from -/// `X-Temper-Principal-Id` / `X-Temper-Principal-Kind` / `X-Temper-Agent-Role` -/// headers and forwarded on internal requests. +/// that loop back to this server via single-use, request-bound credentials. /// /// Security: 180s timeout, 64MB memory, method allowlisting, no filesystem or -/// network access. External APIs go through `[[integration]]` in IOA specs. +/// network access. Host ops (`upload_wasm`/`compile_wasm`) are gated off via +/// `server_repl_config` (ARN-166). External APIs go through `[[integration]]` +/// in IOA specs. #[instrument(skip_all, fields(otel.name = "POST /api/repl"))] pub(crate) async fn handle_repl( State(state): State, headers: HeaderMap, + authenticated: Option>, axum::Json(body): axum::Json, ) -> impl IntoResponse { - let principal_id = headers - .get("x-temper-principal-id") - .and_then(|v| v.to_str().ok()) - .map(String::from); - let principal_kind = headers - .get("x-temper-principal-kind") - .and_then(|v| v.to_str().ok()) - .map(String::from); - let agent_role = headers - .get("x-temper-agent-role") - .and_then(|v| v.to_str().ok()) - .map(String::from); - let agent_type = headers - .get("x-temper-agent-type") - .and_then(|v| v.to_str().ok()) - .map(String::from); - let session_id = headers - .get("x-session-id") - .and_then(|v| v.to_str().ok()) - .map(String::from); - - let tenant = match extract_tenant(&headers, &state) { - Ok(t) => t.as_str().to_string(), - Err(e) => return e.into_response(), + let authenticated = match require_authenticated_context(authenticated.as_deref()) { + Ok(authenticated) => authenticated, + Err(status) => return status.into_response(), + }; + if let Err(status) = require_repl_authorization(&state, authenticated) { + return status.into_response(); + } + // Session and intent come from the canonical observability extractor so the + // REPL honours the same header aliases as every other entrypoint + // (`X-Temper-Observe-Session-Id`/`X-Session-Id`, + // `X-Temper-Observe-Intent`/`X-Intent`). These are correlation-only and + // carry no authority; identity comes from the credential above. + let agent_ctx = crate::request_context::extract_agent_context(&headers); + let session_id = agent_ctx.session_id.clone(); + + let tenant = authenticated.tenant().to_string(); + let agent_type = authenticated + .security_context() + .principal + .agent_type + .clone(); + let Some(capability_issuer) = crate::state::internal_http_capability_issuer( + &state, + authenticated.tenant(), + Some(authenticated.security_context()), + ) else { + tracing::error!("authenticated REPL request has no internal capability issuer"); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); }; + let internal_credential_issuer: temper_sandbox::http::InternalRequestCredentialIssuer = + Arc::new(move |method, url| { + let capability = capability_issuer(method, url)?; + temper_sandbox::http::InternalRequestCredential::new( + capability.bearer_token().to_string(), + capability.tenant().to_string(), + ) + }); - let agent_id = principal_id.clone(); + let agent_id = authenticated.security_context().principal.id.clone(); + let trajectory_agent_type = agent_type.clone(); let port = state.listen_port.get().copied().unwrap_or(4200); let code = body.code; + // The submitted code is the request body of a REPL call; a failed run is + // unreadable without it. Size is bounded when the entry is enqueued. + let submitted_code = code.clone(); let tenant_for_repl = tenant.clone(); + let agent_id_for_repl = agent_id.clone(); // The Monty sandbox types are !Send, so we run in a dedicated // single-threaded runtime via spawn_blocking. @@ -72,16 +144,13 @@ pub(crate) async fn handle_repl( .build() .expect("failed to create REPL runtime"); // determinism-ok: one-shot runtime for sandbox rt.block_on(async move { - let config = temper_sandbox::repl::ReplConfig { - server_port: port, - tenant: tenant_for_repl, - agent_id: principal_id.clone(), - principal_id, - principal_kind, - agent_role, - agent_type, + let config = server_repl_config( + port, + tenant_for_repl, + agent_id_for_repl, session_id, - }; + internal_credential_issuer, + ); temper_sandbox::repl::run_repl(&config, &code).await }) }) @@ -113,17 +182,18 @@ pub(crate) async fn handle_repl( from_status: None, to_status: None, error: Some(e.to_string()), - agent_id: agent_id.clone(), - session_id: None, + agent_id: Some(agent_id.clone()), + session_id: agent_ctx.session_id.clone(), authz_denied: None, denied_resource: None, denied_module: None, source: Some(TrajectorySource::Platform), spec_governed: None, - agent_type: None, - request_body: None, - intent: None, + agent_type: trajectory_agent_type.clone(), + request_body: Some(serde_json::json!({ "code": submitted_code })), + intent: agent_ctx.intent.clone(), matched_policy_ids: None, + capture_seq: None, }; if !state.enqueue_trajectory_entry(entry) { tracing::warn!("failed to enqueue REPL trajectory entry"); @@ -151,3 +221,87 @@ pub(crate) async fn handle_repl( } } } + +#[cfg(test)] +mod tests { + use temper_authz::{AuthenticatedRequestContext, SecurityContext}; + use temper_runtime::ActorSystem; + use temper_runtime::tenant::TenantId; + + use super::require_repl_authorization; + use crate::registry::SpecRegistry; + use crate::state::ServerState; + + use super::server_repl_config; + + /// The server-hosted REPL must never permit host ops (ARN-166). This guards + /// the exact production constructor, so a future flip of the `allow_host_ops` + /// literal is caught here rather than reopening the RCE silently. + #[test] + fn server_repl_config_never_allows_host_ops() { + let issuer: temper_sandbox::http::InternalRequestCredentialIssuer = + std::sync::Arc::new(|_method, _url| Err("unused".to_string())); + let config = server_repl_config( + 3000, + "tenant-a".to_string(), + "agent-1".to_string(), + Some("session-1".to_string()), + issuer, + ); + assert!( + !config.allow_host_ops, + "the server-hosted REPL must not permit host-process ops" + ); + } + + #[test] + fn repl_requires_explicit_tenant_resource_authority() { + let state = ServerState::from_registry(ActorSystem::new("repl-auth"), SpecRegistry::new()); + state + .authz + .reload_tenant_policies( + "tenant-a", + r#" +permit( + principal == Agent::"agent-1", + action == Action::"execute_repl", + resource == Repl::"tenant-a" +); +"#, + ) + .expect("REPL policy should parse"); + let allowed = AuthenticatedRequestContext::new( + TenantId::new("tenant-a"), + SecurityContext::from_resolved_identity("agent-1", "worker", None), + ); + let denied = AuthenticatedRequestContext::new( + TenantId::new("tenant-a"), + SecurityContext::from_resolved_identity("agent-2", "worker", None), + ); + let claimed_admin = AuthenticatedRequestContext::new( + TenantId::new("tenant-a"), + SecurityContext { + principal: temper_authz::Principal { + id: "claimed-admin".to_string(), + kind: temper_authz::PrincipalKind::Admin, + role: None, + acting_for: None, + agent_type: None, + attributes: Default::default(), + }, + context_attrs: Default::default(), + correlation_id: "repl-admin-side-channel-test".to_string(), + }, + ); + + assert!(require_repl_authorization(&state, &allowed).is_ok()); + assert_eq!( + require_repl_authorization(&state, &denied), + Err(axum::http::StatusCode::FORBIDDEN) + ); + assert_eq!( + require_repl_authorization(&state, &claimed_admin), + Err(axum::http::StatusCode::FORBIDDEN) + ); + } +} diff --git a/crates/temper-server/src/api/secrets.rs b/crates/temper-server/src/api/secrets.rs index f7a9687b3..50acd5646 100644 --- a/crates/temper-server/src/api/secrets.rs +++ b/crates/temper-server/src/api/secrets.rs @@ -3,14 +3,77 @@ //! Handles encrypted secret storage, retrieval, and deletion for tenants. //! Secrets are encrypted at rest using the configured vault key. -use axum::extract::{Path, State}; +use std::collections::BTreeMap; + +use axum::extract::{FromRequestParts, Path, State}; use axum::http::StatusCode; +use axum::http::request::Parts; use axum::response::IntoResponse; +use temper_authz::AuthenticatedRequestContext; +use temper_runtime::tenant::TenantId; use tracing::instrument; -use super::PolicyAuthed; +use crate::authz::{ + ResourceAuthorization, require_authenticated_context, require_resource_authorization, + require_tenant_match, +}; use crate::state::ServerState; +const SECRET_COLLECTION_RESOURCE_ID: &str = "__keys__"; + +pub(super) struct SecretAuthed(AuthenticatedRequestContext); + +impl SecretAuthed { + fn tenant(&self) -> &TenantId { + self.0.tenant() + } +} + +impl FromRequestParts for SecretAuthed { + type Rejection = axum::response::Response; + + async fn from_request_parts( + parts: &mut Parts, + state: &ServerState, + ) -> Result { + let Path(params) = Path::>::from_request_parts(parts, state) + .await + .map_err(IntoResponse::into_response)?; + let Some(tenant) = params.get("tenant") else { + return Err(StatusCode::INTERNAL_SERVER_ERROR.into_response()); + }; + let authenticated = + require_authenticated_context(parts.extensions.get::()) + .map_err(IntoResponse::into_response)?; + require_tenant_match(authenticated, tenant).map_err(IntoResponse::into_response)?; + let resource_id = params + .get("key_name") + .map(String::as_str) + .unwrap_or(SECRET_COLLECTION_RESOURCE_ID); + require_resource_authorization( + state, + authenticated, + ResourceAuthorization { + action: "manage_secrets", + resource_type: "Secret", + resource_id, + resource_attrs: BTreeMap::from([ + ( + "tenant".to_string(), + serde_json::Value::String(tenant.clone()), + ), + ( + "key_name".to_string(), + serde_json::Value::String(resource_id.to_string()), + ), + ]), + }, + ) + .map_err(IntoResponse::into_response)?; + Ok(Self(authenticated.clone())) + } +} + /// Check if an error message indicates that the backend is not supported. fn is_backend_not_supported_error(err: &str) -> bool { err.to_ascii_lowercase().contains("not supported") @@ -18,12 +81,13 @@ fn is_backend_not_supported_error(err: &str) -> bool { /// PUT /api/tenants/{tenant}/secrets/{key_name} — encrypt and store a secret. #[instrument(skip_all, fields(tenant, key_name, otel.name = "PUT /api/tenants/{tenant}/secrets/{key_name}"))] -pub(crate) async fn handle_put_secret( +pub(super) async fn handle_put_secret( State(state): State, - Path((tenant, key_name)): Path<(String, String)>, - _auth: PolicyAuthed, + Path((_tenant, key_name)): Path<(String, String)>, + auth: SecretAuthed, body: axum::body::Bytes, ) -> impl IntoResponse { + let tenant = auth.tenant().as_str(); let Some(vault) = state.secrets_vault.as_ref() else { tracing::warn!("secrets vault not configured"); return ( @@ -80,7 +144,7 @@ pub(crate) async fn handle_put_secret( // Persist the encrypted secret. if let Err(e) = state - .upsert_secret(&tenant, &key_name, &ciphertext, &nonce) + .upsert_secret(tenant, &key_name, &ciphertext, &nonce) .await { tracing::error!(error = %e, "secret persistence failed"); @@ -92,9 +156,9 @@ pub(crate) async fn handle_put_secret( } // Cache in memory after successful persistence. - if let Err(e) = vault.cache_secret(&tenant, &key_name, value.to_string()) { + if let Err(e) = vault.cache_secret(tenant, &key_name, value.to_string()) { // Best-effort rollback to keep storage/cache aligned. - let _ = state.delete_secret(&tenant, &key_name).await; + let _ = state.delete_secret(tenant, &key_name).await; tracing::error!(error = %e, "cache update failed after persistence write"); return ( StatusCode::CONFLICT, @@ -108,11 +172,12 @@ pub(crate) async fn handle_put_secret( /// DELETE /api/tenants/{tenant}/secrets/{key_name} — remove a secret. #[instrument(skip_all, fields(tenant, key_name, otel.name = "DELETE /api/tenants/{tenant}/secrets/{key_name}"))] -pub(crate) async fn handle_delete_secret( +pub(super) async fn handle_delete_secret( State(state): State, - Path((tenant, key_name)): Path<(String, String)>, - _auth: PolicyAuthed, + Path((_tenant, key_name)): Path<(String, String)>, + auth: SecretAuthed, ) -> impl IntoResponse { + let tenant = auth.tenant().as_str(); let Some(vault) = state.secrets_vault.as_ref() else { tracing::warn!("secrets vault not configured"); return ( @@ -122,14 +187,14 @@ pub(crate) async fn handle_delete_secret( .into_response(); }; - match state.delete_secret(&tenant, &key_name).await { + match state.delete_secret(tenant, &key_name).await { Ok(true) => { - vault.remove_secret(&tenant, &key_name); + vault.remove_secret(tenant, &key_name); StatusCode::NO_CONTENT.into_response() } Ok(false) => { tracing::warn!("secret not found for deletion"); - vault.remove_secret(&tenant, &key_name); + vault.remove_secret(tenant, &key_name); StatusCode::NOT_FOUND.into_response() } Err(e) => { @@ -146,11 +211,12 @@ pub(crate) async fn handle_delete_secret( /// GET /api/tenants/{tenant}/secrets — list secret key names (never values). #[instrument(skip_all, fields(tenant, otel.name = "GET /api/tenants/{tenant}/secrets"))] -pub(crate) async fn handle_list_secrets( +pub(super) async fn handle_list_secrets( State(state): State, - Path(tenant): Path, - _auth: PolicyAuthed, + Path(_tenant): Path, + auth: SecretAuthed, ) -> impl IntoResponse { + let tenant = auth.tenant().as_str(); let Some(vault) = state.secrets_vault.as_ref() else { tracing::warn!("secrets vault not configured"); return ( @@ -160,7 +226,7 @@ pub(crate) async fn handle_list_secrets( .into_response(); }; - let keys = vault.list_keys(&tenant); + let keys = vault.list_keys(tenant); ( StatusCode::OK, axum::Json(serde_json::json!({"keys": keys})), diff --git a/crates/temper-server/src/api/spec_pin.rs b/crates/temper-server/src/api/spec_pin.rs new file mode 100644 index 000000000..183b97eaa --- /dev/null +++ b/crates/temper-server/src/api/spec_pin.rs @@ -0,0 +1,170 @@ +//! How a declared spec version names a registered spec. +//! +//! A conformance report only means something against the spec the run executed +//! under, so the version a run declares has to be matched against the one the +//! kernel has registered. Harnesses write that version in more than one form: +//! +//! - `0f1e2d…` — the bare content hash, 64 hex characters; +//! - `sha256:0f1e2d…` — the same hash, algorithm-qualified; +//! - `Order@sha256:0f1e2d3c4b5a` — the kernel's own vocabulary: the entity the +//! spec governs, then a digest that may be **truncated**, the way a short +//! object id names a commit. +//! +//! The first two are matched exactly, as they always were. The third is the +//! addition: the entity qualifier selects the spec — Temper registers exactly +//! one spec per (tenant, entity type) — and the digest then has to prefix that +//! spec's hash. +//! +//! # Why a truncated digest has to carry the entity +//! +//! Without it a short digest names nothing in particular: it would have to be +//! searched for across every spec in the tenant, and a collision would judge a +//! run by another actor's rules. With it there is only ever one spec for the +//! digest to agree with, so a prefix identifies as precisely as the full hash +//! does. A bare short digest is therefore refused rather than searched. +//! +//! # What a prefix does not fix +//! +//! It does not make a re-serialized spec match. sha256 avalanches: normalising +//! line endings or re-emitting the TOML changes the whole digest, its first +//! twelve characters included, so a producer that recomputes the hash from a +//! rewritten file disagrees at any length. The cure for that is for producers +//! to read the registered digest off `GET /observe/specs/{entity}` rather than +//! computing one themselves. + +/// Shortest digest accepted in an entity-qualified pin, in hex characters. +/// +/// Twelve hex characters is 48 bits — the length git settled on for short +/// object ids in large repositories, and far past the point where two specs in +/// one tenant collide by accident. +pub(crate) const MIN_PIN_DIGEST_HEX: usize = 12; + +/// Length of a full sha256 digest in hex characters. +const SHA256_HEX_LEN: usize = 64; + +/// What a declared version names. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum PinMatch { + /// The registered spec. + Registered, + /// A spec, but not the registered one. + OtherVersion, + /// A spec belonging to a different entity than the one being checked. + WrongEntity, + /// Nothing in particular: the digest is truncated past the point where it + /// identifies one spec. + DigestTooShort, +} + +/// A spec version as a harness or a request wrote it. +struct DeclaredVersion<'a> { + /// The entity the pin names, when it is qualified with one. + entity: Option<&'a str>, + /// The digest as written: a full hash, or a prefix of one. + digest: &'a str, +} + +impl<'a> DeclaredVersion<'a> { + fn parse(declared: &'a str) -> Self { + let (entity, rest) = match declared.split_once('@') { + Some((entity, rest)) => (Some(entity.trim()), rest), + None => (None, declared), + }; + let rest = rest.trim(); + Self { + entity, + digest: rest.strip_prefix("sha256:").unwrap_or(rest), + } + } + + fn is_hex(&self) -> bool { + !self.digest.is_empty() && self.digest.chars().all(|c| c.is_ascii_hexdigit()) + } +} + +/// Decide what `declared` names, given the spec registered for `entity_type`. +pub(crate) fn classify_pin(declared: &str, entity_type: &str, registered_hash: &str) -> PinMatch { + let declared = DeclaredVersion::parse(declared); + + let Some(entity) = declared.entity else { + // Unqualified: matched exactly, the way it always was. A bare digest + // that happens to prefix the registered hash is not accepted — nothing + // in it says the author meant a prefix rather than a different spec. + return if declared.digest == registered_hash { + PinMatch::Registered + } else { + PinMatch::OtherVersion + }; + }; + + // The qualifier selects the spec before the digest is read, so a pin for + // another actor is refused on its own terms rather than reported as this + // actor having run under some other version. + if entity != entity_type { + return PinMatch::WrongEntity; + } + if !declared.is_hex() { + // A qualified pin carrying something that is not a digest at all — a + // tag, a release name — names a version this kernel cannot resolve. + return PinMatch::OtherVersion; + } + if declared.digest.len() < MIN_PIN_DIGEST_HEX { + return PinMatch::DigestTooShort; + } + if declared.digest.len() > SHA256_HEX_LEN { + return PinMatch::OtherVersion; + } + if hex_prefix_of(registered_hash, declared.digest) { + PinMatch::Registered + } else { + PinMatch::OtherVersion + } +} + +/// Whether two declarations name the same spec. +/// +/// Two spellings agree when they are the same version written two ways, or +/// when both resolve to the registered spec. The written comparison runs +/// first, so two spellings of a version that is no longer registered still +/// read as agreement rather than as the request contradicting the run. +pub(crate) fn declare_same_spec( + left: &str, + right: &str, + entity_type: &str, + registered_hash: &str, +) -> bool { + if same_written_version( + &DeclaredVersion::parse(left), + &DeclaredVersion::parse(right), + ) { + return true; + } + classify_pin(left, entity_type, registered_hash) == PinMatch::Registered + && classify_pin(right, entity_type, registered_hash) == PinMatch::Registered +} + +/// Whether two declarations are one version written two ways. +/// +/// `sha256:0f1e…` and a bare `0f1e…` are one version, and so is the same +/// digest with and without an entity qualifier — the digest is what identifies +/// a spec, and a qualifier present on only one side says nothing against it. +fn same_written_version(left: &DeclaredVersion<'_>, right: &DeclaredVersion<'_>) -> bool { + if left.digest != right.digest { + return false; + } + match (left.entity, right.entity) { + (Some(left), Some(right)) => left == right, + _ => true, + } +} + +/// Whether `prefix` is a leading run of `full`, comparing hex without regard +/// to case — a digest is the same value however a producer cased it. +fn hex_prefix_of(full: &str, prefix: &str) -> bool { + full.len() >= prefix.len() + && full.as_bytes()[..prefix.len()].eq_ignore_ascii_case(prefix.as_bytes()) +} + +#[cfg(test)] +#[path = "spec_pin_test.rs"] +mod spec_pin_test; diff --git a/crates/temper-server/src/api/spec_pin_test.rs b/crates/temper-server/src/api/spec_pin_test.rs new file mode 100644 index 000000000..a8e003fb5 --- /dev/null +++ b/crates/temper-server/src/api/spec_pin_test.rs @@ -0,0 +1,192 @@ +//! Which spec a declared version names. + +use super::*; + +/// A stand-in for a registered content hash: 64 lowercase hex characters. +const REGISTERED: &str = "0f1e2d3c4b5a69788796a5b4c3d2e1f00f1e2d3c4b5a69788796a5b4c3d2e1f0"; + +/// Another spec's hash, sharing no prefix with [`REGISTERED`]. +const OTHER: &str = "abcdef01234567890abcdef01234567890abcdef01234567890abcdef0123456"; + +fn classify(declared: &str) -> PinMatch { + classify_pin(declared, "Order", REGISTERED) +} + +#[test] +fn the_bare_hash_names_the_registered_spec() { + assert_eq!(classify(REGISTERED), PinMatch::Registered); +} + +#[test] +fn the_algorithm_qualified_hash_names_the_registered_spec() { + assert_eq!( + classify(&format!("sha256:{REGISTERED}")), + PinMatch::Registered + ); +} + +#[test] +fn another_spec_s_hash_names_another_version() { + assert_eq!(classify(OTHER), PinMatch::OtherVersion); + assert_eq!(classify("sha256:v1-long-gone"), PinMatch::OtherVersion); +} + +#[test] +fn an_entity_qualified_prefix_names_the_registered_spec() { + // The pin katagami stamps: the entity, then a short digest. + let prefix = ®ISTERED[..MIN_PIN_DIGEST_HEX]; + assert_eq!( + classify(&format!("Order@sha256:{prefix}")), + PinMatch::Registered + ); +} + +#[test] +fn an_entity_qualified_full_hash_names_the_registered_spec() { + assert_eq!( + classify(&format!("Order@sha256:{REGISTERED}")), + PinMatch::Registered + ); +} + +#[test] +fn a_prefix_is_matched_without_regard_to_case() { + let prefix = REGISTERED[..20].to_ascii_uppercase(); + assert_eq!( + classify(&format!("Order@sha256:{prefix}")), + PinMatch::Registered + ); +} + +#[test] +fn an_entity_qualified_prefix_of_another_spec_names_another_version() { + let prefix = &OTHER[..MIN_PIN_DIGEST_HEX]; + assert_eq!( + classify(&format!("Order@sha256:{prefix}")), + PinMatch::OtherVersion + ); +} + +#[test] +fn a_digest_truncated_past_the_minimum_names_nothing() { + // The ambiguity guard: below the floor a digest stops identifying one + // spec, so it is refused rather than resolved to whatever it prefixes. + for length in 1..MIN_PIN_DIGEST_HEX { + let prefix = ®ISTERED[..length]; + assert_eq!( + classify(&format!("Order@sha256:{prefix}")), + PinMatch::DigestTooShort, + "a {length}-character digest must be refused, not resolved" + ); + } +} + +#[test] +fn the_shortest_accepted_digest_is_exactly_the_floor() { + let at_floor = ®ISTERED[..MIN_PIN_DIGEST_HEX]; + let below_floor = ®ISTERED[..MIN_PIN_DIGEST_HEX - 1]; + assert_eq!( + classify(&format!("Order@sha256:{at_floor}")), + PinMatch::Registered + ); + assert_eq!( + classify(&format!("Order@sha256:{below_floor}")), + PinMatch::DigestTooShort + ); +} + +#[test] +fn a_bare_prefix_is_not_accepted_as_a_prefix() { + // Nothing in an unqualified digest says its author meant a prefix rather + // than a different spec, so it stays an exact comparison. + let prefix = ®ISTERED[..MIN_PIN_DIGEST_HEX]; + assert_eq!(classify(prefix), PinMatch::OtherVersion); + assert_eq!( + classify(&format!("sha256:{prefix}")), + PinMatch::OtherVersion + ); +} + +#[test] +fn a_pin_for_another_entity_is_refused_on_its_own_terms() { + // Same digest, wrong actor: reported as the wrong entity rather than as + // this actor having run under some other version. + let prefix = ®ISTERED[..MIN_PIN_DIGEST_HEX]; + assert_eq!( + classify(&format!("Invoice@sha256:{prefix}")), + PinMatch::WrongEntity + ); + assert_eq!( + classify(&format!("Invoice@sha256:{REGISTERED}")), + PinMatch::WrongEntity + ); +} + +#[test] +fn a_qualified_non_digest_names_a_version_this_kernel_cannot_resolve() { + assert_eq!( + classify("Order@sha256:v1-long-gone"), + PinMatch::OtherVersion + ); +} + +#[test] +fn a_digest_longer_than_sha256_names_another_version() { + assert_eq!( + classify(&format!("Order@sha256:{REGISTERED}0")), + PinMatch::OtherVersion + ); +} + +#[test] +fn surrounding_whitespace_does_not_change_what_a_pin_names() { + let prefix = ®ISTERED[..MIN_PIN_DIGEST_HEX]; + assert_eq!( + classify(&format!(" Order @ sha256:{prefix} ")), + PinMatch::Registered + ); +} + +fn agree(left: &str, right: &str) -> bool { + declare_same_spec(left, right, "Order", REGISTERED) +} + +#[test] +fn two_spellings_of_one_version_agree() { + assert!(agree(REGISTERED, &format!("sha256:{REGISTERED}"))); + assert!(agree( + &format!("Order@sha256:{REGISTERED}"), + &format!("sha256:{REGISTERED}") + )); +} + +#[test] +fn a_prefix_pin_agrees_with_the_full_hash_it_prefixes() { + let prefix = ®ISTERED[..MIN_PIN_DIGEST_HEX]; + assert!( + agree(&format!("Order@sha256:{prefix}"), REGISTERED), + "both resolve to the registered spec, so neither contradicts the other" + ); +} + +#[test] +fn two_spellings_of_a_version_that_is_gone_still_agree() { + // Neither resolves, but they do not disagree with each other, and saying + // they do would blame the request for a spec that was simply replaced. + assert!(agree("sha256:v1-long-gone", "v1-long-gone")); +} + +#[test] +fn declarations_naming_different_versions_disagree() { + assert!(!agree(REGISTERED, OTHER)); + assert!(!agree("sha256:v1-long-gone", REGISTERED)); +} + +#[test] +fn a_prefix_pin_for_another_entity_disagrees() { + let prefix = ®ISTERED[..MIN_PIN_DIGEST_HEX]; + assert!( + !agree(&format!("Invoice@sha256:{prefix}"), REGISTERED), + "a pin naming another actor cannot stand in for this run's version" + ); +} diff --git a/crates/temper-server/src/api/trajectory_analysis.rs b/crates/temper-server/src/api/trajectory_analysis.rs new file mode 100644 index 000000000..c37b69555 --- /dev/null +++ b/crates/temper-server/src/api/trajectory_analysis.rs @@ -0,0 +1,409 @@ +//! Trajectory analysis endpoints: conformance checking and ATIF export. + +use axum::extract::{Extension, Path, State}; +use axum::http::StatusCode; +use axum::response::Json; +use serde::Deserialize; +use temper_ots::AtifTrajectory; +use temper_ots::models::OTSTrajectory; +use temper_runtime::tenant::TenantId; +use temper_spec::automaton::Automaton; +use tracing::instrument; + +use crate::authz::{ + AuthenticatedRequestContext, observe_tenant_scope, require_authenticated_context, + require_observe_auth, +}; +use crate::conformance::{ConformanceInput, SpecResolution, check_conformance}; +use crate::state::ServerState; + +use super::spec_pin::{MIN_PIN_DIGEST_HEX, PinMatch, classify_pin, declare_same_spec}; + +/// Largest session the conformance checker will read in one request. +/// +/// The checker holds the whole session in memory and the state-machine walk is +/// order-dependent, so it cannot be paged; the cap bounds the read instead. +const MAX_CONFORMANCE_ROWS: i64 = 5_000; + +/// Denial body for both endpoints. +/// +/// Names the permit an operator has to install, because the gate has no +/// principal-kind bypass and a bare "unauthorized" leaves them guessing which +/// action and resource Cedar was asked about. +const UNAUTHORIZED_DETAIL: &str = "unauthorized: this endpoint returns recorded agent content and \ + requires a Cedar permit for action `read_trajectories` on resource `Trajectory` in the \ + addressed tenant"; + +/// Body of `POST /api/conformance/check`. +#[derive(Debug, Deserialize)] +pub(crate) struct ConformanceCheckRequest { + /// The actor spec to check the run against. + entity_type: String, + /// The session whose rows form the run. + session_id: String, + /// Optional OTS trajectory contributing the agent-side decisions. + #[serde(default)] + trajectory_id: Option, + /// The spec version the run executed under, for a run whose trajectory + /// does not carry one. + /// + /// A hint, never an override: when the trajectory pins a version, that one + /// governs and a conflicting hint is refused. See + /// [`resolve_governing_spec`]. + #[serde(default)] + spec_version: Option, + /// Cap on rows read, bounded by [`MAX_CONFORMANCE_ROWS`]. + #[serde(default)] + limit: Option, +} + +/// POST /api/conformance/check — check one session against its actor spec. +#[instrument(skip_all, fields(otel.name = "POST /api/conformance/check"))] +pub(crate) async fn handle_conformance_check( + State(state): State, + authenticated: Option>, + Json(request): Json, +) -> Result, (StatusCode, String)> { + let authenticated = require_authenticated_context(authenticated.as_deref()) + .map_err(|status| (status, UNAUTHORIZED_DETAIL.to_string()))?; + let tenant = credential_tenant(authenticated); + require_observe_auth(&state, authenticated, "read_trajectories", "Trajectory") + .map_err(|status| (status, UNAUTHORIZED_DETAIL.to_string()))?; + + let limit = match request.limit { + Some(limit) if !(1..=MAX_CONFORMANCE_ROWS).contains(&limit) => { + return Err(( + StatusCode::BAD_REQUEST, + format!("limit must be between 1 and {MAX_CONFORMANCE_ROWS}"), + )); + } + Some(limit) => limit, + None => MAX_CONFORMANCE_ROWS, + }; + + let spec = registered_spec(&state, &tenant, &request.entity_type)?; + + let Some(store) = state.metadata_store_for_tenant(tenant.as_str()).await else { + return Err(( + StatusCode::SERVICE_UNAVAILABLE, + "no durable metadata backend configured; conformance needs stored trajectories" + .to_string(), + )); + }; + + // Entity type is deliberately not pushed into the query: the checker + // reports rows from other actors as skipped, which is how a caller learns + // a session touched entities this spec does not govern. + // + // One row past the cap is read so a session that ends exactly on the cap + // is reported complete instead of partial. The extra row is dropped before + // the walk, which must see a prefix and nothing beyond it. + let mut rows = store + .query_trajectories_by_session( + &request.session_id, + Some(tenant.as_str()), + None, + limit.saturating_add(1), + ) + .await + .map_err(|error| { + tracing::warn!(error = %error, "failed to read session trajectories"); + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("failed to read session trajectories: {error}"), + ) + })?; + let truncated = rows.len() as i64 > limit; + rows.truncate(limit as usize); + + let ots = match request.trajectory_id.as_deref() { + Some(trajectory_id) => { + let (ots_session_id, trajectory) = + load_ots_trajectory(&store, &tenant, trajectory_id).await?; + // Folding another run's decisions into this session's report would + // produce violations that belong to neither run, so the stored + // trajectory has to say which run it came from, and it has to be + // this one. + // + // An empty session column matches nothing rather than everything: + // the upload carried no `X-Session-Id`, so the trajectory is tied + // to no run at all. Read as a wildcard it let one unattributed + // upload be folded into every session a caller cared to name. + if ots_session_id.is_empty() { + return Err(( + StatusCode::BAD_REQUEST, + format!( + "trajectory `{trajectory_id}` was stored without a session, so nothing \ + ties it to session `{}`; re-upload it with an `X-Session-Id` header \ + naming the run it came from", + request.session_id + ), + )); + } + if ots_session_id != request.session_id { + return Err(( + StatusCode::BAD_REQUEST, + format!( + "trajectory `{trajectory_id}` belongs to session `{ots_session_id}`, \ + not `{}`", + request.session_id + ), + )); + } + Some(trajectory) + } + None => None, + }; + + let spec_resolution = resolve_governing_spec( + &spec, + ots.as_ref() + .and_then(|ots| ots.metadata.spec_version.as_deref()), + request.spec_version.as_deref(), + &request.entity_type, + )?; + + // Asked of this server's own capture path: a loss it could not attribute + // to any session means the rows just read may be missing some, with no + // marker in them to say so. + let capture_degraded = state.capture_health.is_degraded(); + let report = check_conformance(ConformanceInput { + automaton: &spec.automaton, + kernel_rows: &rows, + ots_trajectory: ots.as_ref(), + rows_truncated: truncated, + spec_resolution, + capture_degraded, + }); + tracing::info!( + tenant = %tenant, + entity_type = %request.entity_type, + session_id = %request.session_id, + spec_version = %spec.version, + spec_resolution = ?spec_resolution, + rows = rows.len(), + truncated, + verdict = ?report.verdict, + passed = report.passed, + evidence_complete = report.evidence_complete, + violations = report.violations.len(), + "conformance.check" + ); + + Ok(Json(serde_json::json!({ + "tenant": tenant.as_str(), + "entity_type": request.entity_type, + "session_id": request.session_id, + "trajectory_id": request.trajectory_id, + // The spec the run was judged against, so a report can never be read + // without knowing which version produced it. + "spec_version": spec.version, + "row_limit": limit, + "truncated": truncated, + "report": report, + }))) +} + +/// GET /api/ots/trajectories/{id}/atif — export one trajectory as ATIF v1.7. +#[instrument(skip_all, fields(otel.name = "GET /api/ots/trajectories/{id}/atif"))] +pub(crate) async fn handle_get_ots_trajectory_atif( + State(state): State, + authenticated: Option>, + Path(trajectory_id): Path, +) -> Result, (StatusCode, String)> { + let authenticated = require_authenticated_context(authenticated.as_deref()) + .map_err(|status| (status, UNAUTHORIZED_DETAIL.to_string()))?; + let tenant = credential_tenant(authenticated); + require_observe_auth(&state, authenticated, "read_trajectories", "Trajectory") + .map_err(|status| (status, UNAUTHORIZED_DETAIL.to_string()))?; + + let Some(store) = state.metadata_store_for_tenant(tenant.as_str()).await else { + return Err(( + StatusCode::SERVICE_UNAVAILABLE, + "no durable metadata backend configured; nothing to export".to_string(), + )); + }; + + let (session_id, trajectory) = load_ots_trajectory(&store, &tenant, &trajectory_id).await?; + // The session lives on the storage row, not in the OTS document; an empty + // column means the upload carried no session header, so ATIF omits it + // rather than reporting an empty run identity. + let session_id = (!session_id.is_empty()).then_some(session_id); + let atif = temper_ots::to_atif(&trajectory, session_id.as_deref()).map_err(|error| { + // The stored trajectory is intact; it just has no valid ATIF + // rendering, which is the caller's answer rather than a server fault. + ( + StatusCode::UNPROCESSABLE_ENTITY, + format!("trajectory `{trajectory_id}` cannot be exported as ATIF: {error}"), + ) + })?; + Ok(Json(atif)) +} + +/// The tenant this request operates in: always the credential's own. +fn credential_tenant(authenticated: &AuthenticatedRequestContext) -> TenantId { + observe_tenant_scope(authenticated).clone() +} + +/// A registered spec and the identity of the source it was parsed from. +struct RegisteredSpec { + automaton: Automaton, + /// Content hash of the IOA source, the same identity the spec store keeps + /// and the value a harness records as `metadata.spec_version`. + version: String, +} + +/// Load the actor spec for `entity_type`, cloned out of the registry lock. +fn registered_spec( + state: &ServerState, + tenant: &TenantId, + entity_type: &str, +) -> Result { + let registry = state.registry.read().map_err(|_| { + ( + StatusCode::INTERNAL_SERVER_ERROR, + "spec registry lock poisoned".to_string(), + ) + })?; + registry + .get_spec(tenant, entity_type) + .map(|spec| RegisteredSpec { + automaton: spec.automaton.clone(), + version: temper_store_turso::spec_content_hash(&spec.ioa_source), + }) + .ok_or(( + StatusCode::NOT_FOUND, + format!("no spec registered for entity type `{entity_type}` in tenant `{tenant}`"), + )) +} + +/// Refuse to judge a run by a spec that did not govern it. +/// +/// A conformance report only means something against the spec the run executed +/// under. Temper keeps one spec per (tenant, entity type) — a submit replaces +/// the previous version rather than versioning it — so the registered spec is +/// whatever was submitted last, which need not be the one in force when the run +/// happened. Three cases follow. +/// +/// **The trajectory pins a version.** `OTSMetadata.spec_version` is written by +/// the harness that drove the run, from inside the run; it is the provenance. +/// It governs. A request that names a different version is refused rather than +/// honoured: the request body is the one part of this input a caller chooses +/// freely, and letting it override the recorded provenance would turn "check +/// this run against its spec" into "check this run against whichever spec makes +/// it pass". +/// +/// **Only the request names a version.** Nothing recorded contradicts it, so it +/// is used — and, like a pinned version, it has to match the registered spec or +/// the check has nothing to run against. +/// +/// **A named version is not the registered one.** The governing spec is gone, +/// so there is nothing to check against. A report against the current spec +/// would pass behaviour that violated the old one and condemn behaviour that +/// was legal under it, so the request is refused. +/// +/// **Nothing names a version.** The check runs against the registered spec and +/// says so: the resolution is [`SpecResolution::Unresolved`], which the report +/// carries as an evidence gap, so the run cannot come back `passed`. +/// +/// Which spellings of a version name which spec is [`spec_pin`]'s to decide; +/// this function decides what to do about the answer. +fn resolve_governing_spec( + spec: &RegisteredSpec, + pinned_spec_version: Option<&str>, + requested_spec_version: Option<&str>, + entity_type: &str, +) -> Result { + let governing = match (pinned_spec_version, requested_spec_version) { + (Some(pinned), Some(requested)) + if !declare_same_spec(pinned, requested, entity_type, &spec.version) => + { + return Err(( + StatusCode::CONFLICT, + format!( + "the request asks to check `{entity_type}` against spec version \ + `{requested}`, but the trajectory records that the run executed under \ + `{pinned}`; the recorded version is the run's provenance and a request \ + cannot override it" + ), + )); + } + (Some(pinned), _) => pinned, + (None, Some(requested)) => requested, + (None, None) => return Ok(SpecResolution::Unresolved), + }; + match classify_pin(governing, entity_type, &spec.version) { + PinMatch::Registered => Ok(SpecResolution::Pinned), + // Under-specified rather than wrong: the caller is told what to send, + // not that its run executed under a spec that is gone. + PinMatch::DigestTooShort => Err(( + StatusCode::BAD_REQUEST, + format!( + "spec version `{governing}` truncates the digest below {MIN_PIN_DIGEST_HEX} \ + characters, which is short enough to name more than one spec; send at least \ + {MIN_PIN_DIGEST_HEX} hex characters, or the whole hash, which \ + `GET /observe/specs/{entity_type}` reports as `spec_version`" + ), + )), + PinMatch::WrongEntity => Err(( + StatusCode::BAD_REQUEST, + format!( + "spec version `{governing}` pins a spec for a different entity than \ + `{entity_type}`; a run is checked against its own actor's spec, so the pin and \ + the entity being checked have to name the same one" + ), + )), + PinMatch::OtherVersion => Err(( + StatusCode::CONFLICT, + format!( + "the run executed under spec version `{governing}` for `{entity_type}`, but the \ + registered spec is `{}`; Temper stores one version per entity type, so the \ + governing spec is not available to check against and a report against the \ + current one would judge the run by rules it never ran under", + spec.version + ), + )), + } +} + +/// Load and parse a stored OTS trajectory, returning its session id with it. +/// +/// A missing row is a `404`; stored data that no longer parses is a `500` and +/// is logged, because that is a storage fault rather than a bad request. +async fn load_ots_trajectory( + store: &std::sync::Arc, + tenant: &TenantId, + trajectory_id: &str, +) -> Result<(String, OTSTrajectory), (StatusCode, String)> { + let document = store + .get_ots_trajectory(tenant.as_str(), trajectory_id) + .await + .map_err(|error| { + tracing::warn!(error = %error, "failed to read OTS trajectory"); + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("failed to read OTS trajectory: {error}"), + ) + })? + .ok_or(( + StatusCode::NOT_FOUND, + format!("no OTS trajectory `{trajectory_id}` in tenant `{tenant}`"), + ))?; + + let trajectory: OTSTrajectory = serde_json::from_str(&document.data).map_err(|error| { + tracing::error!( + error = %error, + trajectory_id = %trajectory_id, + "stored OTS trajectory does not parse" + ); + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("stored OTS trajectory `{trajectory_id}` does not parse: {error}"), + ) + })?; + Ok((document.session_id, trajectory)) +} + +#[cfg(test)] +#[path = "trajectory_analysis_test.rs"] +mod trajectory_analysis_test; diff --git a/crates/temper-server/src/api/trajectory_analysis_test.rs b/crates/temper-server/src/api/trajectory_analysis_test.rs new file mode 100644 index 000000000..6dc9bd39f --- /dev/null +++ b/crates/temper-server/src/api/trajectory_analysis_test.rs @@ -0,0 +1,1813 @@ +//! Endpoint tests for conformance checking and ATIF export. +//! +//! These drive the real router against a local Turso store seeded with kernel +//! rows and a stored OTS trajectory, so the selector, the tenant scoping, and +//! the JSON shape are all exercised rather than just the pure checker. + +use axum::Router; +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use temper_runtime::ActorSystem; +use temper_spec::csdl::parse_csdl; +use temper_store_turso::{OtsTrajectoryParams, TursoEventStore, TursoTrajectoryInsert}; +use tower::ServiceExt; + +use crate::registry::SpecRegistry; +use crate::state::ServerState; +use crate::storage::StorageStack; + +const CSDL_XML: &str = include_str!("../../../../test-fixtures/specs/model.csdl.xml"); +const ORDER_IOA: &str = include_str!("../../../../test-fixtures/specs/order.ioa.toml"); + +async fn test_app() -> (Router, TursoEventStore) { + build_app(AuthzMode::Permissive).await +} + +/// An app whose Cedar engine carries no permits beyond the built-in +/// system-platform policy: System passes by that built-in permit, and every +/// other principal hits default-deny. +/// +/// The gate tests use THIS app and authenticate as an ordinary agent — a System +/// principal would pass by the built-in permit whether or not the gate ran, so +/// only a non-System principal can prove the gate is on the path. +async fn strict_authz_app() -> Router { + build_app(AuthzMode::Strict).await.0 +} + +/// An app whose Cedar engine permits exactly the trajectory read these +/// endpoints gate on, and nothing else. +async fn policy_permitted_app() -> (Router, TursoEventStore) { + build_app(AuthzMode::PermitTrajectoryReads).await +} + +enum AuthzMode { + Permissive, + Strict, + PermitTrajectoryReads, +} + +async fn build_app(authz: AuthzMode) -> (Router, TursoEventStore) { + static COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let id = COUNTER.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + let dir = + std::env::temp_dir().join(format!("temper-conformance-{}-{}", std::process::id(), id)); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).expect("create temp dir"); + let db_url = format!("file:{}", dir.join("conformance.db").display()); + + let turso = TursoEventStore::new(&db_url, None) + .await + .expect("create local turso db"); + + let csdl = parse_csdl(CSDL_XML).expect("CSDL parses"); + let mut registry = SpecRegistry::new(); + registry.register_tenant( + "default", + csdl, + CSDL_XML.to_string(), + &[("Order", ORDER_IOA)], + ); + let mut state = ServerState::from_registry(ActorSystem::new("test-conformance"), registry); + match authz { + AuthzMode::Permissive => {} + AuthzMode::Strict => { + state.authz = std::sync::Arc::new(temper_authz::AuthzEngine::empty()); + } + AuthzMode::PermitTrajectoryReads => { + let engine = temper_authz::AuthzEngine::empty(); + engine + .reload_tenant_policies( + "default", + r#"permit(principal, action == Action::"read_trajectories", resource is Trajectory);"#, + ) + .expect("policy parses"); + state.authz = std::sync::Arc::new(engine); + } + } + state.data_dir = dir; + state.set_storage_stack(StorageStack::from_turso(turso.clone())); + + let app = Router::new() + .nest("/api", crate::api::build_api_router()) + .with_state(state); + (app, turso) +} + +async fn seed_row( + store: &TursoEventStore, + action: &str, + from: Option<&str>, + to: Option<&str>, + created_at: &str, +) { + store + .persist_trajectory(TursoTrajectoryInsert { + tenant: "default", + entity_type: "Order", + entity_id: "order-1", + action, + success: true, + from_status: from, + to_status: to, + error: None, + agent_id: Some("agent-1"), + session_id: Some("session-1"), + authz_denied: None, + denied_resource: None, + denied_module: None, + source: Some("Entity"), + spec_governed: Some(true), + created_at, + request_body: None, + intent: None, + matched_policy_ids: None, + capture_seq: None, + }) + .await + .expect("persist trajectory row"); +} + +/// Attach the credential the ingress edge would have installed (ADR-0157). +/// +/// These fixtures previously declared `X-Temper-Principal-Kind: admin` — the +/// header-minted authority the auth edge now strips. The tenant rides the +/// credential too, so a request can no longer name a tenant it is not bound to. +fn authenticate(mut request: Request, tenant: Option<&str>) -> Request { + let tenant = tenant + .map(temper_runtime::tenant::TenantId::new) + .unwrap_or_default(); + request + .extensions_mut() + .insert(temper_authz::AuthenticatedRequestContext::new( + tenant, + temper_authz::SecurityContext::system(), + )); + request +} + +/// Attach a credential for a specific principal (for the authorization tests). +fn authenticate_as( + mut request: Request, + tenant: &str, + security_ctx: temper_authz::SecurityContext, +) -> Request { + request + .extensions_mut() + .insert(temper_authz::AuthenticatedRequestContext::new( + temper_runtime::tenant::TenantId::new(tenant), + security_ctx, + )); + request +} + +/// An ordinary resolved agent — no built-in permit, so Cedar decides. +fn agent_context() -> temper_authz::SecurityContext { + temper_authz::SecurityContext::from_resolved_identity("agent-1", "operator", None) +} + +fn admin_post(uri: &str, body: serde_json::Value, tenant: Option<&str>) -> Request { + let request = Request::post(uri) + .header("Content-Type", "application/json") + .body(Body::from(body.to_string())) + .unwrap(); + authenticate(request, tenant) +} + +fn admin_get(uri: &str, tenant: Option<&str>) -> Request { + let request = Request::get(uri).body(Body::empty()).unwrap(); + authenticate(request, tenant) +} + +async fn json_body(response: axum::response::Response) -> serde_json::Value { + let body = axum::body::to_bytes(response.into_body(), 4 * 1024 * 1024) + .await + .expect("read body"); + serde_json::from_slice(&body).expect("body is JSON") +} + +async fn response_text(response: axum::response::Response) -> String { + let body = axum::body::to_bytes(response.into_body(), 4 * 1024 * 1024) + .await + .expect("read body"); + String::from_utf8(body.to_vec()).expect("body is text") +} + +/// The content hash of the spec [`build_app`] registers. +/// +/// A check that names no version cannot come back `passed` — the governing +/// spec is unresolved — so every test asserting a pass names this one. +fn registered_spec_version() -> String { + temper_store_turso::spec_content_hash(ORDER_IOA) +} + +#[tokio::test] +async fn conformance_check_reports_a_clean_session() { + let (app, store) = test_app().await; + seed_row( + &store, + "AddItem", + Some("Draft"), + Some("Draft"), + "2026-01-01T00:00:00Z", + ) + .await; + seed_row( + &store, + "SubmitOrder", + Some("Draft"), + Some("Submitted"), + "2026-01-01T00:00:01Z", + ) + .await; + + let response = app + .oneshot(admin_post( + "/api/conformance/check", + serde_json::json!({ + "entity_type": "Order", + "session_id": "session-1", + "spec_version": registered_spec_version(), + }), + Some("default"), + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let body = json_body(response).await; + assert_eq!(body["tenant"], "default"); + assert_eq!(body["session_id"], "session-1"); + assert_eq!(body["report"]["passed"], serde_json::json!(true)); + assert_eq!(body["report"]["spec_resolution"], "pinned"); + assert_eq!( + body["report"]["evidence_complete"], + serde_json::json!(true), + "a resolved spec and a complete read leave nothing unseen" + ); + assert_eq!(body["report"]["violations"], serde_json::json!([])); + assert_eq!(body["report"]["stats"]["actor_rows"], serde_json::json!(2)); +} + +#[tokio::test] +async fn conformance_check_reports_an_illegal_transition_at_its_index() { + let (app, store) = test_app().await; + seed_row( + &store, + "AddItem", + Some("Draft"), + Some("Draft"), + "2026-01-01T00:00:00Z", + ) + .await; + seed_row( + &store, + "ShipOrder", + Some("Draft"), + Some("Shipped"), + "2026-01-01T00:00:01Z", + ) + .await; + + let response = app + .oneshot(admin_post( + "/api/conformance/check", + serde_json::json!({"entity_type": "Order", "session_id": "session-1"}), + Some("default"), + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let body = json_body(response).await; + assert_eq!(body["report"]["passed"], serde_json::json!(false)); + let violations = body["report"]["violations"].as_array().expect("violations"); + assert_eq!(violations.len(), 1); + assert_eq!(violations[0]["index"], serde_json::json!(1)); + assert_eq!(violations[0]["kind"], "illegal_transition"); + assert_eq!(violations[0]["action"], "ShipOrder"); +} + +#[tokio::test] +async fn conformance_check_requires_a_credential() { + // The tenant is no longer a caller-supplied field to omit — it is bound to + // the credential (ADR-0157), so the failure this endpoint owes a caller is + // "you presented no identity", not "you forgot a header". + let (app, _store) = test_app().await; + + let response = app + .oneshot( + Request::post("/api/conformance/check") + .header("Content-Type", "application/json") + .body(Body::from( + serde_json::json!({"entity_type": "Order", "session_id": "session-1"}) + .to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); +} + +#[tokio::test] +async fn conformance_check_rejects_an_unregistered_entity_type() { + let (app, _store) = test_app().await; + + let response = app + .oneshot(admin_post( + "/api/conformance/check", + serde_json::json!({"entity_type": "Nonexistent", "session_id": "session-1"}), + Some("default"), + )) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::NOT_FOUND); +} + +#[tokio::test] +async fn conformance_check_rejects_an_out_of_range_limit() { + let (app, _store) = test_app().await; + + let response = app + .oneshot(admin_post( + "/api/conformance/check", + serde_json::json!({"entity_type": "Order", "session_id": "session-1", "limit": 0}), + Some("default"), + )) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::BAD_REQUEST); +} + +#[tokio::test] +async fn conformance_check_folds_in_a_stored_ots_trajectory() { + let (app, store) = test_app().await; + seed_row( + &store, + "AddItem", + Some("Draft"), + Some("Draft"), + "2026-01-01T00:00:00Z", + ) + .await; + + // The agent decided on an action the kernel never recorded a row for. + let data = serde_json::json!({ + "trajectory_id": "traj-1", + "version": "0.1.0", + "metadata": { + "task_description": "place an order", + "timestamp_start": "2026-01-01T00:00:00Z", + "agent_id": "agent-1", + "outcome": "failure", + "human_reviewed": false + }, + "context": {}, + "turns": [{ + "turn_id": 1, + "span_id": "span-1", + "timestamp": "2026-01-01T00:00:00Z", + "error": true, + "decisions": [{ + "decision_id": "decision-1", + "decision_type": "tool_selection", + "choice": {"action": "Frobnicate"}, + "consequence": {"success": false} + }] + }] + }) + .to_string(); + store + .persist_ots_trajectory(&OtsTrajectoryParams { + trajectory_id: "traj-1", + tenant: "default", + agent_id: "agent-1", + session_id: "session-1", + outcome: "failure", + turn_count: 1, + data: &data, + }) + .await + .expect("persist OTS trajectory"); + + let response = app + .oneshot(admin_post( + "/api/conformance/check", + serde_json::json!({ + "entity_type": "Order", + "session_id": "session-1", + "trajectory_id": "traj-1" + }), + Some("default"), + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let body = json_body(response).await; + let violations = body["report"]["violations"].as_array().expect("violations"); + assert_eq!(violations.len(), 1); + assert_eq!(violations[0]["kind"], "unknown_action"); + assert_eq!(violations[0]["action"], "Frobnicate"); + assert_eq!( + violations[0]["index"], + serde_json::json!(1), + "the OTS decision is indexed after the single kernel row" + ); + assert_eq!( + body["report"]["stats"]["ots_decisions_checked"], + serde_json::json!(1) + ); +} + +#[tokio::test] +async fn conformance_check_rejects_a_trajectory_from_another_session() { + let (app, store) = test_app().await; + seed_row( + &store, + "AddItem", + Some("Draft"), + Some("Draft"), + "2026-01-01T00:00:00Z", + ) + .await; + let data = r#"{"trajectory_id":"traj-elsewhere","version":"0.1.0","metadata":{"task_description":"t","timestamp_start":"2026-01-01T00:00:00Z","agent_id":"a","outcome":"success","human_reviewed":false},"context":{}}"#; + store + .persist_ots_trajectory(&OtsTrajectoryParams { + trajectory_id: "traj-elsewhere", + tenant: "default", + agent_id: "agent-1", + session_id: "some-other-session", + outcome: "success", + turn_count: 0, + data, + }) + .await + .expect("persist OTS trajectory"); + + let response = app + .oneshot(admin_post( + "/api/conformance/check", + serde_json::json!({ + "entity_type": "Order", + "session_id": "session-1", + "trajectory_id": "traj-elsewhere" + }), + Some("default"), + )) + .await + .unwrap(); + + assert_eq!( + response.status(), + StatusCode::BAD_REQUEST, + "another run's decisions must not be folded into this session's report" + ); +} + +#[tokio::test] +async fn conformance_check_flags_a_truncated_session() { + let (app, store) = test_app().await; + seed_row( + &store, + "AddItem", + Some("Draft"), + Some("Draft"), + "2026-01-01T00:00:00Z", + ) + .await; + seed_row( + &store, + "SubmitOrder", + Some("Draft"), + Some("Submitted"), + "2026-01-01T00:00:01Z", + ) + .await; + + let response = app + .oneshot(admin_post( + "/api/conformance/check", + serde_json::json!({"entity_type": "Order", "session_id": "session-1", "limit": 1}), + Some("default"), + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let body = json_body(response).await; + assert_eq!(body["truncated"], serde_json::json!(true)); + assert_eq!(body["row_limit"], serde_json::json!(1)); + assert_eq!(body["report"]["stats"]["actor_rows"], serde_json::json!(1)); +} + +#[tokio::test] +async fn atif_export_returns_a_v1_7_document_with_the_stored_session() { + let (app, store) = test_app().await; + let data = serde_json::json!({ + "trajectory_id": "traj-atif", + "version": "0.1.0", + "metadata": { + "task_description": "place an order", + "timestamp_start": "2026-01-01T00:00:00Z", + "agent_id": "agent-1", + "outcome": "success", + "human_reviewed": false, + "harness": "temperpaw", + "spec_version": "sha256:abcd", + "agent_version": "temperpaw 3.2" + }, + "context": {}, + "turns": [{ + "turn_id": 1, + "span_id": "span-1", + "timestamp": "2026-01-01T00:00:00Z", + "error": false, + "messages": [{ + "message_id": "message-1", + "role": "assistant", + "timestamp": "2026-01-01T00:00:00Z", + "content": {"type": "text", "text": "Submitting the order."} + }], + "decisions": [{ + "decision_id": "decision-1", + "decision_type": "tool_selection", + "choice": {"action": "SubmitOrder"}, + "consequence": {"success": true, "result_summary": "Submitted"}, + "cause_id": "call-1" + }] + }] + }) + .to_string(); + store + .persist_ots_trajectory(&OtsTrajectoryParams { + trajectory_id: "traj-atif", + tenant: "default", + agent_id: "agent-1", + session_id: "session-7", + outcome: "success", + turn_count: 1, + data: &data, + }) + .await + .expect("persist OTS trajectory"); + + let response = app + .oneshot(admin_get( + "/api/ots/trajectories/traj-atif/atif", + Some("default"), + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let body = json_body(response).await; + assert_eq!(body["schema_version"], "ATIF-v1.7"); + assert_eq!(body["trajectory_id"], "traj-atif"); + assert_eq!( + body["session_id"], "session-7", + "the session comes from the storage row, not the document" + ); + assert_eq!(body["agent"]["name"], "temperpaw"); + assert_eq!( + body["agent"]["version"], "temperpaw 3.2", + "agent.version is the agent system's release" + ); + assert_eq!( + body["agent"]["extra"]["temper.spec_version"], "sha256:abcd", + "the governing spec stays readable without posing as the agent release" + ); + let steps = body["steps"].as_array().expect("steps"); + assert_eq!(steps.len(), 1); + assert_eq!(steps[0]["source"], "agent"); + assert_eq!(steps[0]["message"], "Submitting the order."); + assert_eq!(steps[0]["tool_calls"][0]["tool_call_id"], "call-1"); + assert_eq!(steps[0]["tool_calls"][0]["function_name"], "SubmitOrder"); + assert_eq!( + steps[0]["observation"]["results"][0]["source_call_id"], + "call-1" + ); +} + +#[tokio::test] +async fn atif_export_is_scoped_to_the_requesting_tenant() { + let (app, store) = test_app().await; + let data = r#"{"trajectory_id":"traj-other","version":"0.1.0","metadata":{"task_description":"t","timestamp_start":"2026-01-01T00:00:00Z","agent_id":"a","outcome":"success","human_reviewed":false},"context":{}}"#; + store + .persist_ots_trajectory(&OtsTrajectoryParams { + trajectory_id: "traj-other", + tenant: "other-tenant", + agent_id: "agent-1", + session_id: "session-1", + outcome: "success", + turn_count: 0, + data, + }) + .await + .expect("persist OTS trajectory"); + + let response = app + .oneshot(admin_get( + "/api/ots/trajectories/traj-other/atif", + Some("default"), + )) + .await + .unwrap(); + + assert_eq!( + response.status(), + StatusCode::NOT_FOUND, + "a trajectory belonging to another tenant must not be exportable" + ); +} + +#[tokio::test] +async fn a_conformance_check_never_walks_another_tenants_rows() { + // The check resolves a session's rows by id. Session ids are caller-chosen + // and not globally unique, so the store lookup has to be tenant-bound: drop + // that bind and this session's rows from `other-tenant` — request bodies and + // intents included — would be walked as if they were the caller's own. + let (app, store) = test_app().await; + store + .persist_trajectory(TursoTrajectoryInsert { + tenant: "other-tenant", + entity_type: "Order", + entity_id: "order-1", + action: "ShipOrder", + success: true, + from_status: Some("Draft"), + to_status: Some("Shipped"), + error: None, + agent_id: Some("agent-1"), + session_id: Some("session-1"), + authz_denied: None, + denied_resource: None, + denied_module: None, + source: Some("Entity"), + spec_governed: Some(true), + created_at: "2026-01-01T00:00:00Z", + request_body: None, + intent: None, + matched_policy_ids: None, + capture_seq: None, + }) + .await + .expect("persist foreign-tenant row"); + + let response = app + .oneshot(admin_post( + "/api/conformance/check", + serde_json::json!({"entity_type": "Order", "session_id": "session-1"}), + Some("default"), + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let body = json_body(response).await; + + // The caller's tenant holds no rows for this session, so the walk must come + // back indeterminate. If the store lookup lost its tenant bind it would walk + // the `other-tenant` ShipOrder row instead and return a real verdict. + assert_eq!( + body["report"]["verdict"], "indeterminate", + "a credential for `default` must not walk `other-tenant` rows: {body}" + ); + assert_eq!(body["report"]["passed"], serde_json::json!(false)); +} + +#[tokio::test] +async fn an_x_tenant_id_header_cannot_redirect_a_credential_to_another_tenant() { + // The central claim of ADR-0157: the tenant is bound to the credential, so + // naming a different one in `X-Tenant-Id` buys nothing. Without this the + // property holds only by construction, and a future handler that reads the + // header back would pass every other test in this file. + let (app, store) = test_app().await; + let data = r#"{"trajectory_id":"traj-victim","version":"0.1.0","metadata":{"task_description":"t","timestamp_start":"2026-01-01T00:00:00Z","agent_id":"a","outcome":"success","human_reviewed":false},"context":{}}"#; + store + .persist_ots_trajectory(&OtsTrajectoryParams { + trajectory_id: "traj-victim", + tenant: "victim-tenant", + agent_id: "agent-1", + session_id: "session-1", + outcome: "success", + turn_count: 0, + data, + }) + .await + .expect("persist OTS trajectory"); + + // Credential is bound to `default`; the header asks for `victim-tenant`. + let response = app + .oneshot(authenticate_as( + Request::get("/api/ots/trajectories/traj-victim/atif") + .header("X-Tenant-Id", "victim-tenant") + .body(Body::empty()) + .unwrap(), + "default", + temper_authz::SecurityContext::system(), + )) + .await + .unwrap(); + + assert_eq!( + response.status(), + StatusCode::NOT_FOUND, + "the credential's tenant wins; a header must not redirect the lookup" + ); +} + +#[tokio::test] +async fn atif_export_reports_a_missing_trajectory_as_not_found() { + let (app, _store) = test_app().await; + + let response = app + .oneshot(admin_get( + "/api/ots/trajectories/does-not-exist/atif", + Some("default"), + )) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::NOT_FOUND); +} + +#[tokio::test] +async fn atif_export_requires_a_credential() { + // Same contract change as the conformance check: the export's tenant comes + // from the credential, so an unidentified caller is refused at the edge + // rather than told which header it forgot. + let (app, _store) = test_app().await; + + let response = app + .oneshot( + Request::get("/api/ots/trajectories/traj-atif/atif") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); +} + +#[tokio::test] +async fn both_endpoints_reject_an_unauthorized_principal() { + let app = strict_authz_app().await; + + let check = app + .clone() + .oneshot(authenticate_as( + Request::post("/api/conformance/check") + .header("Content-Type", "application/json") + .body(Body::from( + serde_json::json!({"entity_type": "Order", "session_id": "session-1"}) + .to_string(), + )) + .unwrap(), + "default", + agent_context(), + )) + .await + .unwrap(); + assert_eq!(check.status(), StatusCode::FORBIDDEN); + + let export = app + .clone() + .oneshot(authenticate_as( + Request::get("/api/ots/trajectories/traj-atif/atif") + .body(Body::empty()) + .unwrap(), + "default", + agent_context(), + )) + .await + .unwrap(); + assert_eq!(export.status(), StatusCode::FORBIDDEN); +} + +#[tokio::test] +async fn a_self_declared_admin_header_does_not_open_either_endpoint() { + // The principal kind is a request header the platform does not + // authenticate, so it must buy nothing on a surface that returns one + // named run's prompts, decisions, and request bodies. + let app = strict_authz_app().await; + + // Header only, no credential: the ingress edge strips the claim and admits + // nothing, so the request never reaches Cedar (ADR-0157). + let forged = app + .clone() + .oneshot( + Request::post("/api/conformance/check") + .header("X-Temper-Principal-Kind", "admin") + .header("X-Tenant-Id", "default") + .header("Content-Type", "application/json") + .body(Body::from( + serde_json::json!({"entity_type": "Order", "session_id": "session-1"}) + .to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!( + forged.status(), + StatusCode::UNAUTHORIZED, + "a self-declared admin header is not an identity" + ); + + // And with a real credential, the forged header still buys nothing: the + // principal is the resolved agent, which this engine does not permit. + let check = app + .clone() + .oneshot(authenticate_as( + Request::post("/api/conformance/check") + .header("X-Temper-Principal-Kind", "admin") + .header("Content-Type", "application/json") + .body(Body::from( + serde_json::json!({"entity_type": "Order", "session_id": "session-1"}) + .to_string(), + )) + .unwrap(), + "default", + agent_context(), + )) + .await + .unwrap(); + assert_eq!( + check.status(), + StatusCode::FORBIDDEN, + "declaring yourself admin must not substitute for a Cedar permit" + ); + + let export = app + .oneshot(authenticate_as( + Request::get("/api/ots/trajectories/traj-atif/atif") + .header("X-Temper-Principal-Kind", "admin") + .body(Body::empty()) + .unwrap(), + "default", + agent_context(), + )) + .await + .unwrap(); + assert_eq!(export.status(), StatusCode::FORBIDDEN); +} + +#[tokio::test] +async fn a_cedar_permitted_caller_is_let_through() { + // Positive control for the two 403 tests above: under an engine whose only + // permit is this read, an ordinary principal reaches both handlers, so the + // denials come from the gate rather than from a route that never resolves. + let (app, store) = policy_permitted_app().await; + seed_row( + &store, + "AddItem", + Some("Draft"), + Some("Draft"), + "2026-01-01T00:00:00Z", + ) + .await; + + let check = app + .clone() + .oneshot(authenticate_as( + Request::post("/api/conformance/check") + .header("Content-Type", "application/json") + .body(Body::from( + serde_json::json!({"entity_type": "Order", "session_id": "session-1"}) + .to_string(), + )) + .unwrap(), + "default", + agent_context(), + )) + .await + .unwrap(); + assert_eq!(check.status(), StatusCode::OK); + + let export = app + .oneshot(authenticate_as( + Request::get("/api/ots/trajectories/traj-atif/atif") + .body(Body::empty()) + .unwrap(), + "default", + agent_context(), + )) + .await + .unwrap(); + assert_eq!( + export.status(), + StatusCode::NOT_FOUND, + "authorized, and the trajectory simply does not exist in this app" + ); +} + +/// The OTS document a producer uploads: the id is top-level, as the model +/// defines it. +fn ots_upload(trajectory_id: &str) -> serde_json::Value { + serde_json::json!({ + "trajectory_id": trajectory_id, + "version": "0.1.0", + "metadata": { + "task_description": "place an order", + "timestamp_start": "2026-01-01T00:00:00Z", + "agent_id": "agent-1", + "outcome": "success", + "human_reviewed": false + }, + "context": {}, + "turns": [{ + "turn_id": 1, + "span_id": "span-1", + "timestamp": "2026-01-01T00:00:00Z", + "error": false, + "decisions": [{ + "decision_id": "decision-1", + "decision_type": "tool_selection", + "choice": {"action": "SubmitOrder"}, + "consequence": {"success": true} + }] + }] + }) +} + +#[tokio::test] +async fn a_trajectory_alone_cannot_make_a_session_pass() { + // The end-to-end shape of the fail-open: an uploaded trajectory whose one + // decision names a declared action (`ots_upload` sends `SubmitOrder`), + // against a session the kernel recorded nothing for, with the spec version + // pinned so the resolution is not the thing holding it back. Everything a + // caller controls is present and nothing the kernel controls is. + let (app, store) = test_app().await; + let registered = registered_spec_version(); + let mut data = ots_upload("traj-no-rows"); + data["metadata"]["spec_version"] = serde_json::json!(format!("sha256:{registered}")); + store + .persist_ots_trajectory(&OtsTrajectoryParams { + trajectory_id: "traj-no-rows", + tenant: "default", + agent_id: "agent-1", + session_id: "session-1", + outcome: "success", + turn_count: 1, + data: &data.to_string(), + }) + .await + .expect("persist OTS trajectory"); + + let response = app + .oneshot(admin_post( + "/api/conformance/check", + serde_json::json!({ + "entity_type": "Order", + "session_id": "session-1", + "trajectory_id": "traj-no-rows", + }), + Some("default"), + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let body = json_body(response).await; + assert_eq!(body["report"]["spec_resolution"], "pinned"); + assert_eq!(body["report"]["violations"], serde_json::json!([])); + assert_eq!(body["report"]["stats"]["actor_rows"], serde_json::json!(0)); + assert_eq!( + body["report"]["stats"]["ots_decisions_checked"], + serde_json::json!(1), + "the decision was looked at, which is what used to be mistaken for evidence" + ); + assert_eq!(body["report"]["verdict"], "indeterminate"); + assert_eq!( + body["report"]["passed"], + serde_json::json!(false), + "an uploaded trajectory is the agent's own account and cannot pass a run on its own" + ); + assert_eq!( + body["report"]["evidence_complete"], + serde_json::json!(false) + ); +} + +#[tokio::test] +async fn a_trajectory_stored_without_a_session_is_refused() { + // An upload that carried no `X-Session-Id` is tied to no run. Read as a + // wildcard it folded into whichever session the caller named, which is how + // one unattributed upload could be presented as evidence about every run. + let (app, store) = test_app().await; + seed_row( + &store, + "AddItem", + Some("Draft"), + Some("Draft"), + "2026-01-01T00:00:00Z", + ) + .await; + store + .persist_ots_trajectory(&OtsTrajectoryParams { + trajectory_id: "traj-sessionless", + tenant: "default", + agent_id: "agent-1", + session_id: "", + outcome: "success", + turn_count: 1, + data: &ots_upload("traj-sessionless").to_string(), + }) + .await + .expect("persist OTS trajectory"); + + let response = app + .oneshot(admin_post( + "/api/conformance/check", + serde_json::json!({ + "entity_type": "Order", + "session_id": "session-1", + "trajectory_id": "traj-sessionless", + }), + Some("default"), + )) + .await + .unwrap(); + + assert_eq!( + response.status(), + StatusCode::BAD_REQUEST, + "a trajectory tied to no run cannot be folded into this one" + ); + let detail = response_text(response).await; + assert!( + detail.contains("without a session"), + "the refusal must say why: {detail}" + ); +} + +#[tokio::test] +async fn an_uploaded_trajectory_is_addressable_by_the_id_it_was_uploaded_with() { + // The upload path and the read path must agree on the run's identity. + // Seeding storage directly would prove nothing about the POST handler. + let (app, _store) = test_app().await; + + let upload = app + .clone() + .oneshot({ + // In production the bearer edge lifts X-Session-Id onto the typed + // context; the handler no longer reads raw headers. + let mut request = Request::post("/api/ots/trajectories") + .header("Content-Type", "application/json") + .body(Body::from(ots_upload("traj-roundtrip").to_string())) + .unwrap(); + request.extensions_mut().insert( + temper_authz::AuthenticatedRequestContext::new( + temper_runtime::tenant::TenantId::default(), + temper_authz::SecurityContext::system(), + ) + .with_session_id(Some("session-42".to_string())), + ); + request + }) + .await + .unwrap(); + assert_eq!(upload.status(), StatusCode::ACCEPTED); + + let export = app + .oneshot(admin_get( + "/api/ots/trajectories/traj-roundtrip/atif", + Some("default"), + )) + .await + .unwrap(); + assert_eq!( + export.status(), + StatusCode::OK, + "the uploaded id must resolve to the stored run (read runs as System under the permissive engine; ownership is not what this test proves)" + ); + let body = json_body(export).await; + assert_eq!(body["trajectory_id"], "traj-roundtrip"); + assert_eq!(body["session_id"], "session-42"); +} + +#[tokio::test] +async fn an_upload_with_misaligned_token_signals_is_rejected() { + let (app, _store) = test_app().await; + let mut upload = ots_upload("traj-misaligned"); + // In-domain mask (0/1) with a length that disagrees with the completion: + // an out-of-domain value would be rejected by the earlier domain check and + // the alignment loop under test would never run. + upload["turns"][0]["completion_token_ids"] = serde_json::json!([4, 5]); + upload["turns"][0]["response_mask"] = serde_json::json!([1]); + upload["turns"][0]["logprobs"] = serde_json::json!([-0.1, -0.2, -0.3]); + + let response = app + .oneshot(authenticate_as( + Request::post("/api/ots/trajectories") + .header("Content-Type", "application/json") + .body(Body::from(upload.to_string())) + .unwrap(), + "default", + temper_authz::SecurityContext::system(), + )) + .await + .unwrap(); + + assert_eq!( + response.status(), + StatusCode::BAD_REQUEST, + "positionally inconsistent RL data must not be persisted" + ); + let body = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .expect("read rejection body"); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("completion_token_ids") && body.contains("aligned position for position"), + "the rejection must come from the length-alignment check, not the \ + mask-domain check: {body}" + ); +} + +#[tokio::test] +async fn a_session_ending_exactly_on_the_limit_is_not_reported_truncated() { + let (app, store) = test_app().await; + seed_row( + &store, + "AddItem", + Some("Draft"), + Some("Draft"), + "2026-01-01T00:00:00Z", + ) + .await; + seed_row( + &store, + "SubmitOrder", + Some("Draft"), + Some("Submitted"), + "2026-01-01T00:00:01Z", + ) + .await; + + let response = app + .oneshot(admin_post( + "/api/conformance/check", + serde_json::json!({ + "entity_type": "Order", + "session_id": "session-1", + "limit": 2, + "spec_version": registered_spec_version(), + }), + Some("default"), + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let body = json_body(response).await; + assert_eq!( + body["truncated"], + serde_json::json!(false), + "a complete session that happens to be limit-sized is complete" + ); + assert_eq!(body["report"]["verdict"], "pass"); + assert_eq!(body["report"]["passed"], serde_json::json!(true)); +} + +#[tokio::test] +async fn a_truncated_session_is_indeterminate_rather_than_passing() { + let (app, store) = test_app().await; + seed_row( + &store, + "AddItem", + Some("Draft"), + Some("Draft"), + "2026-01-01T00:00:00Z", + ) + .await; + seed_row( + &store, + "SubmitOrder", + Some("Draft"), + Some("Submitted"), + "2026-01-01T00:00:01Z", + ) + .await; + + let response = app + .oneshot(admin_post( + "/api/conformance/check", + serde_json::json!({"entity_type": "Order", "session_id": "session-1", "limit": 1}), + Some("default"), + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let body = json_body(response).await; + assert_eq!(body["truncated"], serde_json::json!(true)); + assert_eq!(body["report"]["verdict"], "indeterminate"); + assert_eq!( + body["report"]["passed"], + serde_json::json!(false), + "a consumer gating on `passed` must not accept a partially read run" + ); +} + +#[tokio::test] +async fn a_session_with_no_rows_is_indeterminate_rather_than_passing() { + let (app, _store) = test_app().await; + + let response = app + .oneshot(admin_post( + "/api/conformance/check", + serde_json::json!({"entity_type": "Order", "session_id": "no-such-session"}), + Some("default"), + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let body = json_body(response).await; + assert_eq!(body["report"]["verdict"], "indeterminate"); + assert_eq!(body["report"]["passed"], serde_json::json!(false)); +} + +#[tokio::test] +async fn a_run_under_another_spec_version_is_refused_rather_than_judged() { + let (app, store) = test_app().await; + seed_row( + &store, + "AddItem", + Some("Draft"), + Some("Draft"), + "2026-01-01T00:00:00Z", + ) + .await; + + let response = app + .oneshot(admin_post( + "/api/conformance/check", + serde_json::json!({ + "entity_type": "Order", + "session_id": "session-1", + "spec_version": "sha256:a-spec-that-is-not-loaded" + }), + Some("default"), + )) + .await + .unwrap(); + + assert_eq!( + response.status(), + StatusCode::CONFLICT, + "judging a run by a spec it never ran under is worse than refusing" + ); +} + +#[tokio::test] +async fn a_run_under_the_registered_spec_version_is_checked() { + let (app, store) = test_app().await; + seed_row( + &store, + "AddItem", + Some("Draft"), + Some("Draft"), + "2026-01-01T00:00:00Z", + ) + .await; + let registered = temper_store_turso::spec_content_hash(ORDER_IOA); + + let response = app + .oneshot(admin_post( + "/api/conformance/check", + serde_json::json!({ + "entity_type": "Order", + "session_id": "session-1", + "spec_version": format!("sha256:{registered}") + }), + Some("default"), + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let body = json_body(response).await; + assert_eq!( + body["spec_version"], registered, + "the report names the spec it was produced against" + ); +} + +#[tokio::test] +async fn an_ots_trajectory_declaring_another_spec_version_is_refused() { + let (app, store) = test_app().await; + seed_row( + &store, + "AddItem", + Some("Draft"), + Some("Draft"), + "2026-01-01T00:00:00Z", + ) + .await; + let mut data = ots_upload("traj-oldspec"); + data["metadata"]["spec_version"] = serde_json::json!("sha256:v1-long-gone"); + store + .persist_ots_trajectory(&OtsTrajectoryParams { + trajectory_id: "traj-oldspec", + tenant: "default", + agent_id: "agent-1", + session_id: "session-1", + outcome: "success", + turn_count: 1, + data: &data.to_string(), + }) + .await + .expect("persist OTS trajectory"); + + let response = app + .oneshot(admin_post( + "/api/conformance/check", + serde_json::json!({ + "entity_type": "Order", + "session_id": "session-1", + "trajectory_id": "traj-oldspec" + }), + Some("default"), + )) + .await + .unwrap(); + + assert_eq!( + response.status(), + StatusCode::CONFLICT, + "the run records which spec governed it, and it is not the loaded one" + ); +} + +#[tokio::test] +async fn a_run_naming_no_spec_version_is_indeterminate_rather_than_passing() { + // Nothing says this run executed under the spec that is registered now, and + // a submit replaces a spec rather than versioning it. Agreeing with + // whatever is loaded today is not evidence about what ran yesterday. + let (app, store) = test_app().await; + seed_row( + &store, + "AddItem", + Some("Draft"), + Some("Draft"), + "2026-01-01T00:00:00Z", + ) + .await; + + let response = app + .oneshot(admin_post( + "/api/conformance/check", + serde_json::json!({"entity_type": "Order", "session_id": "session-1"}), + Some("default"), + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let body = json_body(response).await; + assert_eq!(body["report"]["spec_resolution"], "unresolved"); + assert_eq!(body["report"]["verdict"], "indeterminate"); + assert_eq!( + body["report"]["passed"], + serde_json::json!(false), + "a consumer gating on `passed` must not accept a run whose governing spec is unknown" + ); + assert_eq!( + body["report"]["evidence_complete"], + serde_json::json!(false) + ); + assert_eq!( + body["report"]["violations"], + serde_json::json!([]), + "an unresolved spec is missing evidence, not a disagreement" + ); +} + +#[tokio::test] +async fn a_request_cannot_override_the_spec_version_the_trajectory_pins() { + // The request body is the one input a caller chooses freely. Letting it + // pick the spec turns "check this run against its spec" into "check this + // run against whichever spec makes it pass". + let (app, store) = test_app().await; + seed_row( + &store, + "AddItem", + Some("Draft"), + Some("Draft"), + "2026-01-01T00:00:00Z", + ) + .await; + let mut data = ots_upload("traj-pinned"); + data["metadata"]["spec_version"] = serde_json::json!("sha256:v1-long-gone"); + store + .persist_ots_trajectory(&OtsTrajectoryParams { + trajectory_id: "traj-pinned", + tenant: "default", + agent_id: "agent-1", + session_id: "session-1", + outcome: "success", + turn_count: 1, + data: &data.to_string(), + }) + .await + .expect("persist OTS trajectory"); + + let response = app + .oneshot(admin_post( + "/api/conformance/check", + serde_json::json!({ + "entity_type": "Order", + "session_id": "session-1", + "trajectory_id": "traj-pinned", + // The registered spec: without the pin this would be accepted. + "spec_version": registered_spec_version(), + }), + Some("default"), + )) + .await + .unwrap(); + + assert_eq!( + response.status(), + StatusCode::CONFLICT, + "the recorded version is the run's provenance and a request cannot replace it" + ); + let detail = response_text(response).await; + assert!( + detail.contains("v1-long-gone"), + "the refusal must name the version the run actually executed under: {detail}" + ); +} + +#[tokio::test] +async fn a_request_agreeing_with_the_pinned_spec_version_is_checked() { + // Same version, one written bare and one `sha256:`-prefixed: two spellings + // of one spec must not read as a conflict. + let (app, store) = test_app().await; + seed_row( + &store, + "AddItem", + Some("Draft"), + Some("Draft"), + "2026-01-01T00:00:00Z", + ) + .await; + let registered = registered_spec_version(); + let mut data = ots_upload("traj-current"); + data["metadata"]["spec_version"] = serde_json::json!(format!("sha256:{registered}")); + store + .persist_ots_trajectory(&OtsTrajectoryParams { + trajectory_id: "traj-current", + tenant: "default", + agent_id: "agent-1", + session_id: "session-1", + outcome: "success", + turn_count: 1, + data: &data.to_string(), + }) + .await + .expect("persist OTS trajectory"); + + let response = app + .oneshot(admin_post( + "/api/conformance/check", + serde_json::json!({ + "entity_type": "Order", + "session_id": "session-1", + "trajectory_id": "traj-current", + "spec_version": registered, + }), + Some("default"), + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let body = json_body(response).await; + assert_eq!(body["report"]["spec_resolution"], "pinned"); +} + +/// Seed one legal row and store a trajectory pinning `spec_version`. +async fn app_with_pinned_trajectory( + trajectory_id: &str, + spec_version: &str, +) -> (Router, TursoEventStore) { + let (app, store) = test_app().await; + seed_row( + &store, + "AddItem", + Some("Draft"), + Some("Draft"), + "2026-01-01T00:00:00Z", + ) + .await; + let mut data = ots_upload(trajectory_id); + data["metadata"]["spec_version"] = serde_json::json!(spec_version); + store + .persist_ots_trajectory(&OtsTrajectoryParams { + trajectory_id, + tenant: "default", + agent_id: "agent-1", + session_id: "session-1", + outcome: "success", + turn_count: 1, + data: &data.to_string(), + }) + .await + .expect("persist OTS trajectory"); + (app, store) +} + +/// The kernel-vocabulary pin katagami stamps: the entity, then a truncated +/// digest. +fn entity_qualified_pin(digest_len: usize) -> String { + let registered = registered_spec_version(); + format!("Order@sha256:{}", ®istered[..digest_len]) +} + +#[tokio::test] +async fn a_run_pinned_by_entity_and_short_digest_is_checked() { + // The point of the form: a spec identified by name plus a truncated + // digest resolves to the same registered spec as the full hash would. + let (app, _store) = + app_with_pinned_trajectory("traj-short-pin", &entity_qualified_pin(12)).await; + + let response = app + .oneshot(admin_post( + "/api/conformance/check", + serde_json::json!({ + "entity_type": "Order", + "session_id": "session-1", + "trajectory_id": "traj-short-pin", + }), + Some("default"), + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let body = json_body(response).await; + assert_eq!(body["report"]["spec_resolution"], "pinned"); + assert_eq!( + body["spec_version"], + registered_spec_version(), + "the report still names the full digest it was produced against" + ); +} + +#[tokio::test] +async fn a_short_pin_and_the_full_hash_are_not_read_as_a_conflict() { + // The trajectory pins by prefix, the request names the whole hash. Both + // resolve to the registered spec, so neither contradicts the other. + let (app, _store) = + app_with_pinned_trajectory("traj-mixed-forms", &entity_qualified_pin(16)).await; + + let response = app + .oneshot(admin_post( + "/api/conformance/check", + serde_json::json!({ + "entity_type": "Order", + "session_id": "session-1", + "trajectory_id": "traj-mixed-forms", + "spec_version": registered_spec_version(), + }), + Some("default"), + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + json_body(response).await["report"]["spec_resolution"], + "pinned" + ); +} + +#[tokio::test] +async fn a_digest_truncated_below_the_floor_is_refused_as_under_specified() { + // Eleven characters names a family of specs rather than one, so it is + // refused with what to send instead — not silently resolved, and not + // reported as the run having executed under a spec that is gone. + let (app, _store) = + app_with_pinned_trajectory("traj-too-short", &entity_qualified_pin(11)).await; + + let response = app + .oneshot(admin_post( + "/api/conformance/check", + serde_json::json!({ + "entity_type": "Order", + "session_id": "session-1", + "trajectory_id": "traj-too-short", + }), + Some("default"), + )) + .await + .unwrap(); + + assert_eq!( + response.status(), + StatusCode::BAD_REQUEST, + "an under-specified pin is a bad request, not a version conflict" + ); + let detail = response_text(response).await; + assert!( + detail.contains("12") && detail.contains("/observe/specs/Order"), + "the refusal must say how long a digest to send and where to read it: {detail}" + ); +} + +#[tokio::test] +async fn a_pin_naming_another_entity_is_refused() { + let (app, _store) = app_with_pinned_trajectory( + "traj-wrong-entity", + &format!("Invoice@sha256:{}", registered_spec_version()), + ) + .await; + + let response = app + .oneshot(admin_post( + "/api/conformance/check", + serde_json::json!({ + "entity_type": "Order", + "session_id": "session-1", + "trajectory_id": "traj-wrong-entity", + }), + Some("default"), + )) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + let detail = response_text(response).await; + assert!( + detail.contains("different entity"), + "the refusal must say the pin belongs to another actor: {detail}" + ); +} + +#[tokio::test] +async fn a_short_pin_for_a_spec_that_is_gone_is_still_refused() { + // Prefix matching widens which spellings name the registered spec. It must + // not widen which runs are accepted: a prefix of some other digest still + // has no spec to check against. + let (app, _store) = + app_with_pinned_trajectory("traj-gone-prefix", "Order@sha256:0123456789abcdef0123").await; + + let response = app + .oneshot(admin_post( + "/api/conformance/check", + serde_json::json!({ + "entity_type": "Order", + "session_id": "session-1", + "trajectory_id": "traj-gone-prefix", + }), + Some("default"), + )) + .await + .unwrap(); + + assert_eq!( + response.status(), + StatusCode::CONFLICT, + "a prefix that names no registered spec is still a version conflict" + ); +} + +#[tokio::test] +async fn a_trajectory_with_no_steps_is_not_exported_as_stepless_atif() { + let (app, store) = test_app().await; + let data = r#"{"trajectory_id":"traj-empty","version":"0.1.0","metadata":{"task_description":"t","timestamp_start":"2026-01-01T00:00:00Z","agent_id":"a","outcome":"success","human_reviewed":false},"context":{}}"#; + store + .persist_ots_trajectory(&OtsTrajectoryParams { + trajectory_id: "traj-empty", + tenant: "default", + agent_id: "agent-1", + session_id: "session-1", + outcome: "success", + turn_count: 0, + data, + }) + .await + .expect("persist OTS trajectory"); + + let response = app + .oneshot(admin_get( + "/api/ots/trajectories/traj-empty/atif", + Some("default"), + )) + .await + .unwrap(); + + assert_eq!( + response.status(), + StatusCode::UNPROCESSABLE_ENTITY, + "ATIF v1.7 requires at least one step; a stepless document is not valid ATIF" + ); +} + +/// A denied `POST /api/authorize` pre-flight must never poison another run's +/// conformance verdict. The probe's action, resource type, and session are all +/// caller-chosen, so its denial row is written `spec_governed = false` and the +/// walk skips it — otherwise any agent could flip a victim session's verdict +/// by probing garbage actions under that session id. +#[tokio::test] +async fn a_denied_preflight_probe_cannot_poison_a_sessions_verdict() { + let (app, store) = policy_permitted_app().await; + + // A genuine governed dispatch in the victim session: verdict passes. + seed_row( + &store, + "AddItem", + Some("Draft"), + Some("Draft"), + "2026-01-01T00:00:00Z", + ) + .await; + + // The attacker's pre-flight: an agent credential carrying the victim's + // session (in production the session rides the correlation header), probing + // an undeclared action against the victim's entity type. The engine only + // permits read_trajectories, so Cedar denies the probe. + let mut probe = Request::post("/api/authorize") + .header("Content-Type", "application/json") + .body(Body::from( + serde_json::json!({ + "agent_id": "agent-1", + "action": "GarbageAction", + "resource_type": "Order", + "resource_id": "order-1", + }) + .to_string(), + )) + .unwrap(); + probe.extensions_mut().insert( + temper_authz::AuthenticatedRequestContext::new( + temper_runtime::tenant::TenantId::default(), + agent_context(), + ) + .with_session_id(Some("session-1".to_string())), + ); + // The oracle answers 200 with the decision; denial is in the body. + let denied = app.clone().oneshot(probe).await.unwrap(); + assert_eq!(denied.status(), StatusCode::OK); + let decision = json_body(denied).await; + assert_eq!( + decision["allowed"], + serde_json::json!(false), + "the probe must be denied: {decision}" + ); + + // Wait for the denial row to drain into the store, then pin the call site: + // it must arrive marked ungoverned. + let mut denial_row = None; + for _ in 0..200 { + let rows = store + .query_trajectories_by_session("session-1", Some("default"), None, 50) + .await + .expect("query session rows"); + if let Some(row) = rows.iter().find(|r| r.action == "GarbageAction") { + denial_row = Some(row.clone()); + break; + } + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + } + let denial_row = denial_row.expect("the denied probe's trajectory row must be persisted"); + assert_eq!( + denial_row.spec_governed, + Some(false), + "a pre-flight denial row must be explicitly ungoverned" + ); + + // And end to end: the victim session's verdict is untouched by the probe. + let check = app + .oneshot(admin_post( + "/api/conformance/check", + serde_json::json!({ + "entity_type": "Order", + "session_id": "session-1", + "spec_version": registered_spec_version(), + }), + Some("default"), + )) + .await + .unwrap(); + assert_eq!(check.status(), StatusCode::OK); + let body = json_body(check).await; + assert_eq!( + body["report"]["passed"], + serde_json::json!(true), + "a denied pre-flight probe must not add violations: {body}" + ); + assert_eq!(body["report"]["violations"], serde_json::json!([])); + assert_eq!( + body["report"]["stats"]["non_governed_rows_skipped"], + serde_json::json!(1), + "the probe's row must be present and skipped, not absent: {body}" + ); +} diff --git a/crates/temper-server/src/authz/edge.rs b/crates/temper-server/src/authz/edge.rs new file mode 100644 index 000000000..24a4a9bd9 --- /dev/null +++ b/crates/temper-server/src/authz/edge.rs @@ -0,0 +1,251 @@ +//! Ingress defense for caller-controlled Temper headers (ADR-0157). +//! +//! Cedar authority is carried in an immutable +//! [`temper_authz::AuthenticatedRequestContext`] request extension. Headers +//! never materialize that authority. This middleware removes the closed +//! `x-temper-*` authority namespace while retaining the two correlation-only +//! namespaces consumed by tracing and workflow observability. + +use axum::extract::Request; +use axum::http::{HeaderName, Method, StatusCode}; +use axum::middleware::Next; +use axum::response::{IntoResponse, Response}; + +const CORRELATION_PREFIXES: [&str; 2] = ["x-temper-observe-", "x-temper-workflow-"]; + +/// Return whether a header belongs to the caller-controlled authority +/// namespace that must be removed before dispatch. +/// +/// `HeaderName` values are normalized to lowercase. The allowlist is narrow on +/// purpose: new `x-temper-*` headers are authority by default unless explicitly +/// placed in a correlation-only namespace. +pub(crate) fn is_caller_authority_header(name: &HeaderName) -> bool { + let name = name.as_str(); + name.starts_with("x-temper-") + && !CORRELATION_PREFIXES + .iter() + .any(|prefix| name.starts_with(prefix)) +} + +/// Remove caller-supplied `x-temper-*` authority headers. +/// +/// Credential resolution communicates identity through +/// [`temper_authz::AuthenticatedRequestContext`], never by adding headers back +/// after this layer. `x-temper-observe-*` and `x-temper-workflow-*` survive only +/// as correlation metadata and are not Cedar inputs. +pub async fn strip_inbound_identity_headers(mut request: Request, next: Next) -> Response { + let names = request + .headers() + .keys() + .filter(|name| is_caller_authority_header(name)) + .cloned() + .collect::>(); + for name in names { + request.headers_mut().remove(name); + } + next.run(request).await +} + +/// Return whether the kernel route has its own non-Class-A admission boundary. +/// +/// This is the single classifier shared by the kernel guard and the outer +/// platform bearer edge. Keep it exact: webhook ingress is admitted by its +/// Class B verifier; the remaining routes expose only bootstrap metadata or +/// immutable static assets. +pub fn is_public_kernel_request(method: &Method, path: &str) -> bool { + if method == Method::GET + && matches!( + path, + "/tdata" + | "/tdata/" + | "/tdata/$metadata" + | "/temper-client.js" + | "/static/temper-client.js" + | "/genesis" + | "/genesis/" + ) + { + return true; + } + (matches!(*method, Method::GET | Method::POST) && path.starts_with("/webhooks/")) + || (method == Method::GET && path.starts_with("/genesis/")) +} + +/// Reject protected kernel routes that lack authenticated typed authority. +/// +/// The guard is installed by [`crate::build_router`] itself so direct embedders +/// cannot accidentally expose a handler that reconstructs identity from HTTP +/// headers. Webhook ingress remains governed by its Class B admission boundary. +pub async fn require_authenticated_request_context(request: Request, next: Next) -> Response { + if is_public_kernel_request(request.method(), request.uri().path()) + || request + .extensions() + .get::() + .is_some() + { + return next.run(request).await; + } + + ( + StatusCode::UNAUTHORIZED, + axum::Json(serde_json::json!({ + "error": { + "code": "AuthenticationRequired", + "message": "A valid tenant credential is required" + } + })), + ) + .into_response() +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::Router; + use axum::body::Body; + use axum::http::{HeaderMap, Request as HttpRequest, StatusCode}; + use axum::routing::get; + use tower::ServiceExt; + + async fn echo_temper_headers(headers: HeaderMap) -> String { + let mut names = headers + .keys() + .map(|name| name.as_str().to_string()) + .filter(|name| name.starts_with("x-temper-")) + .collect::>(); + names.sort(); + names.join(",") + } + + async fn surviving_headers(request: HttpRequest) -> String { + let app = Router::new() + .route("/echo", get(echo_temper_headers)) + .layer(axum::middleware::from_fn(strip_inbound_identity_headers)); + let response = app.oneshot(request).await.expect("request should run"); + assert_eq!(response.status(), StatusCode::OK); + let body = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .expect("response body should be readable"); + String::from_utf8(body.to_vec()).expect("header names should be UTF-8") + } + + #[tokio::test] + async fn strips_complete_authority_namespace_including_legacy_marker() { + let request = HttpRequest::get("/echo") + .header("x-temper-principal-kind", "admin") + .header("x-temper-principal-id", "attacker") + .header("x-temper-agent-type", "operator") + .header("x-temper-agent-role", "supervisor") + .header("x-temper-acting-for", "victim") + .header("x-temper-principal-scopes", "repo:push") + .header("x-temper-attr-approvallimit", "999999") + .header("x-temper-action-context", "composite:App.Fork") + .header("x-temper-ctx-agenttypeverified", "true") + .header("x-temper-internal-trusted-principal", "1") + .header("x-temper-future-authority", "also stripped") + .body(Body::empty()) + .expect("request should build"); + + assert_eq!(surviving_headers(request).await, ""); + } + + #[tokio::test] + async fn preserves_only_observe_and_workflow_correlation_namespaces() { + let request = HttpRequest::get("/echo") + .header("x-temper-observe-session-id", "session-1") + .header("x-temper-observe-intent", "approve invoice") + .header("x-temper-workflow-run-id", "workflow-1") + .header("x-temper-workflow-root-entity-type", "Job") + .header("x-temper-principal-kind", "admin") + .body(Body::empty()) + .expect("request should build"); + + assert_eq!( + surviving_headers(request).await, + "x-temper-observe-intent,x-temper-observe-session-id,x-temper-workflow-root-entity-type,x-temper-workflow-run-id" + ); + } + + #[test] + fn classifier_leaves_non_temper_transport_headers_alone() { + for name in [ + "authorization", + "content-type", + "x-tenant-id", + "traceparent", + ] { + assert!(!is_caller_authority_header(&HeaderName::from_static(name))); + } + } + + #[test] + fn public_route_allowlist_is_exact() { + for path in [ + "/tdata", + "/tdata/", + "/tdata/$metadata", + "/temper-client.js", + "/static/temper-client.js", + "/genesis", + "/genesis/app.js", + "/webhooks/tenant/provider", + ] { + assert!(is_public_kernel_request(&Method::GET, path), "{path}"); + } + for path in [ + "/tdata/Orders", + "/tdata/$hints", + "/tdata/$events", + "/api/authorize", + "/observe/decisions", + "/_admin/reload", + "/not-a-static-route", + ] { + assert!(!is_public_kernel_request(&Method::GET, path), "{path}"); + } + assert!(!is_public_kernel_request(&Method::POST, "/tdata")); + assert!(!is_public_kernel_request( + &Method::DELETE, + "/webhooks/tenant/provider" + )); + } + + #[tokio::test] + async fn protected_route_requires_typed_context_not_identity_headers() { + let app = Router::new() + .route("/protected", get(|| async { StatusCode::OK })) + .layer(axum::middleware::from_fn( + require_authenticated_request_context, + )); + let forged = HttpRequest::get("/protected") + .header("x-temper-principal-kind", "admin") + .header("x-temper-principal-id", "attacker") + .body(Body::empty()) + .expect("request should build"); + assert_eq!( + app.clone() + .oneshot(forged) + .await + .expect("request should run") + .status(), + StatusCode::UNAUTHORIZED + ); + + let mut authenticated = HttpRequest::get("/protected") + .body(Body::empty()) + .expect("request should build"); + authenticated + .extensions_mut() + .insert(temper_authz::AuthenticatedRequestContext::new( + temper_runtime::tenant::TenantId::new("tenant-a"), + temper_authz::SecurityContext::from_resolved_identity("agent-1", "operator", None), + )); + assert_eq!( + app.oneshot(authenticated) + .await + .expect("request should run") + .status(), + StatusCode::OK + ); + } +} diff --git a/crates/temper-server/src/authz/helpers.rs b/crates/temper-server/src/authz/helpers.rs index 87f4ca37e..50389f3d3 100644 --- a/crates/temper-server/src/authz/helpers.rs +++ b/crates/temper-server/src/authz/helpers.rs @@ -6,73 +6,63 @@ use std::collections::BTreeMap; -use axum::http::{HeaderMap, StatusCode}; -use temper_authz::SecurityContext; +use axum::http::StatusCode; +use temper_authz::{AuthenticatedRequestContext, SecurityContext}; use temper_runtime::scheduler::{sim_now, sim_uuid}; use temper_runtime::tenant::TenantId; use crate::request_context::AgentContext; use crate::state::{PendingDecision, TrajectoryEntry, TrajectorySource}; -/// Extract `X-Temper-*` headers from an axum `HeaderMap` into `(key, value)` pairs -/// suitable for `SecurityContext::from_headers`. -pub(crate) fn extract_temper_headers(headers: &HeaderMap) -> Vec<(String, String)> { - headers - .iter() - .filter_map(|(name, value)| { - let key = name.as_str().to_lowercase(); - if key == "x-temper-action-context" { - None - } else if key.starts_with("x-temper-") { - value.to_str().ok().map(|v| (key, v.to_string())) - } else { - None - } - }) - .collect() +/// Require the credential-derived request context installed by the auth edge. +/// +/// Protected handlers use `Option>` so a missing edge context is +/// an explicit `401` rather than axum's generic missing-extension rejection. +pub(crate) fn require_authenticated_context( + authenticated: Option<&AuthenticatedRequestContext>, +) -> Result<&AuthenticatedRequestContext, StatusCode> { + authenticated.ok_or(StatusCode::UNAUTHORIZED) } -/// Build a `SecurityContext` from request headers, optionally enriched with -/// agent identity from an `AgentContext`. -pub(crate) fn security_context_from_headers( - headers: &HeaderMap, - agent_id: Option<&str>, - session_id: Option<&str>, - agent_type: Option<&str>, -) -> SecurityContext { - let temper_headers = extract_temper_headers(headers); - SecurityContext::from_headers(&temper_headers) - .with_agent_context(agent_id, session_id, agent_type) +/// Ensure a tenant path parameter matches the credential-bound tenant. +/// +/// The path remains useful for route shape, but it is never an authority input. +/// A credential resolved for one tenant cannot operate on another tenant's +/// path-scoped resource. +pub(crate) fn require_tenant_match<'a>( + authenticated: &'a AuthenticatedRequestContext, + requested_tenant: &str, +) -> Result<&'a TenantId, StatusCode> { + if authenticated.tenant().as_str() != requested_tenant { + tracing::warn!( + authenticated_tenant = %authenticated.tenant(), + requested_tenant, + "credential tenant does not match protected route tenant" + ); + return Err(StatusCode::UNAUTHORIZED); + } + Ok(authenticated.tenant()) } /// Check Cedar authorization for observe endpoints. /// -/// Admin and System principals bypass the check. Other principals must have the -/// specified `action` on `resource_type`. Returns `Ok(())` if authorized or +/// Every principal must have the specified `action` on `resource_type`. +/// System authority is represented by the built-in Cedar policy, not a caller +/// side channel. Returns `Ok(())` if authorized or /// `Err(StatusCode::FORBIDDEN)` if denied. pub(crate) fn require_observe_auth( state: &crate::state::ServerState, - headers: &HeaderMap, + authenticated: &AuthenticatedRequestContext, action: &str, resource_type: &str, ) -> Result<(), axum::http::StatusCode> { - let security_ctx = security_context_from_headers(headers, None, None, None); - if matches!( - security_ctx.principal.kind, - temper_authz::PrincipalKind::Admin | temper_authz::PrincipalKind::System - ) { - return Ok(()); - } - let tenant = headers - .get("x-tenant-id") - .and_then(|v| v.to_str().ok()) - .unwrap_or("system"); + let security_ctx = authenticated.security_context(); if let Err(denial) = state.authorize_with_context( - &security_ctx, + security_ctx, action, resource_type, - &std::collections::BTreeMap::new(), - tenant, + &BTreeMap::new(), + authenticated.tenant().as_str(), ) { tracing::warn!(reason = %denial, action, resource_type, "unauthorized observe access"); return Err(axum::http::StatusCode::FORBIDDEN); @@ -82,45 +72,58 @@ pub(crate) fn require_observe_auth( /// Resolve the tenant scope for an observe endpoint. /// -/// Returns `Some(tenant)` when results should be filtered to a single tenant, -/// or `None` when the caller is authorized for a cross-tenant admin view. -/// -/// - If `X-Tenant-Id` is present → filter to that tenant. -/// - If missing and principal is Admin/System → cross-tenant view (`None`). -/// - If missing in multi-tenant mode for non-admin → `403 Forbidden`. +/// Every credential is tenant-bound, including operator credentials. Observe +/// handlers therefore always filter to the authenticated tenant; a cross-tenant +/// view requires an explicit higher-level aggregation rather than an absent +/// tenant header. #[allow(dead_code)] // False positive: used by observe/ handlers via crate::authz re-export -pub(crate) fn observe_tenant_scope( - state: &crate::state::ServerState, - headers: &axum::http::HeaderMap, -) -> Result, axum::http::StatusCode> { - // If the caller provided an explicit tenant, always scope to it. - if let Some(val) = headers.get("x-tenant-id") - && let Ok(s) = val.to_str() - && !s.is_empty() - { - return Ok(Some(TenantId::new(s))); - } +pub(crate) fn observe_tenant_scope(authenticated: &AuthenticatedRequestContext) -> &TenantId { + authenticated.tenant() +} - // No tenant header — admin/system get cross-tenant view. - let security_ctx = security_context_from_headers(headers, None, None, None); - if matches!( - security_ctx.principal.kind, - temper_authz::PrincipalKind::Admin | temper_authz::PrincipalKind::System - ) { - return Ok(None); - } +/// One exact Cedar resource check derived from authenticated request authority. +#[cfg(feature = "observe")] +pub(crate) struct ResourceAuthorization<'a> { + pub action: &'a str, + pub resource_type: &'a str, + pub resource_id: &'a str, + pub resource_attrs: BTreeMap, +} - // Non-admin without tenant in multi-tenant mode: reject. - if !state.single_tenant_mode { +/// Require Cedar authorization for one credential-tenant resource. +/// +/// The tenant and principal always come from [`AuthenticatedRequestContext`]. +/// Callers may add resource-specific attributes, while this helper guarantees +/// that Cedar receives the exact resource id used by the operation. +#[cfg(feature = "observe")] +pub(crate) fn require_resource_authorization( + state: &crate::state::ServerState, + authenticated: &AuthenticatedRequestContext, + mut input: ResourceAuthorization<'_>, +) -> Result<(), StatusCode> { + input.resource_attrs.insert( + "id".to_string(), + serde_json::Value::String(input.resource_id.to_string()), + ); + if let Err(denial) = state.authorize_with_context( + authenticated.security_context(), + input.action, + input.resource_type, + &input.resource_attrs, + authenticated.tenant().as_str(), + ) { tracing::warn!( - principal = %security_ctx.principal.id, - "non-admin observe request without X-Tenant-Id in multi-tenant mode" + reason = %denial, + tenant = %authenticated.tenant(), + principal_id = %authenticated.security_context().principal.id, + action = input.action, + resource_type = input.resource_type, + resource_id = input.resource_id, + "credential-bound resource authorization denied" ); - return Err(axum::http::StatusCode::FORBIDDEN); + return Err(StatusCode::FORBIDDEN); } - - // Single-tenant compat: cross-tenant view for all principals. - Ok(None) + Ok(()) } /// Input for recording an authorization denial. @@ -145,6 +148,29 @@ pub(crate) struct DenialInput<'a> { pub module_name: Option, /// Entity status at the time of denial. pub from_status: Option, + /// Caller-supplied intent (`X-Intent`) for the denied request. + /// + /// A denial without the intent behind it tells the Evolution Engine what + /// was blocked but not what the agent was trying to accomplish, which is + /// the half that drives policy proposals. + pub intent: Option, + /// Whether the denied operation was a spec-governed dispatch. + /// + /// `Some(false)` marks rows the conformance checker must never judge: + /// pre-flight probes and management-plane denials, whose entity type, + /// action, and session can be caller-chosen. Left `None` only by the OData + /// entity-dispatch guards, where the row describes a genuine attempted + /// dispatch of a registered action (`crate::conformance::walk::row_disposition` + /// walks `None` as actor execution, so `None` is an assertion that the row + /// belongs in a run's verdict). + pub spec_governed: Option, + /// Session the gate resolved for this request, when it had one. + /// + /// Passed in rather than recomputed: the caller-declared session rides the + /// request context now (it must never reach Cedar), so recomputing it from + /// `context_attrs` would leave exactly the session-scoped denials the gate + /// admitted with no session on their record. + pub session_id: Option, } /// Input for a resumable management mutation authorization check. @@ -169,24 +195,23 @@ pub(crate) struct GovernedMutationAuth<'a> { #[allow(unused)] // Staged for governed management endpoints after the latency package lands. pub(crate) async fn require_governed_mutation_auth( state: &crate::state::ServerState, - headers: &HeaderMap, + authenticated: &AuthenticatedRequestContext, mut input: GovernedMutationAuth<'_>, ) -> Option<(StatusCode, String)> { - let security_ctx = security_context_from_headers(headers, None, None, None); - if matches!( - security_ctx.principal.kind, - temper_authz::PrincipalKind::Admin - ) { - return None; + if require_tenant_match(authenticated, input.tenant).is_err() { + return Some(( + StatusCode::UNAUTHORIZED, + "credential tenant does not match protected route tenant".to_string(), + )); } - + let security_ctx = authenticated.security_context(); input .resource_attrs .entry("id".to_string()) .or_insert_with(|| serde_json::Value::String(input.resource_id.to_string())); let Err(denial) = state.authorize_with_context( - &security_ctx, + security_ctx, input.action, input.resource_type, &input.resource_attrs, @@ -196,10 +221,16 @@ pub(crate) async fn require_governed_mutation_auth( }; let reason = denial.to_string(); - let session_id = security_ctx - .context_attrs - .get("sessionId") - .and_then(|v| v.as_str()); + // Either source is valid here, and neither is an authorization input: the + // request context carries the caller's declared session (telemetry only), + // while `context_attrs` carries one a server-side path minted itself — the + // WASM/agent dispatch contexts do this, and their session is not a header. + let session_id = authenticated.session_id().or_else(|| { + security_ctx + .context_attrs + .get("sessionId") + .and_then(|value| value.as_str()) + }); if matches!( security_ctx.principal.kind, temper_authz::PrincipalKind::Agent @@ -211,7 +242,7 @@ pub(crate) async fn require_governed_mutation_auth( state, DenialInput { tenant: input.tenant, - security_ctx: &security_ctx, + security_ctx, agent_id_override: None, action: input.action, resource_type: input.resource_type, @@ -220,6 +251,10 @@ pub(crate) async fn require_governed_mutation_auth( reason: &reason, module_name: input.module_name.map(str::to_string), from_status: input.from_status.map(str::to_string), + intent: authenticated.intent().map(str::to_string), + session_id: session_id.map(str::to_string), + // Management-plane denial, not a spec-governed dispatch. + spec_governed: Some(false), }, ) .await; @@ -261,12 +296,15 @@ pub(crate) async fn record_authz_denial( .agent_id_override .unwrap_or(input.security_ctx.principal.id.as_str()); let denied_module = input.module_name.clone(); - let session_id = input - .security_ctx - .context_attrs - .get("sessionId") - .and_then(|v| v.as_str()) - .map(|s| s.to_string()); + let denial_request_body = input.resource_attrs.clone(); + let session_id = input.session_id.clone().or_else(|| { + input + .security_ctx + .context_attrs + .get("sessionId") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()) + }); let mut pd = PendingDecision::from_denial( input.tenant, @@ -358,11 +396,15 @@ pub(crate) async fn record_authz_denial( denied_resource: Some(format!("{}:{}", input.resource_type, input.resource_id)), denied_module, source: Some(TrajectorySource::Authz), - spec_governed: None, + spec_governed: input.spec_governed, agent_type: input.security_ctx.principal.agent_type.clone(), - request_body: None, - intent: None, + // The Cedar-evaluated resource attributes are the request payload for + // an authorization decision; without them a denial cannot be replayed + // against a revised policy. + request_body: Some(denial_request_body), + intent: input.intent.clone(), matched_policy_ids: None, + capture_seq: None, }; if !state.enqueue_trajectory_entry(traj.clone()) { tracing::warn!("failed to enqueue authz trajectory"); @@ -399,117 +441,35 @@ pub(crate) async fn record_authz_denial( #[cfg(test)] mod tests { use super::*; - use axum::http::header::HeaderValue as AxumHeaderValue; - use axum::http::{HeaderMap, HeaderName, HeaderValue}; - use serde_json::Value; - use temper_authz::PrincipalKind; + use temper_authz::AuthenticatedRequestContext; #[test] - fn extract_temper_headers_filters_correctly() { - let mut headers = HeaderMap::new(); - headers.insert( - HeaderName::from_static("x-temper-principal"), - HeaderValue::from_static("agent-007"), - ); - headers.insert( - HeaderName::from_static("x-temper-session"), - HeaderValue::from_static("sess-123"), - ); - headers.insert( - HeaderName::from_static("content-type"), - HeaderValue::from_static("application/json"), - ); - headers.insert( - HeaderName::from_static("authorization"), - HeaderValue::from_static("Bearer tok"), + fn missing_authenticated_context_is_unauthorized() { + assert_eq!( + require_authenticated_context(None).unwrap_err(), + StatusCode::UNAUTHORIZED ); - - let result = extract_temper_headers(&headers); - assert_eq!(result.len(), 2); - assert!(result.contains(&("x-temper-principal".to_string(), "agent-007".to_string()))); - assert!(result.contains(&("x-temper-session".to_string(), "sess-123".to_string()))); } #[test] - fn extract_temper_headers_empty() { - let headers = HeaderMap::new(); - assert!(extract_temper_headers(&headers).is_empty()); - } + fn typed_context_preserves_resolved_identity_and_tenant() { + let authenticated = + AuthenticatedRequestContext::new(TenantId::new("tenant-a"), SecurityContext::system()); - #[test] - fn extract_temper_headers_case_insensitive() { - let mut headers = HeaderMap::new(); - headers.insert( - HeaderName::from_static("x-temper-test"), - HeaderValue::from_static("val"), - ); - let result = extract_temper_headers(&headers); - assert_eq!(result.len(), 1); - assert_eq!(result[0].0, "x-temper-test"); - } - - #[test] - fn security_context_from_headers_preserves_agent_type_from_http_header() { - let mut headers = HeaderMap::new(); - headers.insert( - "x-temper-principal-id", - AxumHeaderValue::from_static("bot-1"), - ); - headers.insert( - "x-temper-principal-kind", - AxumHeaderValue::from_static("agent"), - ); - headers.insert( - "x-temper-agent-type", - AxumHeaderValue::from_static("supervisor"), - ); + let resolved = require_authenticated_context(Some(&authenticated)).unwrap(); - let ctx = security_context_from_headers(&headers, None, None, None); - assert_eq!(ctx.principal.kind, PrincipalKind::Agent); - assert_eq!(ctx.principal.id, "bot-1"); - assert_eq!(ctx.principal.agent_type.as_deref(), Some("supervisor")); + assert_eq!(resolved.tenant().as_str(), "tenant-a"); + assert_eq!(resolved.security_context().principal.id, "system"); } #[test] - fn security_context_from_headers_preserves_principal_attrs_from_http_headers() { - let mut headers = HeaderMap::new(); - headers.insert( - "x-temper-principal-id", - AxumHeaderValue::from_static("bot-1"), - ); - headers.insert( - "x-temper-principal-kind", - AxumHeaderValue::from_static("agent"), - ); - headers.insert( - "x-temper-attr-region", - AxumHeaderValue::from_static("us-east-1"), - ); + fn tenant_mismatch_is_unauthorized() { + let authenticated = + AuthenticatedRequestContext::new(TenantId::new("tenant-a"), SecurityContext::system()); - let ctx = security_context_from_headers(&headers, None, None, None); assert_eq!( - ctx.principal.attributes.get("region"), - Some(&Value::String("us-east-1".to_string())) - ); - } - - #[test] - fn security_context_from_headers_drops_action_context_from_http_headers() { - let mut headers = HeaderMap::new(); - headers.insert( - "x-temper-principal-id", - AxumHeaderValue::from_static("bot-1"), - ); - headers.insert( - "x-temper-principal-kind", - AxumHeaderValue::from_static("agent"), + require_tenant_match(&authenticated, "tenant-b").unwrap_err(), + StatusCode::UNAUTHORIZED ); - headers.insert( - "x-temper-action-context", - AxumHeaderValue::from_static("composite:App.Fork"), - ); - - let ctx = security_context_from_headers(&headers, None, None, None); - assert!(!ctx.principal.attributes.contains_key("action_context")); } } diff --git a/crates/temper-server/src/authz/mod.rs b/crates/temper-server/src/authz/mod.rs index 118423405..1f3f10cc8 100644 --- a/crates/temper-server/src/authz/mod.rs +++ b/crates/temper-server/src/authz/mod.rs @@ -1,13 +1,24 @@ //! Authorization: Cedar policy enforcement and WASM host-function gating. +pub mod edge; mod helpers; pub mod policy_persistence; pub mod wasm_gate; +pub use edge::{ + is_public_kernel_request, require_authenticated_request_context, strip_inbound_identity_headers, +}; +pub use temper_authz::AuthenticatedRequestContext; + #[allow(unused_imports)] // Used by observe/ handlers via crate::authz::observe_tenant_scope pub(crate) use helpers::{ DenialInput, GovernedMutationAuth, observe_tenant_scope, record_authz_denial, - require_governed_mutation_auth, require_observe_auth, security_context_from_headers, + require_authenticated_context, require_governed_mutation_auth, require_observe_auth, + require_tenant_match, +}; +#[cfg(feature = "observe")] +pub(crate) use helpers::{ResourceAuthorization, require_resource_authorization}; +pub use policy_persistence::{ + load_and_activate_tenant_policies, persist_and_activate_policy, record_policy_change, }; -pub use policy_persistence::{load_and_activate_tenant_policies, persist_and_activate_policy}; pub use wasm_gate::{CedarWasmAuthzGate, PermissiveWasmAuthzGate}; diff --git a/crates/temper-server/src/authz/policy_persistence.rs b/crates/temper-server/src/authz/policy_persistence.rs index 2ed6d796b..5165887e4 100644 --- a/crates/temper-server/src/authz/policy_persistence.rs +++ b/crates/temper-server/src/authz/policy_persistence.rs @@ -64,44 +64,48 @@ pub async fn persist_and_activate_policy( }; if changed { - // Log a trajectory entry so the policy change is observable in the - // Evolution Engine dashboard and trajectory analytics. - let now = sim_now().to_rfc3339(); - let entry = TrajectoryEntry { - timestamp: now, - tenant: tenant.to_string(), - entity_type: "_cedar".to_string(), - entity_id: tenant.to_string(), - action: "policy_saved".to_string(), - success: true, - from_status: None, - to_status: None, - error: None, - agent_id: Some(created_by.to_string()), - session_id: None, - authz_denied: None, - denied_resource: None, - denied_module: None, - source: Some(TrajectorySource::Platform), - spec_governed: Some(false), - agent_type: None, - request_body: None, - intent: None, - matched_policy_ids: None, - }; - if !state.enqueue_trajectory_entry(entry) { - tracing::warn!( - tenant, - policy_id, - "failed to enqueue policy_saved trajectory entry" - ); - } - tracing::info!(tenant, policy_id, created_by, "Cedar policy change logged"); + record_policy_change(state, tenant, policy_id, created_by); } changed } +/// Emit the standard trajectory record after a policy is durably committed +/// and activated. +pub fn record_policy_change(state: &ServerState, tenant: &str, policy_id: &str, created_by: &str) { + let entry = TrajectoryEntry { + timestamp: sim_now().to_rfc3339(), + tenant: tenant.to_string(), + entity_type: "_cedar".to_string(), + entity_id: tenant.to_string(), + action: "policy_saved".to_string(), + success: true, + from_status: None, + to_status: None, + error: None, + agent_id: Some(created_by.to_string()), + session_id: None, + authz_denied: None, + denied_resource: None, + denied_module: None, + source: Some(TrajectorySource::Platform), + spec_governed: Some(false), + agent_type: None, + request_body: None, + intent: None, + matched_policy_ids: None, + capture_seq: None, + }; + if !state.enqueue_trajectory_entry(entry) { + tracing::warn!( + tenant, + policy_id, + "failed to enqueue policy_saved trajectory entry" + ); + } + tracing::info!(tenant, policy_id, created_by, "Cedar policy change logged"); +} + /// Load all persisted Cedar policies for a tenant and activate them. /// /// Reads every row from the `policies` table for `tenant`, concatenates the diff --git a/crates/temper-server/src/authz/wasm_gate.rs b/crates/temper-server/src/authz/wasm_gate.rs index 40bdd2500..f2eed4c2a 100644 --- a/crates/temper-server/src/authz/wasm_gate.rs +++ b/crates/temper-server/src/authz/wasm_gate.rs @@ -40,10 +40,6 @@ impl WasmAuthzGate for CedarWasmAuthzGate { url: &str, ctx: &WasmAuthzContext, ) -> WasmAuthzDecision { - if crate::blob_store::is_local_internal_blob_endpoint(url) { - return WasmAuthzDecision::Allow; - } - let security_ctx = build_wasm_security_context(ctx); // Build resource attrs with BTreeMap (DST compliant) @@ -110,6 +106,58 @@ impl WasmAuthzGate for CedarWasmAuthzGate { } } +/// Capability gate bound to one tenant-authorized local blob endpoint. +/// +/// The generic Cedar gate deliberately has no localhost exception. The +/// exception exists only on a concrete host after that host has been given the +/// tenant's authorized `blob_endpoint` bootstrap secret. +struct BoundLocalBlobWasmAuthzGate { + inner: Arc, + endpoint: crate::blob_store::LocalInternalBlobEndpoint, +} + +impl WasmAuthzGate for BoundLocalBlobWasmAuthzGate { + fn authorize_http_call( + &self, + domain: &str, + method: &str, + url: &str, + ctx: &WasmAuthzContext, + ) -> WasmAuthzDecision { + if matches!(method.to_ascii_uppercase().as_str(), "GET" | "PUT") + && self.endpoint.object_key(url).is_some() + { + return WasmAuthzDecision::Allow; + } + self.inner.authorize_http_call(domain, method, url, ctx) + } + + fn authorize_secret_access( + &self, + secret_key: &str, + ctx: &WasmAuthzContext, + ) -> WasmAuthzDecision { + self.inner.authorize_secret_access(secret_key, ctx) + } +} + +/// Bind the local-blob exception to the exact endpoint authorized for this +/// host. Invalid, remote, or near-match endpoint values leave the Cedar gate +/// unchanged. +pub(crate) fn bind_local_blob_endpoint( + gate: Arc, + endpoint: Option<&str>, +) -> Arc { + let Some(endpoint) = endpoint.and_then(crate::blob_store::LocalInternalBlobEndpoint::parse) + else { + return gate; + }; + Arc::new(BoundLocalBlobWasmAuthzGate { + inner: gate, + endpoint, + }) +} + /// Permissive gate that allows all WASM host function calls. /// /// Used when Cedar WASM gating is not configured, preserving backward @@ -220,7 +268,7 @@ mod tests { } #[test] - fn cedar_gate_allows_local_internal_blob_endpoint() { + fn generic_cedar_gate_has_no_loopback_blob_bypass() { let engine = Arc::new(AuthzEngine::empty()); let gate = CedarWasmAuthzGate::new(engine); let ctx = test_ctx(); @@ -232,7 +280,66 @@ mod tests { &ctx, ); - assert_eq!(result, WasmAuthzDecision::Allow); + assert_eq!( + result, + WasmAuthzDecision::Deny("no matching permit policy".to_string()) + ); + } + + #[test] + fn local_blob_gate_allows_only_the_bound_origin_path_and_methods() { + let inner: Arc = + Arc::new(CedarWasmAuthzGate::new(Arc::new(AuthzEngine::empty()))); + let gate = bind_local_blob_endpoint(inner, Some("http://127.0.0.1:3000/_internal/blobs")); + let ctx = test_ctx(); + + for (method, url) in [ + ( + "GET", + "http://127.0.0.1:3000/_internal/blobs/field-overflow/sha256/a.json", + ), + ( + "PUT", + "http://127.0.0.1:3000/_internal/blobs/field-overflow/sha256/a.json", + ), + ] { + assert_eq!( + gate.authorize_http_call("127.0.0.1", method, url, &ctx), + WasmAuthzDecision::Allow, + "{method} {url}" + ); + } + + for (method, url) in [ + ( + "GET", + "http://127.0.0.1:3001/_internal/blobs/field-overflow/sha256/a.json", + ), + ( + "GET", + "http://127.0.0.1:3000/proxy/_internal/blobs/field-overflow/sha256/a.json", + ), + ( + "GET", + "http://127.0.0.1:3000/_internal/blobs.evil/field-overflow/sha256/a.json", + ), + ( + "GET", + "http://127.0.0.1:3000/_internal/blobs/field-overflow/sha256/a.json?redirect=1", + ), + ( + "POST", + "http://127.0.0.1:3000/_internal/blobs/field-overflow/sha256/a.json", + ), + ] { + assert!( + matches!( + gate.authorize_http_call("127.0.0.1", method, url, &ctx), + WasmAuthzDecision::Deny(_) + ), + "near-match unexpectedly allowed: {method} {url}" + ); + } } #[test] @@ -247,6 +354,9 @@ mod tests { }; "#; let engine = Arc::new(AuthzEngine::new(policy).unwrap()); + engine + .reload_tenant_policies("test-tenant", policy) + .expect("tenant policy should load"); let gate = CedarWasmAuthzGate::new(engine); let ctx = test_ctx(); @@ -271,6 +381,9 @@ mod tests { }; "#; let engine = Arc::new(AuthzEngine::new(policy).unwrap()); + engine + .reload_tenant_policies("test-tenant", policy) + .expect("tenant policy should load"); let gate = CedarWasmAuthzGate::new(engine); let ctx = test_ctx(); diff --git a/crates/temper-server/src/blob_store.rs b/crates/temper-server/src/blob_store.rs index 5fd8f9fb1..df39b6860 100644 --- a/crates/temper-server/src/blob_store.rs +++ b/crates/temper-server/src/blob_store.rs @@ -3,33 +3,41 @@ //! New blob writes go through this boundary. The Turso `blobs` table remains a //! legacy read fallback for installations that already wrote blob bytes there. -use std::path::{Path, PathBuf}; -use std::sync::OnceLock; +use std::path::PathBuf; use std::time::{Duration, Instant}; use reqwest::header::HeaderMap; use reqwest::{Method, StatusCode}; -use temper_runtime::tenant::TenantId; -use tokio::sync::Semaphore; use tracing::Instrument; use crate::aws_sigv4; use crate::blob_transport_observability::{ BlobTransportError, BlobTransportFinish, blob_transport_span, finish_blob_transport, }; -use crate::state::ServerState; - -const DEFAULT_BLOB_IO_MAX_CONCURRENCY: usize = 32; -pub(crate) const DEFAULT_BLOB_BUCKET: &str = "temper-fs"; -pub(crate) const WASM_ARTIFACT_PREFIX: &str = "wasm-modules/"; - -pub(crate) fn wasm_artifact_key(sha256_hash: &str) -> String { - format!("{WASM_ARTIFACT_PREFIX}{sha256_hash}") -} +mod endpoint; +mod keys; +mod limits; +mod local; +mod raw_ingest; +mod state; +mod streaming; +pub(crate) use endpoint::{LocalInternalBlobEndpoint, is_local_internal_blob_endpoint}; +pub(crate) use keys::{DEFAULT_BLOB_BUCKET, hex_lower, wasm_artifact_key}; +use limits::{BLOB_BUFFERED_OPERATION_TIMEOUT, BLOB_IO_QUEUE_TIMEOUT, blob_io_semaphore}; +use local::{get_local_blob_observed, local_blob_path, put_local_blob_observed}; +pub use raw_ingest::BlobByteStream; +#[cfg(test)] +pub(crate) use raw_ingest::BlobIngestProgressPolicy; +pub(crate) use raw_ingest::{ + BlobIngestAdmissionError, BlobIngestBudget, BlobStageError, MAX_RAW_BLOB_BYTES, +}; +pub(crate) use streaming::BlobReadBounded; +pub use streaming::{BlobObjectStream, BlobStreamRead, decode_json_base64_stream}; #[derive(Clone, Debug)] pub(crate) struct BlobStore { backend: BlobStoreBackend, + staging_root: PathBuf, } #[derive(Clone, Debug)] @@ -42,27 +50,18 @@ enum BlobStoreBackend { struct S3BlobStore { endpoint: String, bucket: String, + key_prefix: Option, access_key: Option, secret_key: Option, client: reqwest::Client, } -fn blob_io_semaphore() -> &'static Semaphore { - static SEMAPHORE: OnceLock = OnceLock::new(); - SEMAPHORE.get_or_init(|| { - let limit = std::env::var("TEMPER_BLOB_IO_MAX_CONCURRENCY") // determinism-ok: startup-only tuning knob - .ok() - .and_then(|v| v.parse::().ok()) - .filter(|v| *v > 0) - .unwrap_or(DEFAULT_BLOB_IO_MAX_CONCURRENCY); - Semaphore::new(limit) - }) -} - impl BlobStore { pub(crate) fn local_fs(root: impl Into) -> Self { + let root = root.into(); Self { - backend: BlobStoreBackend::LocalFs { root: root.into() }, + staging_root: root.join(".ingest-staging"), + backend: BlobStoreBackend::LocalFs { root }, } } @@ -71,14 +70,21 @@ impl BlobStore { bucket: impl Into, access_key: Option, secret_key: Option, + staging_root: impl Into, + key_prefix: Option, ) -> Self { Self { + staging_root: staging_root.into(), backend: BlobStoreBackend::S3(S3BlobStore { endpoint: endpoint.into().trim_end_matches('/').to_string(), bucket: bucket.into().trim_matches('/').to_string(), + key_prefix, access_key, secret_key, - client: reqwest::Client::new(), + client: reqwest::Client::builder() + .connect_timeout(std::time::Duration::from_secs(10)) + .build() + .expect("static blob HTTP client configuration must be valid"), // ci-ok: static reqwest builder options }), } } @@ -90,10 +96,11 @@ impl BlobStore { ttl: Option, ) -> Result<(), String> { let queued_at = Instant::now(); // determinism-ok: production blob I/O queue metric only - let _permit = blob_io_semaphore() - .acquire() - .await - .expect("blob semaphore closed"); // ci-ok: process-global and never closed + let _permit = + tokio::time::timeout(BLOB_IO_QUEUE_TIMEOUT, blob_io_semaphore().acquire_owned()) + .await + .map_err(|_| "blob put queue deadline exceeded".to_string())? + .expect("blob semaphore closed"); // ci-ok: process-global and never closed let wait_duration = queued_at.elapsed(); crate::runtime_metrics::record_blob_io_wait_duration(wait_duration, "put"); if wait_duration.as_millis() > 0 { @@ -107,12 +114,16 @@ impl BlobStore { ); } - match &self.backend { - BlobStoreBackend::LocalFs { root } => { - put_local_blob_observed(root, key, body, "put").await + tokio::time::timeout(BLOB_BUFFERED_OPERATION_TIMEOUT, async { + match &self.backend { + BlobStoreBackend::LocalFs { root } => { + put_local_blob_observed(root, key, body, "put").await + } + BlobStoreBackend::S3(store) => store.put_if_absent(key, body).await, } - BlobStoreBackend::S3(store) => store.put_if_absent(key, body).await, - } + }) + .await + .map_err(|_| format!("blob put timed out for '{key}'"))? } /// Write bytes to a content-addressed key. @@ -127,10 +138,11 @@ impl BlobStore { ttl: Option, ) -> Result<(), String> { let queued_at = Instant::now(); // determinism-ok: production blob I/O queue metric only - let _permit = blob_io_semaphore() - .acquire() - .await - .expect("blob semaphore closed"); // ci-ok: process-global and never closed + let _permit = + tokio::time::timeout(BLOB_IO_QUEUE_TIMEOUT, blob_io_semaphore().acquire_owned()) + .await + .map_err(|_| "content-addressed blob put queue deadline exceeded".to_string())? + .expect("blob semaphore closed"); // ci-ok: process-global and never closed let wait_duration = queued_at.elapsed(); crate::runtime_metrics::record_blob_io_wait_duration(wait_duration, "put_content"); if wait_duration.as_millis() > 0 { @@ -144,138 +156,41 @@ impl BlobStore { ); } - match &self.backend { - BlobStoreBackend::LocalFs { root } => { - put_local_blob_observed(root, key, body, "put_content").await + tokio::time::timeout(BLOB_BUFFERED_OPERATION_TIMEOUT, async { + match &self.backend { + BlobStoreBackend::LocalFs { root } => { + local::put_local_blob_replace_observed(root, key, body, "put_content").await + } + BlobStoreBackend::S3(store) => { + store.put_with_operation("put_content", key, body).await + } } - BlobStoreBackend::S3(store) => store.put_with_operation("put_content", key, body).await, - } + }) + .await + .map_err(|_| format!("content-addressed blob put timed out for '{key}'"))? } pub(crate) async fn get(&self, key: &str) -> Result>, String> { let queued_at = Instant::now(); // determinism-ok: production blob I/O queue metric only - let _permit = blob_io_semaphore() - .acquire() - .await - .expect("blob semaphore closed"); // ci-ok: process-global and never closed + let _permit = + tokio::time::timeout(BLOB_IO_QUEUE_TIMEOUT, blob_io_semaphore().acquire_owned()) + .await + .map_err(|_| "blob get queue deadline exceeded".to_string())? + .expect("blob semaphore closed"); // ci-ok: process-global and never closed let wait_duration = queued_at.elapsed(); crate::runtime_metrics::record_blob_io_wait_duration(wait_duration, "get"); if wait_duration.as_millis() > 0 { tracing::info!(path = %key, wait_ms = wait_duration.as_millis() as u64, "blob get queued"); } - match &self.backend { - BlobStoreBackend::LocalFs { root } => get_local_blob_observed(root, key).await, - BlobStoreBackend::S3(store) => store.get(key).await, - } - } -} - -impl ServerState { - pub(crate) fn blob_store_for_tenant(&self, tenant: &TenantId) -> Result { - if let Some(vault) = self.secrets_vault.as_ref() - && let Some(endpoint) = vault.get_secret(tenant.as_str(), "blob_endpoint") - && !endpoint.trim().is_empty() - { - if is_local_internal_blob_endpoint(&endpoint) { - return self.local_blob_store().ok_or_else(|| { - "internal DB-backed blob endpoint is disabled; set TEMPER_LOCAL_BLOB_DIR or configure BLOB_ENDPOINT for R2/S3" - .to_string() - }); + tokio::time::timeout(BLOB_BUFFERED_OPERATION_TIMEOUT, async { + match &self.backend { + BlobStoreBackend::LocalFs { root } => get_local_blob_observed(root, key).await, + BlobStoreBackend::S3(store) => store.get(key).await, } - - let bucket = vault - .get_secret(tenant.as_str(), "blob_bucket") - .unwrap_or_else(|| DEFAULT_BLOB_BUCKET.to_string()); - let access_key = vault.get_secret(tenant.as_str(), "blob_access_key"); - let secret_key = vault.get_secret(tenant.as_str(), "blob_secret_key"); - return Ok(BlobStore::s3(endpoint, bucket, access_key, secret_key)); - } - - self.local_blob_store().ok_or_else(|| { - "blob object store is not configured; set BLOB_ENDPOINT/BLOB_BUCKET/BLOB_ACCESS_KEY/BLOB_SECRET_KEY or TEMPER_LOCAL_BLOB_DIR" - .to_string() }) - } - - pub(crate) async fn put_blob_object( - &self, - tenant: &TenantId, - key: &str, - body: &[u8], - ttl: Option, - ) -> Result<(), String> { - let store = self.blob_store_for_tenant(tenant)?; - store.put_if_absent(key, body, ttl).await?; - self.put_metadata_blob_shadow(tenant, key, body, ttl).await - } - - /// Write bytes to a tenant-scoped content-addressed blob key. - pub(crate) async fn put_content_addressed_blob( - &self, - tenant: &TenantId, - key: &str, - body: &[u8], - ttl: Option, - ) -> Result<(), String> { - let store = self.blob_store_for_tenant(tenant)?; - store.put_content_addressed(key, body, ttl).await?; - self.put_metadata_blob_shadow(tenant, key, body, ttl).await - } - - pub async fn get_blob_with_legacy_fallback( - &self, - tenant: &TenantId, - key: &str, - ) -> Result>, String> { - match self.blob_store_for_tenant(tenant) { - Ok(store) => match store.get(key).await { - Ok(Some(bytes)) => return Ok(Some(bytes)), - Ok(None) => {} - Err(error) => { - tracing::warn!(%key, %error, "object blob store read failed; trying legacy DB blob fallback"); - } - }, - Err(error) => { - tracing::debug!(%key, %error, "object blob store unavailable; trying legacy DB blob fallback"); - } - } - - let Some(store) = self.metadata_store_for_tenant(tenant.as_str()).await else { - return Ok(None); - }; - store - .get_blob(key) - .await - .map_err(|e| format!("legacy DB blob read failed for '{key}': {e}")) - } - - fn local_blob_store(&self) -> Option { - if let Ok(root) = std::env::var("TEMPER_LOCAL_BLOB_DIR") // determinism-ok: deployment config read - && !root.trim().is_empty() - { - return Some(BlobStore::local_fs(root)); - } - if !self.data_dir.as_os_str().is_empty() { - return Some(BlobStore::local_fs(self.data_dir.join("blobs"))); - } - None - } - - async fn put_metadata_blob_shadow( - &self, - tenant: &TenantId, - key: &str, - body: &[u8], - ttl: Option, - ) -> Result<(), String> { - let Some(store) = self.metadata_store_for_tenant(tenant.as_str()).await else { - return Ok(()); - }; - store - .put_blob_with_ttl(key, body, ttl) - .await - .map_err(|error| format!("metadata blob shadow write failed for '{key}': {error}")) + .await + .map_err(|_| format!("blob get timed out for '{key}'"))? } } @@ -352,6 +267,77 @@ impl S3BlobStore { } } + async fn put_stream_with_operation( + &self, + operation: &'static str, + key: &str, + stream: BlobByteStream, + content_len: u64, + ) -> Result<(), String> { + let started_at = Instant::now(); // determinism-ok: production blob transport metric only + let span = blob_transport_span(operation, "s3", content_len); + let result = async { + let url = self.object_url(key); + let mut request = self + .client + .put(&url) + .header(reqwest::header::CONTENT_LENGTH, content_len) + .timeout(raw_ingest::BLOB_BACKEND_OPERATION_TIMEOUT) + .body(reqwest::Body::wrap_stream(stream)); + let headers = self + .signed_headers(Method::PUT, &url) + .map_err(BlobTransportError::message)?; + for (header_name, header_value) in &headers { + request = request.header(header_name, header_value); + } + + let response = request.send().await.map_err(|error| { + BlobTransportError::message(format!( + "streaming blob PUT request failed for '{key}': {error}" + )) + })?; + let status = response.status(); + if status.is_success() { + return Ok(status); + } + Err(BlobTransportError::status( + format!("streaming blob PUT failed for '{key}' with HTTP {status}"), + status, + )) + } + .instrument(span.clone()) + .await; + + match result { + Ok(status) => { + finish_blob_transport(BlobTransportFinish { + started_at, + span: &span, + operation, + backend: "s3", + outcome: "ok", + status: Some(status), + request_bytes: content_len, + response_bytes: 0, + }); + Ok(()) + } + Err(error) => { + finish_blob_transport(BlobTransportFinish { + started_at, + span: &span, + operation, + backend: "s3", + outcome: "error", + status: error.status, + request_bytes: content_len, + response_bytes: 0, + }); + Err(error.message) + } + } + } + async fn get(&self, key: &str) -> Result>, String> { let started_at = Instant::now(); // determinism-ok: production blob transport metric only let span = blob_transport_span("get", "s3", 0); @@ -481,12 +467,12 @@ impl S3BlobStore { } fn object_url(&self, key: &str) -> String { - format!( - "{}/{}/{}", - self.endpoint, - self.bucket, - key.trim_start_matches('/') - ) + let key = key.trim_start_matches('/'); + let object_path = self + .key_prefix + .as_deref() + .map_or_else(|| key.to_string(), |prefix| format!("{prefix}/{key}")); + format!("{}/{}/{}", self.endpoint, self.bucket, object_path) } fn signed_headers(&self, method: Method, url: &str) -> Result { @@ -512,152 +498,3 @@ impl S3BlobStore { }) } } - -async fn put_local_blob(root: &Path, key: &str, body: &[u8]) -> Result<(), String> { - let path = local_blob_path(root, key)?; - if tokio::fs::try_exists(&path) - .await - .map_err(|e| format!("failed to check local blob '{}': {e}", path.display()))? - { - return Ok(()); - } - if let Some(parent) = path.parent() { - tokio::fs::create_dir_all(parent).await.map_err(|e| { - format!( - "failed to create local blob dir '{}': {e}", - parent.display() - ) - })?; - } - tokio::fs::write(&path, body) - .await - .map_err(|e| format!("failed to write local blob '{}': {e}", path.display())) -} - -async fn put_local_blob_observed( - root: &Path, - key: &str, - body: &[u8], - operation: &'static str, -) -> Result<(), String> { - let request_bytes = body.len() as u64; - let started_at = Instant::now(); // determinism-ok: production blob transport metric only - let span = blob_transport_span(operation, "local_fs", request_bytes); - let result = put_local_blob(root, key, body) - .instrument(span.clone()) - .await; - let outcome = if result.is_ok() { "ok" } else { "error" }; - finish_blob_transport(BlobTransportFinish { - started_at, - span: &span, - operation, - backend: "local_fs", - outcome, - status: None, - request_bytes, - response_bytes: 0, - }); - result -} - -async fn get_local_blob(root: &Path, key: &str) -> Result>, String> { - let path = local_blob_path(root, key)?; - if !tokio::fs::try_exists(&path) - .await - .map_err(|e| format!("failed to check local blob '{}': {e}", path.display()))? - { - return Ok(None); - } - tokio::fs::read(&path) - .await - .map(Some) - .map_err(|e| format!("failed to read local blob '{}': {e}", path.display())) -} - -async fn get_local_blob_observed(root: &Path, key: &str) -> Result>, String> { - let started_at = Instant::now(); // determinism-ok: production blob transport metric only - let span = blob_transport_span("get", "local_fs", 0); - let result = get_local_blob(root, key).instrument(span.clone()).await; - let (outcome, response_bytes) = match &result { - Ok(Some(bytes)) => ("ok", bytes.len() as u64), - Ok(None) => ("not_found", 0), - Err(_) => ("error", 0), - }; - finish_blob_transport(BlobTransportFinish { - started_at, - span: &span, - operation: "get", - backend: "local_fs", - outcome, - status: None, - request_bytes: 0, - response_bytes, - }); - result -} - -fn local_blob_path(root: &Path, key: &str) -> Result { - let mut path = root.to_path_buf(); - let mut saw_component = false; - for component in key.split('/') { - if component.is_empty() - || component == "." - || component == ".." - || component.contains('\\') - || component.starts_with(std::path::MAIN_SEPARATOR) - { - return Err(format!("invalid blob key '{key}'")); - } - saw_component = true; - path.push(component); - } - if !saw_component { - return Err("invalid empty blob key".to_string()); - } - Ok(path) -} - -pub(crate) fn is_local_internal_blob_endpoint(endpoint: &str) -> bool { - let normalized = endpoint.trim_end_matches('/'); - (normalized.starts_with("http://127.0.0.1:") || normalized.starts_with("http://localhost:")) - && normalized.contains("/_internal/blobs") -} - -#[cfg(test)] -mod tests { - use super::*; - - #[tokio::test] - async fn local_blob_store_round_trips_without_database() { - let dir = tempfile::tempdir().expect("tempdir"); - let store = BlobStore::local_fs(dir.path()); - - store - .put_if_absent("wasm-modules/hash-a", b"hello", None) - .await - .expect("put"); - - let bytes = store - .get("wasm-modules/hash-a") - .await - .expect("get") - .expect("present"); - assert_eq!(bytes, b"hello"); - assert!( - dir.path().join("wasm-modules").join("hash-a").is_file(), - "blob is stored in the filesystem object store" - ); - } - - #[tokio::test] - async fn local_blob_store_rejects_path_traversal_keys() { - let dir = tempfile::tempdir().expect("tempdir"); - let store = BlobStore::local_fs(dir.path()); - - let err = store - .put_if_absent("../escape", b"nope", None) - .await - .expect_err("path traversal rejected"); - assert!(err.contains("invalid blob key")); - } -} diff --git a/crates/temper-server/src/blob_store/endpoint.rs b/crates/temper-server/src/blob_store/endpoint.rs new file mode 100644 index 000000000..06fb80eca --- /dev/null +++ b/crates/temper-server/src/blob_store/endpoint.rs @@ -0,0 +1,117 @@ +//! Exact classification for the server-owned loopback blob endpoint. + +const INTERNAL_BLOB_PATH: &str = "/_internal/blobs"; + +/// Parsed loopback blob endpoint used to bind a WASM host capability. +/// +/// Keeping the parsed origin and path together prevents a loose localhost or +/// substring check from granting HTTP authority to a different local service. +#[derive(Clone, Debug)] +pub(crate) struct LocalInternalBlobEndpoint { + base: reqwest::Url, +} + +impl LocalInternalBlobEndpoint { + /// Parse only the exact server-owned endpoint base. + pub(crate) fn parse(endpoint: &str) -> Option { + let mut base = reqwest::Url::parse(endpoint).ok()?; + if base.scheme() != "http" + || !base.username().is_empty() + || base.password().is_some() + || base.query().is_some() + || base.fragment().is_some() + || base.port().is_none() + || !is_loopback_host(base.host_str()?) + { + return None; + } + let normalized_path = base.path().trim_end_matches('/'); + if normalized_path != INTERNAL_BLOB_PATH { + return None; + } + base.set_path(INTERNAL_BLOB_PATH); + Some(Self { base }) + } + + /// Return the canonical object key when `url` belongs to this exact bound + /// endpoint. Near-match paths, other ports, credentials, queries, and + /// fragments are rejected. + pub(crate) fn object_key(&self, url: &str) -> Option { + let request = reqwest::Url::parse(url).ok()?; + if request.scheme() != self.base.scheme() + || request.host_str() != self.base.host_str() + || request.port_or_known_default() != self.base.port_or_known_default() + || !request.username().is_empty() + || request.password().is_some() + || request.query().is_some() + || request.fragment().is_some() + { + return None; + } + let key = request + .path() + .strip_prefix(&format!("{INTERNAL_BLOB_PATH}/"))?; + (!key.is_empty()).then(|| key.to_string()) + } +} + +pub(crate) fn is_local_internal_blob_endpoint(endpoint: &str) -> bool { + LocalInternalBlobEndpoint::parse(endpoint).is_some() +} + +fn is_loopback_host(host: &str) -> bool { + matches!(host, "127.0.0.1" | "localhost" | "::1" | "[::1]") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn endpoint_base_requires_exact_loopback_origin_and_path() { + for endpoint in [ + "http://127.0.0.1:3000/_internal/blobs", + "http://localhost:3000/_internal/blobs/", + "http://[::1]:3000/_internal/blobs", + ] { + assert!( + LocalInternalBlobEndpoint::parse(endpoint).is_some(), + "{endpoint}" + ); + } + for endpoint in [ + "https://127.0.0.1:3000/_internal/blobs", + "http://127.0.0.1/_internal/blobs", + "http://127.0.0.1:3000/proxy/_internal/blobs", + "http://127.0.0.1:3000/_internal/blobs.evil", + "http://user@127.0.0.1:3000/_internal/blobs", + "http://127.0.0.1:3000/_internal/blobs?mode=write", + ] { + assert!( + LocalInternalBlobEndpoint::parse(endpoint).is_none(), + "{endpoint}" + ); + } + } + + #[test] + fn object_url_must_match_the_bound_origin_and_path() { + let endpoint = LocalInternalBlobEndpoint::parse("http://127.0.0.1:3000/_internal/blobs") + .expect("valid endpoint"); + assert_eq!( + endpoint + .object_key("http://127.0.0.1:3000/_internal/blobs/field-overflow/sha256/a.json",), + Some("field-overflow/sha256/a.json".to_string()) + ); + for url in [ + "http://127.0.0.1:3001/_internal/blobs/key", + "http://localhost:3000/_internal/blobs/key", + "http://127.0.0.1:3000/proxy/_internal/blobs/key", + "http://127.0.0.1:3000/_internal/blobs.evil/key", + "http://127.0.0.1:3000/_internal/blobs/key?redirect=1", + "http://user@127.0.0.1:3000/_internal/blobs/key", + ] { + assert!(endpoint.object_key(url).is_none(), "{url}"); + } + } +} diff --git a/crates/temper-server/src/blob_store/keys.rs b/crates/temper-server/src/blob_store/keys.rs new file mode 100644 index 000000000..de713bdbf --- /dev/null +++ b/crates/temper-server/src/blob_store/keys.rs @@ -0,0 +1,18 @@ +//! Canonical object-store key construction. + +const WASM_ARTIFACT_PREFIX: &str = "wasm-modules/"; +pub(crate) const DEFAULT_BLOB_BUCKET: &str = "temper-fs"; + +pub(crate) fn wasm_artifact_key(sha256_hash: &str) -> String { + format!("{WASM_ARTIFACT_PREFIX}{sha256_hash}") +} + +pub(crate) fn hex_lower(bytes: &[u8]) -> String { + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut output = String::with_capacity(bytes.len() * 2); + for &byte in bytes { + output.push(HEX[(byte >> 4) as usize] as char); + output.push(HEX[(byte & 0x0f) as usize] as char); + } + output +} diff --git a/crates/temper-server/src/blob_store/limits.rs b/crates/temper-server/src/blob_store/limits.rs new file mode 100644 index 000000000..5c512843a --- /dev/null +++ b/crates/temper-server/src/blob_store/limits.rs @@ -0,0 +1,23 @@ +//! Process-wide bounds for production blob I/O. + +use std::sync::{Arc, OnceLock}; +use std::time::Duration; + +use tokio::sync::Semaphore; + +const DEFAULT_BLOB_IO_MAX_CONCURRENCY: usize = 32; + +pub(super) const BLOB_IO_QUEUE_TIMEOUT: Duration = Duration::from_secs(30); +pub(super) const BLOB_BUFFERED_OPERATION_TIMEOUT: Duration = Duration::from_secs(5 * 60); + +pub(super) fn blob_io_semaphore() -> Arc { + static SEMAPHORE: OnceLock> = OnceLock::new(); + Arc::clone(SEMAPHORE.get_or_init(|| { + let limit = std::env::var("TEMPER_BLOB_IO_MAX_CONCURRENCY") // determinism-ok: startup-only tuning knob + .ok() + .and_then(|value| value.parse::().ok()) + .filter(|value| *value > 0) + .unwrap_or(DEFAULT_BLOB_IO_MAX_CONCURRENCY); + Arc::new(Semaphore::new(limit)) + })) +} diff --git a/crates/temper-server/src/blob_store/local.rs b/crates/temper-server/src/blob_store/local.rs new file mode 100644 index 000000000..de9aba5ac --- /dev/null +++ b/crates/temper-server/src/blob_store/local.rs @@ -0,0 +1,295 @@ +use std::path::{Path, PathBuf}; +use std::time::Instant; + +use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; +use tracing::Instrument as _; + +use crate::blob_transport_observability::{ + BlobTransportFinish, blob_transport_span, finish_blob_transport, +}; + +pub(super) async fn put_local_blob_observed( + root: &Path, + key: &str, + body: &[u8], + operation: &'static str, +) -> Result<(), String> { + put_local_blob_mode_observed(root, key, body, operation, false).await +} + +pub(super) async fn put_local_blob_replace_observed( + root: &Path, + key: &str, + body: &[u8], + operation: &'static str, +) -> Result<(), String> { + put_local_blob_mode_observed(root, key, body, operation, true).await +} + +async fn put_local_blob_mode_observed( + root: &Path, + key: &str, + body: &[u8], + operation: &'static str, + replace_existing: bool, +) -> Result<(), String> { + let request_bytes = body.len() as u64; + let started_at = Instant::now(); // determinism-ok: production blob transport metric only + let span = blob_transport_span(operation, "local_fs", request_bytes); + let result = put_local_blob_atomic(root, key, body, replace_existing) + .instrument(span.clone()) + .await; + finish_blob_transport(BlobTransportFinish { + started_at, + span: &span, + operation, + backend: "local_fs", + outcome: if result.is_ok() { "ok" } else { "error" }, + status: None, + request_bytes, + response_bytes: 0, + }); + result +} + +pub(super) async fn get_local_blob_observed( + root: &Path, + key: &str, +) -> Result>, String> { + let started_at = Instant::now(); // determinism-ok: production blob transport metric only + let span = blob_transport_span("get", "local_fs", 0); + let result = get_local_blob(root, key).instrument(span.clone()).await; + let (outcome, response_bytes) = match &result { + Ok(Some(bytes)) => ("ok", bytes.len() as u64), + Ok(None) => ("not_found", 0), + Err(_) => ("error", 0), + }; + finish_blob_transport(BlobTransportFinish { + started_at, + span: &span, + operation: "get", + backend: "local_fs", + outcome, + status: None, + request_bytes: 0, + response_bytes, + }); + result +} + +pub(super) async fn get_local_blob_bounded_observed( + root: &Path, + key: &str, + max_bytes: usize, +) -> Result { + let started_at = Instant::now(); // determinism-ok: production blob transport metric only + let span = blob_transport_span("get_bounded", "local_fs", 0); + let result = get_local_blob_bounded(root, key, max_bytes) + .instrument(span.clone()) + .await; + let (outcome, response_bytes) = match &result { + Ok(super::BlobReadBounded::Found(bytes)) => ("ok", bytes.len() as u64), + Ok(super::BlobReadBounded::Missing) => ("not_found", 0), + Ok(super::BlobReadBounded::TooLarge { .. }) => ("too_large", 0), + Err(_) => ("error", 0), + }; + finish_blob_transport(BlobTransportFinish { + started_at, + span: &span, + operation: "get_bounded", + backend: "local_fs", + outcome, + status: None, + request_bytes: 0, + response_bytes, + }); + result +} + +pub(super) fn local_blob_path(root: &Path, key: &str) -> Result { + let mut path = root.to_path_buf(); + let mut saw_component = false; + for component in key.split('/') { + if component.is_empty() + || component == "." + || component == ".." + || component.contains('\\') + || component.starts_with(std::path::MAIN_SEPARATOR) + { + return Err(format!("invalid blob key '{key}'")); + } + saw_component = true; + path.push(component); + } + if !saw_component { + return Err("invalid empty blob key".to_string()); + } + Ok(path) +} + +async fn put_local_blob_atomic( + root: &Path, + key: &str, + body: &[u8], + replace_existing: bool, +) -> Result<(), String> { + let path = local_blob_path(root, key)?; + if !replace_existing + && tokio::fs::try_exists(&path) + .await + .map_err(|error| format!("failed to check local blob '{}': {error}", path.display()))? + { + return Ok(()); + } + let parent = path + .parent() + .ok_or_else(|| format!("local blob '{}' has no parent", path.display()))?; + tokio::fs::create_dir_all(parent).await.map_err(|error| { + format!( + "failed to create local blob dir '{}': {error}", + parent.display() + ) + })?; + let parent = parent.to_path_buf(); + // determinism-ok: production object-store filesystem boundary + let staged = tokio::task::spawn_blocking(move || { + tempfile::Builder::new() + .prefix("object-put-") + .tempfile_in(parent) + }) + .await + .map_err(|error| format!("local blob staging task failed: {error}"))? + .map_err(|error| format!("failed to create local blob staging file: {error}"))?; + let (file, staged_path) = staged.into_parts(); + let mut file = tokio::fs::File::from_std(file); + file.write_all(body) + .await + .map_err(|error| format!("failed to write local blob '{}': {error}", path.display()))?; + file.flush().await.map_err(|error| error.to_string())?; + file.sync_data().await.map_err(|error| error.to_string())?; + drop(file); + if replace_existing { + tokio::fs::rename(staged_path.to_path_buf(), &path) + .await + .map_err(|error| format!("failed to replace local blob '{}': {error}", path.display())) + } else { + match tokio::fs::hard_link(staged_path.to_path_buf(), &path).await { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => Ok(()), + Err(error) => Err(format!( + "failed to publish local blob '{}': {error}", + path.display() + )), + } + } +} + +async fn get_local_blob(root: &Path, key: &str) -> Result>, String> { + let path = local_blob_path(root, key)?; + if !tokio::fs::try_exists(&path) + .await + .map_err(|error| format!("failed to check local blob '{}': {error}", path.display()))? + { + return Ok(None); + } + tokio::fs::read(&path) + .await + .map(Some) + .map_err(|error| format!("failed to read local blob '{}': {error}", path.display())) +} + +async fn get_local_blob_bounded( + root: &Path, + key: &str, + max_bytes: usize, +) -> Result { + let path = local_blob_path(root, key)?; + let file = match tokio::fs::File::open(&path).await { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Ok(super::BlobReadBounded::Missing); + } + Err(error) => { + return Err(format!( + "failed to open local blob '{}': {error}", + path.display() + )); + } + }; + let metadata = file + .metadata() + .await + .map_err(|error| format!("failed to stat local blob '{}': {error}", path.display()))?; + if metadata.len() > max_bytes as u64 { + return Ok(super::BlobReadBounded::TooLarge { + actual_bytes: Some(metadata.len()), + }); + } + + let bounded_len = max_bytes + .checked_add(1) + .ok_or_else(|| "bounded blob read limit overflowed usize".to_string())?; + let mut bytes = Vec::with_capacity(metadata.len() as usize); + file.take(bounded_len as u64) + .read_to_end(&mut bytes) + .await + .map_err(|error| format!("failed to read local blob '{}': {error}", path.display()))?; + if bytes.len() > max_bytes { + return Ok(super::BlobReadBounded::TooLarge { + actual_bytes: Some(bytes.len() as u64), + }); + } + Ok(super::BlobReadBounded::Found(bytes)) +} + +#[cfg(test)] +mod tests { + use super::super::BlobStore; + + #[tokio::test] + async fn local_blob_store_round_trips_without_database() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = BlobStore::local_fs(dir.path()); + store + .put_if_absent("wasm-modules/hash-a", b"hello", None) + .await + .expect("put"); + assert_eq!( + store.get("wasm-modules/hash-a").await.unwrap().unwrap(), + b"hello" + ); + } + + #[tokio::test] + async fn local_blob_store_rejects_path_traversal_keys() { + let dir = tempfile::tempdir().expect("tempdir"); + let error = BlobStore::local_fs(dir.path()) + .put_if_absent("../escape", b"nope", None) + .await + .expect_err("path traversal rejected"); + assert!(error.contains("invalid blob key")); + } + + #[tokio::test] + async fn content_addressed_write_repairs_existing_local_object() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = BlobStore::local_fs(dir.path()); + store + .put_if_absent("field-overflow/sha256/value.json", b"corrupt", None) + .await + .expect("seed corrupt object"); + + store + .put_content_addressed("field-overflow/sha256/value.json", b"canonical", None) + .await + .expect("repair object"); + + assert_eq!( + store + .get("field-overflow/sha256/value.json") + .await + .expect("read object"), + Some(b"canonical".to_vec()) + ); + } +} diff --git a/crates/temper-server/src/blob_store/raw_ingest.rs b/crates/temper-server/src/blob_store/raw_ingest.rs new file mode 100644 index 000000000..99f7d5680 --- /dev/null +++ b/crates/temper-server/src/blob_store/raw_ingest.rs @@ -0,0 +1,485 @@ +use std::io::Write as _; +use std::pin::Pin; +use std::time::{Duration, Instant}; + +use base64::Engine as _; +use bytes::Bytes; +use futures_util::{Stream, StreamExt as _}; +use sha1::Digest as _; +use sha2::Sha256; +use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; +use tracing::Instrument as _; + +use super::{ + BLOB_IO_QUEUE_TIMEOUT, BlobStore, BlobStoreBackend, blob_io_semaphore, keys::hex_lower, + local_blob_path, +}; +use crate::blob_transport_observability::{ + BlobTransportFinish, blob_transport_span, finish_blob_transport, +}; + +const STREAM_CHUNK_BYTES: usize = 64 * 1024; +pub(super) const BLOB_BACKEND_OPERATION_TIMEOUT: Duration = Duration::from_secs(30 * 60); +pub(crate) const MAX_RAW_BLOB_BYTES: usize = 2 * 1024 * 1024 * 1024; + +mod admission; +pub(crate) use admission::{ + BlobIngestAdmissionError, BlobIngestBudget, BlobIngestPermit, BlobIngestProgressPolicy, + BlobStageError, +}; + +/// Bounded asynchronous byte stream used at the object-store boundary. +pub type BlobByteStream = + Pin> + Send + 'static>>; + +/// Disk-backed body whose temporary file is deleted when this value drops. +pub(crate) struct StagedBlob { + path: tempfile::TempPath, + declared_len: usize, + canonical_sha1: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct Base64JsonDescriptor { + pub(crate) sha256: String, + pub(crate) serialized_len: usize, +} + +impl BlobStore { + /// Copy an untrusted body stream into an RAII staging file while hashing + /// the caller-supplied canonical prefix and exact declared body bytes. + pub(crate) async fn stage_canonical_stream( + &self, + mut stream: BlobByteStream, + declared_len: usize, + canonical_prefix: &[u8], + progress: &BlobIngestProgressPolicy, + admission: &mut BlobIngestPermit, + ) -> Result { + let started = tokio::time::Instant::now(); // determinism-ok: production upload deadline + let total_deadline = started + progress.total_timeout; + tokio::time::timeout_at( + total_deadline, + tokio::fs::create_dir_all(&self.staging_root), + ) + .await + .map_err(|_| BlobStageError::TotalDeadline { received: 0 })? + .map_err(|error| { + BlobStageError::Storage(format!( + "failed to create blob staging directory '{}': {error}", + self.staging_root.display() + )) + })?; + let staging_root = self.staging_root.clone(); + let staged = tokio::time::timeout_at( + total_deadline, + // determinism-ok: production object-store filesystem boundary + tokio::task::spawn_blocking(move || { + tempfile::Builder::new() + .prefix("raw-ingest-") + .tempfile_in(staging_root) + }), + ) + .await + .map_err(|_| BlobStageError::TotalDeadline { received: 0 })? + .map_err(|error| BlobStageError::Storage(format!("blob staging task failed: {error}")))? + .map_err(|error| { + BlobStageError::Storage(format!( + "failed to create blob staging file in '{}': {error}", + self.staging_root.display() + )) + })?; + let (file, path) = staged.into_parts(); + let mut file = tokio::fs::File::from_std(file); + let mut canonical_hasher = sha1::Sha1::new(); + canonical_hasher.update(canonical_prefix); + let mut received = 0usize; + let body_started = tokio::time::Instant::now(); // determinism-ok: production upload progress + let mut last_progress = body_started; + let mut throughput_check = body_started + progress.throughput_grace; + + loop { + let next = tokio::select! { + _ = tokio::time::sleep_until(total_deadline) => { + return Err(BlobStageError::TotalDeadline { received }); + } + _ = tokio::time::sleep_until(last_progress + progress.idle_timeout) => { + return Err(BlobStageError::IdleTimeout { received }); + } + _ = tokio::time::sleep_until(throughput_check) => { + let elapsed = tokio::time::Instant::now() // determinism-ok: production upload throughput clock + .saturating_duration_since(body_started); + let elapsed_millis = elapsed.as_millis().min(u128::from(u64::MAX)) as u64; + let required = progress + .min_bytes_per_second + .saturating_mul(elapsed_millis) + / 1000; + if (received as u64) < required { + return Err(BlobStageError::ThroughputTooLow { received, required }); + } + throughput_check += progress.throughput_check_interval; + continue; + } + next = stream.next() => next, + }; + let Some(chunk) = next else { + break; + }; + let chunk = chunk.map_err(|error| BlobStageError::BodyStream(error.to_string()))?; + if chunk.is_empty() { + tokio::task::yield_now().await; + continue; + } + received = received.checked_add(chunk.len()).ok_or_else(|| { + BlobStageError::Storage("body byte count overflowed usize".to_string()) + })?; + if received > declared_len { + return Err(BlobStageError::BodyExceedsDeclaredLength { + declared: declared_len, + }); + } + admission.reserve_received_bytes(received)?; + tokio::time::timeout_at(total_deadline, file.write_all(&chunk)) + .await + .map_err(|_| BlobStageError::TotalDeadline { received })? + .map_err(|error| { + BlobStageError::Storage(format!("failed to write staged Blob body: {error}")) + })?; + canonical_hasher.update(&chunk); + last_progress = tokio::time::Instant::now(); // determinism-ok: production upload progress + } + if received != declared_len { + return Err(BlobStageError::BodyShorterThanDeclaredLength { + declared: declared_len, + received, + }); + } + tokio::time::timeout_at(total_deadline, file.flush()) + .await + .map_err(|_| BlobStageError::TotalDeadline { received })? + .map_err(|error| { + BlobStageError::Storage(format!("failed to flush staged Blob body: {error}")) + })?; + tokio::time::timeout_at(total_deadline, file.sync_data()) + .await + .map_err(|_| BlobStageError::TotalDeadline { received })? + .map_err(|error| { + BlobStageError::Storage(format!("failed to sync staged Blob body: {error}")) + })?; + Ok(StagedBlob { + path, + declared_len, + canonical_sha1: hex_lower(&canonical_hasher.finalize()), + }) + } + + /// Stream JSON base64(prefix || staged bytes) into the object store. + pub(crate) async fn put_staged_base64_json( + &self, + key: &str, + staged: &StagedBlob, + prefix: &[u8], + serialized_len: usize, + ) -> Result<(), String> { + let expected_len = staged.base64_json_len(prefix)?; + if serialized_len != expected_len { + return Err(format!( + "base64 JSON length mismatch: descriptor {serialized_len}, expected {expected_len}" + )); + } + let stream = staged.base64_json_stream(prefix.to_vec()); + self.put_content_addressed_stream(key, stream, serialized_len as u64) + .await + } + + async fn put_content_addressed_stream( + &self, + key: &str, + stream: BlobByteStream, + content_len: u64, + ) -> Result<(), String> { + let queued_at = Instant::now(); // determinism-ok: production blob I/O queue metric only + let _permit = + tokio::time::timeout(BLOB_IO_QUEUE_TIMEOUT, blob_io_semaphore().acquire_owned()) + .await + .map_err(|_| "blob object-store queue deadline exceeded".to_string())? + .expect("blob semaphore closed"); // ci-ok: process-global and never closed + crate::runtime_metrics::record_blob_io_wait_duration( + queued_at.elapsed(), + "put_content_stream", + ); + let stream = enforce_outgoing_length(stream, content_len); + let operation = async { + match &self.backend { + BlobStoreBackend::LocalFs { root } => { + put_local_blob_stream_observed(root, key, stream, content_len).await + } + BlobStoreBackend::S3(store) => { + store + .put_stream_with_operation("put_content_stream", key, stream, content_len) + .await + } + } + }; + tokio::time::timeout(BLOB_BACKEND_OPERATION_TIMEOUT, operation) + .await + .map_err(|_| "blob object-store write deadline exceeded".to_string())? + } +} + +fn enforce_outgoing_length(mut stream: BlobByteStream, expected_bytes: u64) -> BlobByteStream { + Box::pin(async_stream::try_stream! { + let mut emitted = 0u64; + while let Some(chunk) = stream.next().await { + let chunk = chunk?; + emitted = emitted + .checked_add(chunk.len() as u64) + .ok_or_else(|| std::io::Error::other("outgoing blob byte count overflow"))?; + if emitted > expected_bytes { + Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "outgoing blob stream exceeded its declared length", + ))?; + } + if !chunk.is_empty() { + yield chunk; + } + } + if emitted != expected_bytes { + Err(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + format!("outgoing blob stream ended at {emitted} bytes; expected {expected_bytes}"), + ))?; + } + }) +} + +impl StagedBlob { + pub(crate) fn canonical_sha1(&self) -> &str { + &self.canonical_sha1 + } + + pub(crate) async fn base64_json_descriptor( + &self, + prefix: &[u8], + ) -> Result { + tokio::time::timeout( + BLOB_BACKEND_OPERATION_TIMEOUT, + self.base64_json_descriptor_inner(prefix), + ) + .await + .map_err(|_| "blob descriptor computation deadline exceeded".to_string())? + } + + async fn base64_json_descriptor_inner( + &self, + prefix: &[u8], + ) -> Result { + let serialized_len = self.base64_json_len(prefix)?; + let mut file = tokio::fs::File::open(self.path.to_path_buf()) + .await + .map_err(|error| format!("failed to reopen staged Blob body: {error}"))?; + let mut hasher = Sha256::new(); + hasher.update(b"\""); + { + let sink = DigestWriter(&mut hasher); + let mut encoder = + base64::write::EncoderWriter::new(sink, &base64::engine::general_purpose::STANDARD); + encoder + .write_all(prefix) + .map_err(|error| format!("failed to hash canonical base64 prefix: {error}"))?; + let mut buffer = vec![0u8; STREAM_CHUNK_BYTES]; + loop { + let read = file + .read(&mut buffer) + .await + .map_err(|error| format!("failed to read staged Blob body: {error}"))?; + if read == 0 { + break; + } + encoder + .write_all(&buffer[..read]) + .map_err(|error| format!("failed to hash staged base64 bytes: {error}"))?; + } + encoder + .finish() + .map_err(|error| format!("failed to finish staged base64 hash: {error}"))?; + } + hasher.update(b"\""); + Ok(Base64JsonDescriptor { + sha256: hex_lower(&hasher.finalize()), + serialized_len, + }) + } + + fn base64_json_len(&self, prefix: &[u8]) -> Result { + let raw_len = prefix + .len() + .checked_add(self.declared_len) + .ok_or_else(|| "canonical Blob representation length overflowed usize".to_string())?; + base64::encoded_len(raw_len, true) + .and_then(|encoded| encoded.checked_add(2)) + .ok_or_else(|| "base64 JSON representation length overflowed usize".to_string()) + } + + fn base64_json_stream(&self, prefix: Vec) -> BlobByteStream { + let path = self.path.to_path_buf(); + Box::pin(async_stream::try_stream! { + yield Bytes::from_static(b"\""); + let mut encoder = Base64ChunkEncoder::default(); + if let Some(encoded) = encoder.push(&prefix) { + yield encoded; + } + let mut file = tokio::fs::File::open(&path).await?; + let mut buffer = vec![0u8; STREAM_CHUNK_BYTES]; + loop { + let read = file.read(&mut buffer).await?; + if read == 0 { + break; + } + if let Some(encoded) = encoder.push(&buffer[..read]) { + yield encoded; + } + } + if let Some(encoded) = encoder.finish() { + yield encoded; + } + yield Bytes::from_static(b"\""); + }) + } +} + +struct DigestWriter<'a>(&'a mut Sha256); + +impl std::io::Write for DigestWriter<'_> { + fn write(&mut self, buffer: &[u8]) -> std::io::Result { + self.0.update(buffer); + Ok(buffer.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +#[derive(Default)] +struct Base64ChunkEncoder { + carry: Vec, +} + +impl Base64ChunkEncoder { + fn push(&mut self, input: &[u8]) -> Option { + if input.is_empty() { + return None; + } + let mut combined = Vec::with_capacity(self.carry.len() + input.len()); + combined.extend_from_slice(&self.carry); + combined.extend_from_slice(input); + let complete_len = (combined.len() / 3) * 3; + self.carry.clear(); + self.carry.extend_from_slice(&combined[complete_len..]); + if complete_len == 0 { + return None; + } + Some(Bytes::from( + base64::engine::general_purpose::STANDARD.encode(&combined[..complete_len]), + )) + } + + fn finish(&mut self) -> Option { + if self.carry.is_empty() { + return None; + } + let encoded = base64::engine::general_purpose::STANDARD.encode(&self.carry); + self.carry.clear(); + Some(Bytes::from(encoded)) + } +} + +async fn put_local_blob_stream( + root: &std::path::Path, + key: &str, + mut stream: BlobByteStream, + expected_bytes: u64, +) -> Result<(), String> { + let path = local_blob_path(root, key)?; + let parent = path + .parent() + .ok_or_else(|| format!("local blob '{}' has no parent directory", path.display()))?; + tokio::fs::create_dir_all(parent).await.map_err(|error| { + format!( + "failed to create local blob dir '{}': {error}", + parent.display() + ) + })?; + let parent = parent.to_path_buf(); + let staged = tokio::time::timeout( + BLOB_IO_QUEUE_TIMEOUT, + // determinism-ok: production object-store filesystem boundary + tokio::task::spawn_blocking(move || { + tempfile::Builder::new() + .prefix("object-put-") + .tempfile_in(parent) + }), + ) + .await + .map_err(|_| "local blob staging-file creation timed out".to_string())? + .map_err(|error| format!("local blob staging task failed: {error}"))? + .map_err(|error| format!("failed to create local blob staging file: {error}"))?; + let (file, staged_path) = staged.into_parts(); + let mut file = tokio::fs::File::from_std(file); + let mut written = 0u64; + while let Some(chunk) = stream.next().await { + let chunk = chunk.map_err(|error| error.to_string())?; + written = written + .checked_add(chunk.len() as u64) + .ok_or_else(|| "local blob byte count overflowed u64".to_string())?; + if written > expected_bytes { + return Err(format!( + "local blob stream exceeded declared length {expected_bytes}" + )); + } + file.write_all(&chunk) + .await + .map_err(|error| format!("failed to write local blob '{}': {error}", path.display()))?; + } + if written != expected_bytes { + return Err(format!( + "local blob stream ended at {written} bytes; expected {expected_bytes}" + )); + } + file.flush().await.map_err(|error| error.to_string())?; + file.sync_data().await.map_err(|error| error.to_string())?; + drop(file); + tokio::fs::rename(staged_path.to_path_buf(), &path) + .await + .map_err(|error| format!("failed to publish local blob '{}': {error}", path.display())) +} + +async fn put_local_blob_stream_observed( + root: &std::path::Path, + key: &str, + stream: BlobByteStream, + request_bytes: u64, +) -> Result<(), String> { + let started_at = Instant::now(); // determinism-ok: production blob transport metric only + let span = blob_transport_span("put_content_stream", "local_fs", request_bytes); + let result = put_local_blob_stream(root, key, stream, request_bytes) + .instrument(span.clone()) + .await; + finish_blob_transport(BlobTransportFinish { + started_at, + span: &span, + operation: "put_content_stream", + backend: "local_fs", + outcome: if result.is_ok() { "ok" } else { "error" }, + status: None, + request_bytes, + response_bytes: 0, + }); + result +} + +#[cfg(test)] +#[path = "raw_ingest/tests.rs"] +mod tests; diff --git a/crates/temper-server/src/blob_store/raw_ingest/admission.rs b/crates/temper-server/src/blob_store/raw_ingest/admission.rs new file mode 100644 index 000000000..855cfc800 --- /dev/null +++ b/crates/temper-server/src/blob_store/raw_ingest/admission.rs @@ -0,0 +1,302 @@ +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use temper_runtime::tenant::TenantId; +use tokio::sync::{OwnedSemaphorePermit, Semaphore}; + +use super::MAX_RAW_BLOB_BYTES; + +const DEFAULT_BUDGET_BYTES: usize = 4 * 1024 * 1024 * 1024; +const DEFAULT_BUDGET_UNIT_BYTES: usize = 1024 * 1024; +const DEFAULT_GLOBAL_CONCURRENCY: usize = 8; +const DEFAULT_PER_TENANT_CONCURRENCY: usize = 1; +const MAX_TENANT_ENTRIES: usize = 4096; +const DEFAULT_IDLE_TIMEOUT: Duration = Duration::from_secs(30); +const DEFAULT_TOTAL_TIMEOUT: Duration = Duration::from_secs(30 * 60); +const DEFAULT_THROUGHPUT_GRACE: Duration = Duration::from_secs(10); +const DEFAULT_THROUGHPUT_CHECK: Duration = Duration::from_secs(5); +const DEFAULT_MIN_BYTES_PER_SECOND: u64 = 64 * 1024; + +/// Process-local admission budget for disk-backed raw Blob staging. +#[derive(Clone, Debug)] +pub(crate) struct BlobIngestBudget { + permits: Arc, + global_slots: Arc, + tenant_slots: Arc>>, + unit_bytes: usize, + capacity_bytes: usize, + per_tenant_concurrency: usize, + progress_policy: BlobIngestProgressPolicy, +} + +#[derive(Clone, Debug)] +pub(crate) struct BlobIngestProgressPolicy { + pub(super) idle_timeout: Duration, + pub(super) total_timeout: Duration, + pub(super) throughput_grace: Duration, + pub(super) throughput_check_interval: Duration, + pub(super) min_bytes_per_second: u64, +} + +pub(crate) struct BlobIngestPermit { + byte_budget: Arc, + byte_permits: Vec, + unit_bytes: usize, + reserved_units: usize, + _global_permit: OwnedSemaphorePermit, + _tenant_permit: BlobTenantPermit, +} + +struct BlobTenantPermit { + slots: Arc>>, + tenant: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum BlobIngestAdmissionError { + ObjectTooLarge, + BudgetExhausted, + TenantBusy, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum BlobStageError { + BodyStream(String), + BodyExceedsDeclaredLength { declared: usize }, + BodyShorterThanDeclaredLength { declared: usize, received: usize }, + IdleTimeout { received: usize }, + TotalDeadline { received: usize }, + ThroughputTooLow { received: usize, required: u64 }, + StagingBudgetExhausted { received: usize }, + Storage(String), +} + +impl std::fmt::Display for BlobStageError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::BodyStream(error) => write!(formatter, "body stream failed: {error}"), + Self::BodyExceedsDeclaredLength { declared } => { + write!( + formatter, + "body bytes exceed declared Content-Length {declared}" + ) + } + Self::BodyShorterThanDeclaredLength { declared, received } => { + write!(formatter, "expected {declared} body bytes, got {received}") + } + Self::IdleTimeout { received } => write!( + formatter, + "body made no progress before idle deadline at {received} bytes" + ), + Self::TotalDeadline { received } => { + write!( + formatter, + "body exceeded total upload deadline at {received} bytes" + ) + } + Self::ThroughputTooLow { received, required } => write!( + formatter, + "body throughput was below the minimum: received {received} bytes, required {required}" + ), + Self::StagingBudgetExhausted { received } => write!( + formatter, + "aggregate staged bytes exhausted the process budget at {received} bytes" + ), + Self::Storage(error) => formatter.write_str(error), + } + } +} + +impl BlobIngestProgressPolicy { + fn runtime() -> Self { + Self { + idle_timeout: DEFAULT_IDLE_TIMEOUT, + total_timeout: DEFAULT_TOTAL_TIMEOUT, + throughput_grace: DEFAULT_THROUGHPUT_GRACE, + throughput_check_interval: DEFAULT_THROUGHPUT_CHECK, + min_bytes_per_second: DEFAULT_MIN_BYTES_PER_SECOND, + } + } + + #[cfg(test)] + pub(crate) fn new( + idle_timeout: Duration, + total_timeout: Duration, + throughput_grace: Duration, + throughput_check_interval: Duration, + min_bytes_per_second: u64, + ) -> Self { + assert!( + !idle_timeout.is_zero(), + "blob ingest idle timeout must be positive" + ); + assert!( + !total_timeout.is_zero(), + "blob ingest total timeout must be positive" + ); + assert!( + !throughput_check_interval.is_zero(), + "blob ingest throughput interval must be positive" + ); + assert!( + min_bytes_per_second > 0, + "blob ingest minimum throughput must be positive" + ); + Self { + idle_timeout, + total_timeout, + throughput_grace, + throughput_check_interval, + min_bytes_per_second, + } + } +} + +impl BlobIngestBudget { + pub(crate) fn runtime() -> Self { + Self::new(DEFAULT_BUDGET_BYTES, DEFAULT_BUDGET_UNIT_BYTES) + } + + pub(crate) fn new(capacity_bytes: usize, unit_bytes: usize) -> Self { + Self::with_limits( + capacity_bytes, + unit_bytes, + DEFAULT_GLOBAL_CONCURRENCY, + DEFAULT_PER_TENANT_CONCURRENCY, + BlobIngestProgressPolicy::runtime(), + ) + } + + pub(crate) fn with_limits( + capacity_bytes: usize, + unit_bytes: usize, + global_concurrency: usize, + per_tenant_concurrency: usize, + progress_policy: BlobIngestProgressPolicy, + ) -> Self { + assert!(capacity_bytes > 0, "blob ingest budget must be positive"); + assert!(unit_bytes > 0, "blob ingest budget unit must be positive"); + assert!( + capacity_bytes >= unit_bytes, + "blob ingest budget must contain at least one complete accounting unit" + ); + assert!( + global_concurrency > 0, + "global ingest concurrency must be positive" + ); + assert!( + per_tenant_concurrency > 0, + "tenant ingest concurrency must be positive" + ); + // Round down so unit accounting can never admit more actual staged + // bytes than the configured capacity. + let permit_count = capacity_bytes / unit_bytes; + assert!( + u32::try_from(permit_count).is_ok(), + "blob ingest budget permit count must fit in u32" + ); + Self { + permits: Arc::new(Semaphore::new(permit_count)), + global_slots: Arc::new(Semaphore::new(global_concurrency)), + tenant_slots: Arc::new(Mutex::new(BTreeMap::new())), + unit_bytes, + capacity_bytes, + per_tenant_concurrency, + progress_policy, + } + } + + pub(crate) fn try_reserve( + &self, + tenant: &TenantId, + declared_len: usize, + ) -> Result { + if declared_len > MAX_RAW_BLOB_BYTES || declared_len > self.capacity_bytes { + return Err(BlobIngestAdmissionError::ObjectTooLarge); + } + let tenant_permit = self.try_reserve_tenant(tenant)?; + let global_permit = self + .global_slots + .clone() + .try_acquire_owned() + .map_err(|_| BlobIngestAdmissionError::BudgetExhausted)?; + let byte_permit = self + .permits + .clone() + .try_acquire_owned() + .map_err(|_| BlobIngestAdmissionError::BudgetExhausted)?; + Ok(BlobIngestPermit { + byte_budget: Arc::clone(&self.permits), + byte_permits: vec![byte_permit], + unit_bytes: self.unit_bytes, + reserved_units: 1, + _global_permit: global_permit, + _tenant_permit: tenant_permit, + }) + } + + pub(crate) fn capacity_bytes(&self) -> usize { + self.capacity_bytes + } + + pub(crate) fn progress_policy(&self) -> &BlobIngestProgressPolicy { + &self.progress_policy + } + + fn try_reserve_tenant( + &self, + tenant: &TenantId, + ) -> Result { + let tenant = tenant.as_str().to_string(); + let mut slots = self + .tenant_slots + .lock() + .unwrap_or_else(|error| error.into_inner()); + if slots.get(&tenant).copied().unwrap_or(0) >= self.per_tenant_concurrency { + return Err(BlobIngestAdmissionError::TenantBusy); + } + if !slots.contains_key(&tenant) && slots.len() >= MAX_TENANT_ENTRIES { + return Err(BlobIngestAdmissionError::BudgetExhausted); + } + *slots.entry(tenant.clone()).or_insert(0) += 1; + drop(slots); + Ok(BlobTenantPermit { + slots: Arc::clone(&self.tenant_slots), + tenant, + }) + } +} + +impl BlobIngestPermit { + pub(super) fn reserve_received_bytes(&mut self, received: usize) -> Result<(), BlobStageError> { + let target_units = received.max(1).div_ceil(self.unit_bytes); + if target_units <= self.reserved_units { + return Ok(()); + } + let additional = target_units - self.reserved_units; + let additional = u32::try_from(additional) + .map_err(|_| BlobStageError::StagingBudgetExhausted { received })?; + let permit = self + .byte_budget + .clone() + .try_acquire_many_owned(additional) + .map_err(|_| BlobStageError::StagingBudgetExhausted { received })?; + self.byte_permits.push(permit); + self.reserved_units = target_units; + Ok(()) + } +} + +impl Drop for BlobTenantPermit { + fn drop(&mut self) { + let mut slots = self.slots.lock().unwrap_or_else(|error| error.into_inner()); + let Some(count) = slots.get_mut(&self.tenant) else { + return; + }; + *count = count.saturating_sub(1); + if *count == 0 { + slots.remove(&self.tenant); + } + } +} diff --git a/crates/temper-server/src/blob_store/raw_ingest/tests.rs b/crates/temper-server/src/blob_store/raw_ingest/tests.rs new file mode 100644 index 000000000..55997d227 --- /dev/null +++ b/crates/temper-server/src/blob_store/raw_ingest/tests.rs @@ -0,0 +1,96 @@ +use std::time::Duration; + +use bytes::Bytes; +use temper_runtime::tenant::TenantId; + +use super::{ + BlobByteStream, BlobIngestAdmissionError, BlobIngestBudget, BlobIngestProgressPolicy, + BlobStageError, put_local_blob_stream, +}; + +fn policy() -> BlobIngestProgressPolicy { + BlobIngestProgressPolicy::new( + Duration::from_secs(1), + Duration::from_secs(2), + Duration::from_millis(100), + Duration::from_millis(100), + 1, + ) +} + +#[test] +fn admission_is_tenant_fair_and_releases_on_drop() { + let budget = BlobIngestBudget::with_limits(4, 1, 2, 1, policy()); + let tenant_a = TenantId::new("tenant-a"); + let tenant_b = TenantId::new("tenant-b"); + + let permit_a = budget + .try_reserve(&tenant_a, 4) + .expect("first tenant admitted"); + assert_eq!( + budget.try_reserve(&tenant_a, 1).err(), + Some(BlobIngestAdmissionError::TenantBusy) + ); + let permit_b = budget + .try_reserve(&tenant_b, 4) + .expect("different tenant retains a fair slot"); + + drop(permit_a); + budget + .try_reserve(&tenant_a, 1) + .expect("tenant slot released by RAII"); + drop(permit_b); +} + +#[test] +fn staging_bytes_grow_with_progress_instead_of_declared_size() { + let budget = BlobIngestBudget::with_limits(3, 1, 2, 1, policy()); + let tenant_a = TenantId::new("tenant-a"); + let tenant_b = TenantId::new("tenant-b"); + let mut permit_a = budget + .try_reserve(&tenant_a, 3) + .expect("large declaration holds only one staging unit"); + let mut permit_b = budget + .try_reserve(&tenant_b, 3) + .expect("another tenant is not starved by the declaration"); + + permit_a + .reserve_received_bytes(2) + .expect("first upload grows to two actual bytes"); + assert_eq!( + permit_b.reserve_received_bytes(2), + Err(BlobStageError::StagingBudgetExhausted { received: 2 }) + ); + drop(permit_a); + permit_b + .reserve_received_bytes(2) + .expect("actual staging capacity is released on cancellation"); +} + +#[test] +fn staging_capacity_rounds_down_to_complete_units() { + let budget = BlobIngestBudget::with_limits(5, 4, 2, 1, policy()); + let first = budget + .try_reserve(&TenantId::new("tenant-a"), 5) + .expect("one complete unit is available"); + + assert_eq!( + budget.try_reserve(&TenantId::new("tenant-b"), 1).err(), + Some(BlobIngestAdmissionError::BudgetExhausted) + ); + drop(first); +} + +#[tokio::test] +async fn local_stream_rejects_length_mismatch_before_publication() { + let root = tempfile::tempdir().expect("blob root"); + let stream: BlobByteStream = + Box::pin(futures_util::stream::iter([Ok(Bytes::from_static(b"abc"))])); + + let error = put_local_blob_stream(root.path(), "objects/value", stream, 2) + .await + .expect_err("oversized stream must fail"); + + assert!(error.contains("exceeded declared length")); + assert!(!root.path().join("objects/value").exists()); +} diff --git a/crates/temper-server/src/blob_store/state.rs b/crates/temper-server/src/blob_store/state.rs new file mode 100644 index 000000000..80ce455ff --- /dev/null +++ b/crates/temper-server/src/blob_store/state.rs @@ -0,0 +1,271 @@ +use std::path::PathBuf; +use std::time::Duration; + +use sha2::{Digest as _, Sha256}; +use temper_runtime::tenant::TenantId; + +use super::{BlobReadBounded, BlobStore, DEFAULT_BLOB_BUCKET, is_local_internal_blob_endpoint}; +use crate::state::ServerState; + +impl ServerState { + pub(crate) fn blob_store_for_tenant(&self, tenant: &TenantId) -> Result { + if let Some(vault) = self.secrets_vault.as_ref() + && let Some(endpoint) = vault.get_secret(tenant.as_str(), "blob_endpoint") + && !endpoint.trim().is_empty() + { + if is_local_internal_blob_endpoint(&endpoint) { + return self.local_blob_store(tenant).ok_or_else(|| { + "internal DB-backed blob endpoint is disabled; set TEMPER_LOCAL_BLOB_DIR or configure BLOB_ENDPOINT for R2/S3" + .to_string() + }); + } + let bucket = vault + .get_secret(tenant.as_str(), "blob_bucket") + .unwrap_or_else(|| DEFAULT_BLOB_BUCKET.to_string()); + return Ok(BlobStore::s3( + endpoint, + bucket, + vault.get_secret(tenant.as_str(), "blob_access_key"), + vault.get_secret(tenant.as_str(), "blob_secret_key"), + self.blob_staging_root(tenant), + tenant_object_namespace(tenant).map(|namespace| format!("tenants/{namespace}")), + )); + } + self.local_blob_store(tenant).ok_or_else(|| { + "blob object store is not configured; set BLOB_ENDPOINT/BLOB_BUCKET/BLOB_ACCESS_KEY/BLOB_SECRET_KEY or TEMPER_LOCAL_BLOB_DIR" + .to_string() + }) + } + + pub(crate) async fn put_blob_object( + &self, + tenant: &TenantId, + key: &str, + body: &[u8], + ttl: Option, + ) -> Result<(), String> { + let store = self.blob_store_for_tenant(tenant)?; + store.put_if_absent(key, body, ttl).await?; + self.put_metadata_blob_shadow(tenant, key, body, ttl).await + } + + /// Write bytes to a tenant-scoped content-addressed blob key. + pub(crate) async fn put_content_addressed_blob( + &self, + tenant: &TenantId, + key: &str, + body: &[u8], + ttl: Option, + ) -> Result<(), String> { + let store = self.blob_store_for_tenant(tenant)?; + store.put_content_addressed(key, body, ttl).await?; + self.put_metadata_blob_shadow(tenant, key, body, ttl).await + } + + pub async fn get_blob_with_legacy_fallback( + &self, + tenant: &TenantId, + key: &str, + ) -> Result>, String> { + match self.blob_store_for_tenant(tenant) { + Ok(store) => match store.get(key).await { + Ok(Some(bytes)) => return Ok(Some(bytes)), + Ok(None) => {} + Err(error) => { + tracing::warn!(%key, %error, "object blob store read failed; trying legacy DB blob fallback"); + } + }, + Err(error) => { + tracing::debug!(%key, %error, "object blob store unavailable; trying legacy DB blob fallback"); + } + } + if tenant != &TenantId::default() { + return Ok(None); + } + let Some(store) = self.metadata_store_for_tenant(tenant.as_str()).await else { + return Ok(None); + }; + store + .get_blob(key) + .await + .map_err(|error| format!("legacy DB blob read failed for '{key}': {error}")) + } + + pub(crate) async fn get_blob_with_legacy_fallback_bounded( + &self, + tenant: &TenantId, + key: &str, + max_bytes: usize, + ) -> Result { + match self.blob_store_for_tenant(tenant) { + Ok(store) => match store.get_bounded(key, max_bytes).await { + Ok(BlobReadBounded::Found(bytes)) => { + return Ok(BlobReadBounded::Found(bytes)); + } + Ok(BlobReadBounded::TooLarge { actual_bytes }) => { + return Ok(BlobReadBounded::TooLarge { actual_bytes }); + } + Ok(BlobReadBounded::Missing) => {} + Err(error) => { + tracing::warn!(%key, %error, "bounded object blob read failed; trying legacy DB fallback"); + } + }, + Err(error) => { + tracing::debug!(%key, %error, "object blob store unavailable; trying bounded legacy DB fallback"); + } + } + if tenant != &TenantId::default() { + return Ok(BlobReadBounded::Missing); + } + let Some(store) = self.metadata_store_for_tenant(tenant.as_str()).await else { + return Ok(BlobReadBounded::Missing); + }; + store + .get_blob_if_size_at_most(key, max_bytes) + .await + .map(|bytes| match bytes { + Some(bytes) => BlobReadBounded::Found(bytes), + None => BlobReadBounded::Missing, + }) + .map_err(|error| format!("bounded legacy DB blob read failed for '{key}': {error}")) + } + + /// Open a tenant-scoped object-store blob as a bounded stream. + /// + /// Large field-overflow objects are never read from the legacy database + /// fallback because that interface is buffered; callers receive `Missing` + /// and can retain the media descriptor instead. + pub async fn stream_blob_object( + &self, + tenant: &TenantId, + key: &str, + max_bytes: u64, + ) -> Result { + self.blob_store_for_tenant(tenant)? + .get_stream(key, max_bytes) + .await + } + + fn local_blob_store(&self, tenant: &TenantId) -> Option { + if let Ok(root) = std::env::var("TEMPER_LOCAL_BLOB_DIR") // determinism-ok: deployment config read + && !root.trim().is_empty() + { + return Some(BlobStore::local_fs(tenant_blob_root(root.into(), tenant))); + } + if !self.data_dir.as_os_str().is_empty() { + return Some(BlobStore::local_fs(tenant_blob_root( + self.data_dir.join("blobs"), + tenant, + ))); + } + None + } + + fn blob_staging_root(&self, tenant: &TenantId) -> PathBuf { + let root = if !self.data_dir.as_os_str().is_empty() { + self.data_dir.join("blob-ingest-staging") + } else { + std::env::temp_dir() // determinism-ok: production object-store I/O staging path + .join("temper-blob-ingest-staging") + }; + tenant_blob_root(root, tenant) + } + + async fn put_metadata_blob_shadow( + &self, + tenant: &TenantId, + key: &str, + body: &[u8], + ttl: Option, + ) -> Result<(), String> { + if tenant != &TenantId::default() { + return Ok(()); + } + let Some(store) = self.metadata_store_for_tenant(tenant.as_str()).await else { + return Ok(()); + }; + store + .put_blob_with_ttl(key, body, ttl) + .await + .map_err(|error| format!("metadata blob shadow write failed for '{key}': {error}")) + } +} + +fn tenant_object_namespace(tenant: &TenantId) -> Option { + (tenant != &TenantId::default()) + .then(|| super::hex_lower(&Sha256::digest(tenant.as_str().as_bytes()))) +} + +fn tenant_blob_root(root: PathBuf, tenant: &TenantId) -> PathBuf { + match tenant_object_namespace(tenant) { + Some(namespace) => root.join("tenants").join(namespace), + None => root, + } +} + +#[cfg(test)] +mod tests { + use temper_runtime::ActorSystem; + use temper_spec::csdl::CsdlDocument; + + use super::*; + + #[test] + fn tenant_object_namespaces_are_stable_and_default_compatible() { + assert_eq!(tenant_object_namespace(&TenantId::default()), None); + let namespace = tenant_object_namespace(&TenantId::new("tenant-a")) + .expect("non-default tenant namespace"); + assert_eq!(namespace.len(), 64); + assert_eq!( + tenant_object_namespace(&TenantId::new("tenant-a")), + Some(namespace) + ); + } + + #[tokio::test] + async fn local_object_storage_is_namespaced_by_tenant() { + let data_dir = tempfile::tempdir().expect("data dir"); + let mut state = ServerState::new( + ActorSystem::new("tenant-blob-isolation"), + CsdlDocument { + version: "4.0".to_string(), + schemas: Vec::new(), + }, + String::new(), + ); + state.data_dir = data_dir.path().to_path_buf(); + let tenant_a = TenantId::new("tenant-a"); + let tenant_b = TenantId::new("tenant-b"); + + state + .put_blob_object(&tenant_a, "field-overflow/value", b"tenant-a", None) + .await + .expect("tenant A write"); + state + .put_blob_object(&tenant_b, "field-overflow/value", b"tenant-b", None) + .await + .expect("tenant B write"); + + assert_eq!( + state + .get_blob_with_legacy_fallback(&tenant_a, "field-overflow/value") + .await + .expect("tenant A read"), + Some(b"tenant-a".to_vec()) + ); + assert_eq!( + state + .get_blob_with_legacy_fallback(&tenant_b, "field-overflow/value") + .await + .expect("tenant B read"), + Some(b"tenant-b".to_vec()) + ); + assert_eq!( + state + .get_blob_with_legacy_fallback(&TenantId::new("tenant-c"), "field-overflow/value",) + .await + .expect("tenant C read"), + None + ); + } +} diff --git a/crates/temper-server/src/blob_store/streaming.rs b/crates/temper-server/src/blob_store/streaming.rs new file mode 100644 index 000000000..493c00168 --- /dev/null +++ b/crates/temper-server/src/blob_store/streaming.rs @@ -0,0 +1,493 @@ +use std::sync::{Arc, OnceLock}; +use std::time::{Duration, Instant}; + +use base64::Engine as _; +use bytes::Bytes; +use futures_util::StreamExt as _; +use reqwest::{Method, StatusCode}; +use sha2::{Digest as _, Sha256}; +use tokio::io::AsyncReadExt as _; +use tokio::sync::OwnedSemaphorePermit; +use tracing::Instrument as _; + +use super::{ + BLOB_IO_QUEUE_TIMEOUT, BlobByteStream, BlobStore, BlobStoreBackend, S3BlobStore, + blob_io_semaphore, keys::hex_lower, local_blob_path, +}; +use crate::blob_store::local::get_local_blob_bounded_observed; +use crate::blob_transport_observability::{ + BlobTransportError, BlobTransportFinish, blob_transport_span, finish_blob_transport, +}; + +const STREAM_CHUNK_BYTES: usize = 64 * 1024; +const BASE64_INPUT_CHUNK_BYTES: usize = 64 * 1024; +const BLOB_STREAM_OPEN_TIMEOUT: Duration = Duration::from_secs(30); +const BLOB_BOUNDED_READ_TIMEOUT: Duration = Duration::from_secs(2 * 60); +const BLOB_STREAM_IDLE_TIMEOUT: Duration = Duration::from_secs(30); +const BLOB_STREAM_TOTAL_TIMEOUT: Duration = Duration::from_secs(30 * 60); +const DEFAULT_BLOB_STREAM_MAX_CONCURRENCY: usize = 32; + +fn blob_stream_semaphore() -> Arc { + static SEMAPHORE: OnceLock> = OnceLock::new(); + Arc::clone(SEMAPHORE.get_or_init(|| { + let limit = std::env::var("TEMPER_BLOB_STREAM_MAX_CONCURRENCY") // determinism-ok: startup-only tuning knob + .ok() + .and_then(|value| value.parse::().ok()) + .filter(|value| *value > 0) + .unwrap_or(DEFAULT_BLOB_STREAM_MAX_CONCURRENCY); + Arc::new(tokio::sync::Semaphore::new(limit)) + })) +} + +#[derive(Debug)] +pub(crate) enum BlobReadBounded { + Found(Vec), + Missing, + TooLarge { actual_bytes: Option }, +} + +/// Streaming object-store response with a declared byte length. +pub struct BlobObjectStream { + content_length: u64, + stream: BlobByteStream, +} + +impl BlobObjectStream { + /// Number of bytes the stream must yield before completing. + pub fn content_length(&self) -> u64 { + self.content_length + } + + /// Consume this descriptor and return its bounded byte stream. + pub fn into_stream(self) -> BlobByteStream { + self.stream + } + + /// Verify the serialized object's content-addressed SHA-256 while reading. + pub fn verify_sha256(self, expected_sha256: &str) -> Self { + let content_length = self.content_length; + let expected_sha256 = expected_sha256.to_string(); + let mut source = self.stream; + let stream = Box::pin(async_stream::try_stream! { + let mut hasher = Sha256::new(); + while let Some(chunk) = source.next().await { + let chunk = chunk?; + hasher.update(&chunk); + yield chunk; + } + let actual = hex_lower(&hasher.finalize()); + if actual != expected_sha256 { + Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "field-overflow object failed SHA-256 verification", + ))?; + } + }); + Self { + content_length, + stream, + } + } + + fn hold_stream_permit(self, permit: OwnedSemaphorePermit) -> Self { + let content_length = self.content_length; + let mut source = self.stream; + let stream = Box::pin(async_stream::try_stream! { + let _permit = permit; + while let Some(chunk) = source.next().await { + yield chunk?; + } + }); + Self { + content_length, + stream, + } + } +} + +/// Result of opening an object as a bounded stream. +pub enum BlobStreamRead { + /// Object exists and is within the caller's encoded-size boundary. + Found(BlobObjectStream), + /// Object does not exist. + Missing, + /// Object metadata exceeds the caller's boundary. + TooLarge { actual_bytes: Option }, +} + +impl BlobStore { + pub(crate) async fn get_bounded( + &self, + key: &str, + max_bytes: usize, + ) -> Result { + let queued_at = Instant::now(); // determinism-ok: production blob I/O queue metric only + let _permit = + tokio::time::timeout(BLOB_IO_QUEUE_TIMEOUT, blob_io_semaphore().acquire_owned()) + .await + .map_err(|_| "bounded blob read queue deadline exceeded".to_string())? + .expect("blob semaphore closed"); // ci-ok: process-global and never closed + crate::runtime_metrics::record_blob_io_wait_duration(queued_at.elapsed(), "get_bounded"); + + match &self.backend { + BlobStoreBackend::LocalFs { root } => tokio::time::timeout( + BLOB_BOUNDED_READ_TIMEOUT, + get_local_blob_bounded_observed(root, key, max_bytes), + ) + .await + .map_err(|_| format!("bounded local blob read timed out for '{key}'"))?, + BlobStoreBackend::S3(store) => store.get_bounded(key, max_bytes).await, + } + } + + /// Open an object as a bounded stream without buffering it in memory. + pub(crate) async fn get_stream( + &self, + key: &str, + max_bytes: u64, + ) -> Result { + let queued_at = Instant::now(); // determinism-ok: production blob I/O queue metric only + let stream_permit = tokio::time::timeout( + BLOB_IO_QUEUE_TIMEOUT, + blob_stream_semaphore().acquire_owned(), + ) + .await + .map_err(|_| "blob stream concurrency queue deadline exceeded".to_string())? + .expect("blob semaphore closed"); // ci-ok: process-global and never closed + let _io_permit = + tokio::time::timeout(BLOB_IO_QUEUE_TIMEOUT, blob_io_semaphore().acquire_owned()) + .await + .map_err(|_| "blob I/O queue deadline exceeded".to_string())? + .expect("blob semaphore closed"); // ci-ok: process-global and never closed + crate::runtime_metrics::record_blob_io_wait_duration(queued_at.elapsed(), "get_stream"); + let opened = match &self.backend { + BlobStoreBackend::LocalFs { root } => open_local_stream(root, key, max_bytes).await, + BlobStoreBackend::S3(store) => store.get_stream(key, max_bytes).await, + }?; + Ok(match opened { + BlobStreamRead::Found(stream) => { + BlobStreamRead::Found(stream.hold_stream_permit(stream_permit)) + } + BlobStreamRead::Missing => BlobStreamRead::Missing, + BlobStreamRead::TooLarge { actual_bytes } => BlobStreamRead::TooLarge { actual_bytes }, + }) + } +} + +impl S3BlobStore { + async fn get_bounded(&self, key: &str, max_bytes: usize) -> Result { + let started_at = Instant::now(); // determinism-ok: production blob transport metric only + let span = blob_transport_span("get_bounded", "s3", 0); + let result = async { + let url = self.object_url(key); + let mut request = self.client.get(&url).timeout(BLOB_BOUNDED_READ_TIMEOUT); + let headers = self + .signed_headers(Method::GET, &url) + .map_err(BlobTransportError::message)?; + for (header_name, header_value) in &headers { + request = request.header(header_name, header_value); + } + + let response = request.send().await.map_err(|error| { + BlobTransportError::message(format!( + "bounded blob GET request failed for '{key}': {error}" + )) + })?; + let status = response.status(); + if status == StatusCode::NOT_FOUND { + return Ok((status, BlobReadBounded::Missing)); + } + if !status.is_success() { + return Err(BlobTransportError::status( + format!("bounded blob GET failed for '{key}' with HTTP {status}"), + status, + )); + } + if let Some(actual_bytes) = response.content_length() + && actual_bytes > max_bytes as u64 + { + return Ok(( + status, + BlobReadBounded::TooLarge { + actual_bytes: Some(actual_bytes), + }, + )); + } + + let mut stream = response.bytes_stream(); + let mut bytes = Vec::new(); + while let Some(chunk) = stream.next().await { + let chunk = chunk.map_err(|error| { + BlobTransportError::message(format!( + "bounded blob GET body failed for '{key}': {error}" + )) + })?; + if bytes.len().saturating_add(chunk.len()) > max_bytes { + return Ok((status, BlobReadBounded::TooLarge { actual_bytes: None })); + } + bytes.extend_from_slice(&chunk); + } + Ok((status, BlobReadBounded::Found(bytes))) + } + .instrument(span.clone()) + .await; + + match result { + Ok((status, outcome)) => { + let (label, response_bytes) = match &outcome { + BlobReadBounded::Found(bytes) => ("ok", bytes.len() as u64), + BlobReadBounded::Missing => ("not_found", 0), + BlobReadBounded::TooLarge { .. } => ("too_large", 0), + }; + finish_blob_transport(BlobTransportFinish { + started_at, + span: &span, + operation: "get_bounded", + backend: "s3", + outcome: label, + status: Some(status), + request_bytes: 0, + response_bytes, + }); + Ok(outcome) + } + Err(error) => { + finish_blob_transport(BlobTransportFinish { + started_at, + span: &span, + operation: "get_bounded", + backend: "s3", + outcome: "error", + status: error.status, + request_bytes: 0, + response_bytes: 0, + }); + Err(error.message) + } + } + } + + async fn get_stream(&self, key: &str, max_bytes: u64) -> Result { + let url = self.object_url(key); + let mut request = self.client.get(&url).timeout(BLOB_STREAM_TOTAL_TIMEOUT); + let headers = self.signed_headers(Method::GET, &url)?; + for (header_name, header_value) in &headers { + request = request.header(header_name, header_value); + } + let response = tokio::time::timeout(BLOB_STREAM_OPEN_TIMEOUT, request.send()) + .await + .map_err(|_| format!("streaming blob GET open timed out for '{key}'"))? + .map_err(|error| format!("streaming blob GET request failed for '{key}': {error}"))?; + let status = response.status(); + if status == StatusCode::NOT_FOUND { + return Ok(BlobStreamRead::Missing); + } + if !status.is_success() { + return Err(format!( + "streaming blob GET failed for '{key}' with HTTP {status}" + )); + } + let Some(content_length) = response.content_length() else { + return Err(format!( + "streaming blob GET for '{key}' omitted Content-Length" + )); + }; + if content_length > max_bytes { + return Ok(BlobStreamRead::TooLarge { + actual_bytes: Some(content_length), + }); + } + let source: BlobByteStream = Box::pin( + response + .bytes_stream() + .map(|item| item.map_err(std::io::Error::other)), + ); + Ok(BlobStreamRead::Found(BlobObjectStream { + content_length, + stream: enforce_stream_bounds(source, content_length), + })) + } +} + +async fn open_local_stream( + root: &std::path::Path, + key: &str, + max_bytes: u64, +) -> Result { + let path = local_blob_path(root, key)?; + let file = match tokio::time::timeout(BLOB_STREAM_OPEN_TIMEOUT, tokio::fs::File::open(&path)) + .await + .map_err(|_| format!("opening local blob '{}' timed out", path.display()))? + { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Ok(BlobStreamRead::Missing); + } + Err(error) => { + return Err(format!( + "failed to open local blob '{}': {error}", + path.display() + )); + } + }; + let content_length = tokio::time::timeout(BLOB_STREAM_OPEN_TIMEOUT, file.metadata()) + .await + .map_err(|_| format!("stating local blob '{}' timed out", path.display()))? + .map_err(|error| format!("failed to stat local blob '{}': {error}", path.display()))? + .len(); + if content_length > max_bytes { + return Ok(BlobStreamRead::TooLarge { + actual_bytes: Some(content_length), + }); + } + let source: BlobByteStream = Box::pin(async_stream::try_stream! { + let mut file = file; + let mut buffer = vec![0u8; STREAM_CHUNK_BYTES]; + loop { + let read = file.read(&mut buffer).await?; + if read == 0 { + break; + } + yield Bytes::copy_from_slice(&buffer[..read]); + } + }); + Ok(BlobStreamRead::Found(BlobObjectStream { + content_length, + stream: enforce_stream_bounds(source, content_length), + })) +} + +fn enforce_stream_bounds(mut source: BlobByteStream, expected_bytes: u64) -> BlobByteStream { + Box::pin(async_stream::try_stream! { + let started = tokio::time::Instant::now(); // determinism-ok: production object-store I/O deadline + let deadline = started + BLOB_STREAM_TOTAL_TIMEOUT; + let mut emitted = 0u64; + loop { + let now = tokio::time::Instant::now(); // determinism-ok: production object-store I/O deadline + if now >= deadline { + Err(std::io::Error::new(std::io::ErrorKind::TimedOut, "blob stream total deadline exceeded"))?; + } + let wait = BLOB_STREAM_IDLE_TIMEOUT.min(deadline.saturating_duration_since(now)); + let next = tokio::time::timeout(wait, source.next()) + .await + .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "blob stream idle deadline exceeded"))?; + let Some(chunk) = next else { + break; + }; + let chunk = chunk?; + emitted = emitted + .checked_add(chunk.len() as u64) + .ok_or_else(|| std::io::Error::other("blob stream byte count overflow"))?; + if emitted > expected_bytes { + Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "blob stream exceeded Content-Length"))?; + } + if !chunk.is_empty() { + yield chunk; + } + } + if emitted != expected_bytes { + Err(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + format!("blob stream ended at {emitted} bytes; expected {expected_bytes}"), + ))?; + } + }) +} + +/// Incrementally decode a JSON string containing standard base64. +pub fn decode_json_base64_stream( + encoded: BlobObjectStream, + expected_decoded_bytes: u64, +) -> BlobObjectStream { + let mut source = encoded.into_stream(); + let stream: BlobByteStream = Box::pin(async_stream::try_stream! { + let mut opened = false; + let mut pending = None; + let mut encoded_buffer = Vec::with_capacity(BASE64_INPUT_CHUNK_BYTES); + let mut decoded_bytes = 0u64; + let mut padding_seen = false; + + while let Some(chunk) = source.next().await { + let chunk = chunk?; + for byte in chunk { + if !opened { + if byte != b'"' { + Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "overflow blob is not a JSON string"))?; + } + opened = true; + continue; + } + if let Some(previous) = pending.replace(byte) { + push_base64_byte(previous, &mut encoded_buffer, padding_seen)?; + } + if encoded_buffer.len() == BASE64_INPUT_CHUNK_BYTES { + let group = base64::engine::general_purpose::STANDARD + .decode(&encoded_buffer) + .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?; + decoded_bytes = decoded_bytes + .checked_add(group.len() as u64) + .ok_or_else(|| std::io::Error::other("decoded blob byte count overflow"))?; + if decoded_bytes > expected_decoded_bytes { + Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "decoded blob exceeded expected length"))?; + } + padding_seen = encoded_buffer.contains(&b'='); + encoded_buffer.clear(); + yield Bytes::from(group); + } + } + } + if !opened || pending != Some(b'"') { + Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "overflow blob JSON string is truncated"))?; + } + if !encoded_buffer.is_empty() { + if encoded_buffer.len() % 4 != 0 { + Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "overflow blob has incomplete base64"))?; + } + let group = base64::engine::general_purpose::STANDARD + .decode(&encoded_buffer) + .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?; + decoded_bytes = decoded_bytes + .checked_add(group.len() as u64) + .ok_or_else(|| std::io::Error::other("decoded blob byte count overflow"))?; + if decoded_bytes > expected_decoded_bytes { + Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "decoded blob exceeded expected length"))?; + } + yield Bytes::from(group); + } + if decoded_bytes != expected_decoded_bytes { + Err(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + format!("decoded blob ended at {decoded_bytes} bytes; expected {expected_decoded_bytes}"), + ))?; + } + }); + BlobObjectStream { + content_length: expected_decoded_bytes, + stream, + } +} + +fn push_base64_byte( + byte: u8, + encoded_buffer: &mut Vec, + padding_seen: bool, +) -> Result<(), std::io::Error> { + if padding_seen { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "base64 data followed padding", + )); + } + if !(byte.is_ascii_alphanumeric() || matches!(byte, b'+' | b'/' | b'=')) { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "overflow blob contains non-base64 data", + )); + } + encoded_buffer.push(byte); + Ok(()) +} + +#[cfg(test)] +#[path = "streaming/tests.rs"] +mod tests; diff --git a/crates/temper-server/src/blob_store/streaming/tests.rs b/crates/temper-server/src/blob_store/streaming/tests.rs new file mode 100644 index 000000000..e73b1b1ac --- /dev/null +++ b/crates/temper-server/src/blob_store/streaming/tests.rs @@ -0,0 +1,103 @@ +use base64::Engine as _; +use bytes::Bytes; +use futures_util::{StreamExt as _, TryStreamExt as _, stream}; +use sha2::Digest as _; + +use super::{BlobByteStream, BlobObjectStream, decode_json_base64_stream}; + +fn object(chunks: &[&'static [u8]]) -> BlobObjectStream { + let content_length = chunks.iter().map(|chunk| chunk.len() as u64).sum(); + let chunks = chunks + .iter() + .map(|chunk| Ok::<_, std::io::Error>(Bytes::from_static(chunk))) + .collect::>(); + let stream: BlobByteStream = Box::pin(stream::iter(chunks)); + BlobObjectStream { + content_length, + stream, + } +} + +#[tokio::test] +async fn json_base64_decoder_handles_arbitrary_chunk_boundaries() { + let decoded = decode_json_base64_stream(object(&[b"\"Y", b"WJ", b"jZ", b"GVm", b"\""]), 6) + .into_stream() + .try_collect::>() + .await + .expect("decode split base64") + .concat(); + assert_eq!(decoded, b"abcdef"); +} + +#[tokio::test] +async fn json_base64_decoder_rejects_data_after_padding() { + let error = decode_json_base64_stream(object(&[b"\"YQ==x\""]), 1) + .into_stream() + .collect::>() + .await + .into_iter() + .find_map(Result::err) + .expect("invalid stream must fail"); + assert_eq!(error.kind(), std::io::ErrorKind::InvalidData); +} + +#[tokio::test] +async fn json_base64_decoder_rejects_decoded_length_mismatch() { + let error = decode_json_base64_stream(object(&[b"\"YQ==\""]), 2) + .into_stream() + .collect::>() + .await + .into_iter() + .find_map(Result::err) + .expect("short decoded stream must fail"); + assert_eq!(error.kind(), std::io::ErrorKind::UnexpectedEof); +} + +#[tokio::test] +async fn json_base64_decoder_coalesces_output_into_bounded_chunks() { + let expected = vec![0x5au8; 128 * 1024]; + let encoded = serde_json::to_vec(&base64::engine::general_purpose::STANDARD.encode(&expected)) + .expect("serialize base64 string"); + let content_length = encoded.len() as u64; + let source: BlobByteStream = Box::pin(stream::iter([Ok(Bytes::from(encoded))])); + let chunks = decode_json_base64_stream( + BlobObjectStream { + content_length, + stream: source, + }, + expected.len() as u64, + ) + .into_stream() + .try_collect::>() + .await + .expect("decode large base64 string"); + + assert_eq!( + chunks.len(), + 3, + "decoder must not emit one chunk per quartet" + ); + assert_eq!(chunks.concat(), expected); +} + +#[tokio::test] +async fn content_addressed_stream_verifies_sha256() { + let expected = format!("{:x}", sha2::Sha256::digest(b"abcdef")); + let verified = object(&[b"abc", b"def"]) + .verify_sha256(&expected) + .into_stream() + .try_collect::>() + .await + .expect("matching digest"); + assert_eq!(verified.concat(), b"abcdef"); + + let error = object(&[b"tampered"]) + .verify_sha256(&expected) + .into_stream() + .collect::>() + .await + .into_iter() + .find_map(Result::err) + .expect("digest mismatch must fail"); + assert_eq!(error.kind(), std::io::ErrorKind::InvalidData); +} diff --git a/crates/temper-server/src/blobs.rs b/crates/temper-server/src/blobs.rs index 5939ef1a9..a4d74fdc6 100644 --- a/crates/temper-server/src/blobs.rs +++ b/crates/temper-server/src/blobs.rs @@ -4,20 +4,62 @@ //! read only as a legacy fallback for data written by older releases. use axum::body::Bytes; -use axum::extract::{Path, State}; +use axum::extract::{Extension, Path, State}; use axum::http::StatusCode; use axum::response::IntoResponse; use serde_json::Value; -use temper_runtime::tenant::TenantId; +use temper_authz::AuthenticatedRequestContext; use crate::blob_store::BlobStore; use crate::state::ServerState; +mod descriptor; +mod hydration; +pub use descriptor::{ + FieldOverflowDescriptor, field_overflow_descriptor, field_overflow_sha256, + is_valid_field_overflow_key, +}; +pub(crate) use hydration::{ + BlobHydrationBudget, hydrate_blob_refs_for_tenant, hydrate_blob_refs_for_tenant_with_budget, +}; +#[cfg(test)] +pub(crate) use hydration::{hydrate_blob_refs_in_value, hydrate_blob_refs_in_value_with_ceiling}; + pub(crate) const FIELD_OVERFLOW_BLOB_PREFIX: &str = "field-overflow/sha256/"; pub(crate) const FIELD_OVERFLOW_REF_KEY: &str = "__temper_blob_ref"; pub(crate) const FIELD_OVERFLOW_SIZE_KEY: &str = "__temper_blob_size"; pub(crate) const FIELD_OVERFLOW_ENCODING_KEY: &str = "__temper_blob_encoding"; +fn require_blob_object_authorization( + state: &ServerState, + authenticated: &AuthenticatedRequestContext, + action: &str, + key: &str, +) -> Result<(), StatusCode> { + state + .authorize_with_context( + authenticated.security_context(), + action, + "BlobObject", + &std::collections::BTreeMap::from([( + "id".to_string(), + serde_json::Value::String(key.to_string()), + )]), + authenticated.tenant().as_str(), + ) + .map_err(|denial| { + tracing::warn!( + reason = %denial, + tenant = %authenticated.tenant(), + principal_id = %authenticated.security_context().principal.id, + key, + action, + "internal blob object access denied" + ); + StatusCode::FORBIDDEN + }) +} + #[derive(Debug, Clone)] pub struct OverflowBlobWrite { pub key: String, @@ -27,14 +69,6 @@ pub struct OverflowBlobWrite { pub ttl_seconds: Option, } -#[cfg(test)] -pub(crate) async fn get_blob_bytes( - store: &BlobStore, - key: &str, -) -> Result>, String> { - store.get(key).await -} - pub(crate) async fn put_overflow_blobs( store: &BlobStore, blobs: &[OverflowBlobWrite], @@ -64,207 +98,21 @@ pub(crate) fn blob_ref_value(key: &str, size_bytes: usize) -> Value { }) } -fn blob_ref_key(value: &Value) -> Option<&str> { - value - .as_object() - .and_then(|obj| obj.get(FIELD_OVERFLOW_REF_KEY)) - .and_then(|value| value.as_str()) -} - -fn collect_blob_ref_pointers(value: &Value, pointer: &str, out: &mut Vec) { - if blob_ref_key(value).is_some() { - out.push(pointer.to_string()); - return; - } - - match value { - Value::Array(values) => { - for (index, child) in values.iter().enumerate() { - let child_pointer = format!("{pointer}/{index}"); - collect_blob_ref_pointers(child, &child_pointer, out); - } - } - Value::Object(map) => { - for (key, child) in map { - let escaped = key.replace('~', "~0").replace('/', "~1"); - let child_pointer = format!("{pointer}/{escaped}"); - collect_blob_ref_pointers(child, &child_pointer, out); - } - } - Value::Null | Value::Bool(_) | Value::Number(_) | Value::String(_) => {} - } -} - -enum BlobReadSource<'a> { - #[cfg(test)] - Store(&'a BlobStore), - Tenant { - state: &'a ServerState, - tenant: &'a TenantId, - }, -} - -async fn read_blob_ref_bytes( - source: &BlobReadSource<'_>, - key: &str, -) -> Result>, String> { - match source { - #[cfg(test)] - BlobReadSource::Store(store) => get_blob_bytes(store, key).await, - BlobReadSource::Tenant { state, tenant } => { - state.get_blob_with_legacy_fallback(tenant, key).await - } - } -} - -#[cfg(test)] -pub(crate) async fn hydrate_blob_refs_in_value(store: &BlobStore, value: &mut Value) { - // OData callers want full inline hydration regardless of size. - let _deferred = hydrate_blob_refs_in_value_with_ceiling(store, value, usize::MAX).await; -} - -/// Hydrate blob refs in `value` below `max_inline_bytes` in place; return a -/// `BTreeMap` of blob keys to bytes for refs at or above the ceiling (the -/// "deferred" set). Callers that hand `value` off to a WASM guest forward -/// the deferred map as `blob_cache` so guests can resolve oversize fields -/// via `host_read_field_stream`. See ADR-0046. -#[cfg(test)] -pub(crate) async fn hydrate_blob_refs_in_value_with_ceiling( - store: &BlobStore, - value: &mut Value, - max_inline_bytes: usize, -) -> std::collections::BTreeMap> { - hydrate_blob_refs_with_source(&BlobReadSource::Store(store), value, max_inline_bytes).await -} - -async fn hydrate_blob_refs_with_source( - source: &BlobReadSource<'_>, - value: &mut Value, - max_inline_bytes: usize, -) -> std::collections::BTreeMap> { - use std::collections::BTreeMap; - - let mut deferred_blobs: BTreeMap> = BTreeMap::new(); - let mut pointers = Vec::new(); - collect_blob_ref_pointers(value, "", &mut pointers); - // DST: deterministic fetch order across runs with the same ref set. - pointers.sort(); - - for pointer in pointers { - let (key, declared_size) = { - let slot = if pointer.is_empty() { - Some(&*value) - } else { - value.pointer(&pointer) - }; - let Some(slot) = slot else { - continue; - }; - let key = slot - .as_object() - .and_then(|obj| obj.get(FIELD_OVERFLOW_REF_KEY)) - .and_then(|v| v.as_str()) - .map(str::to_owned); - let size = slot - .as_object() - .and_then(|obj| obj.get(FIELD_OVERFLOW_SIZE_KEY)) - .and_then(|v| v.as_u64()) - .map(|n| n as usize); - match key { - Some(k) => (k, size), - None => continue, - } - }; - - // Fast-path: if the envelope declares a size above the ceiling, don't - // fetch inline — just fetch once into the deferred map. - if let Some(size) = declared_size - && size > max_inline_bytes - { - match read_blob_ref_bytes(source, &key).await { - Ok(Some(bytes)) => { - deferred_blobs.insert(key, bytes); - } - Ok(None) => { - tracing::warn!(%key, "deferred field-overflow blob missing"); - } - Err(error) => { - tracing::warn!(%key, %error, "failed to fetch deferred field-overflow blob"); - } - } - continue; - } - - match read_blob_ref_bytes(source, &key).await { - Ok(Some(bytes)) => { - // Post-fetch size check in case the envelope lied (missing size key). - if bytes.len() > max_inline_bytes { - deferred_blobs.insert(key, bytes); - continue; - } - match serde_json::from_slice::(&bytes) { - Ok(restored) => { - if pointer.is_empty() { - *value = restored; - } else if let Some(slot) = value.pointer_mut(&pointer) { - *slot = restored; - } - } - Err(error) => { - tracing::warn!(%key, %error, "failed to decode hydrated field-overflow blob"); - } - } - } - Ok(None) => { - tracing::warn!(%key, "field-overflow blob missing during hydration"); - } - Err(error) => { - tracing::warn!(%key, %error, "failed to hydrate field-overflow blob"); - } - } - } - - deferred_blobs -} - -pub(crate) async fn hydrate_blob_refs_for_tenant( - state: &ServerState, - tenant: &TenantId, - value: &mut Value, -) { - let _deferred = - hydrate_blob_refs_with_source(&BlobReadSource::Tenant { state, tenant }, value, usize::MAX) - .await; -} - -/// Tenant-scoped variant of `hydrate_blob_refs_in_value_with_ceiling`. -/// -/// Returns an empty map if no Turso store is configured for the tenant — in -/// that case, the entity state stays untouched, which is consistent with -/// `hydrate_blob_refs_for_tenant`'s no-op behavior. -pub(crate) async fn hydrate_blob_refs_for_tenant_with_ceiling( - state: &ServerState, - tenant: &TenantId, - value: &mut Value, - max_inline_bytes: usize, -) -> std::collections::BTreeMap> { - hydrate_blob_refs_with_source( - &BlobReadSource::Tenant { state, tenant }, - value, - max_inline_bytes, - ) - .await -} - /// `PUT /_internal/blobs/{*path}` — store a blob. pub async fn put_blob( State(state): State, + Extension(authenticated): Extension, Path(path): Path, body: Bytes, ) -> impl IntoResponse { - let tenant = TenantId::new("default"); + let tenant = authenticated.tenant(); + if let Err(status) = + require_blob_object_authorization(&state, &authenticated, "write_blob_object", &path) + { + return status.into_response(); + } - match state.put_blob_object(&tenant, &path, &body, None).await { + match state.put_blob_object(tenant, &path, &body, None).await { Ok(()) => StatusCode::NO_CONTENT.into_response(), Err(e) => { tracing::error!(error = %e, path = %path, "blob put failed"); @@ -276,11 +124,17 @@ pub async fn put_blob( /// `GET /_internal/blobs/{*path}` — retrieve a blob. pub async fn get_blob( State(state): State, + Extension(authenticated): Extension, Path(path): Path, ) -> impl IntoResponse { - let tenant = TenantId::new("default"); + let tenant = authenticated.tenant(); + if let Err(status) = + require_blob_object_authorization(&state, &authenticated, "read_blob_object", &path) + { + return status.into_response(); + } - match state.get_blob_with_legacy_fallback(&tenant, &path).await { + match state.get_blob_with_legacy_fallback(tenant, &path).await { Ok(Some(data)) => ( StatusCode::OK, [(axum::http::header::CONTENT_TYPE, "application/octet-stream")], @@ -644,3 +498,11 @@ mod tests { ); } } + +#[cfg(test)] +#[path = "blobs/read_budget_tests.rs"] +mod read_budget_tests; + +#[cfg(test)] +#[path = "blobs/endpoint_tests.rs"] +mod endpoint_tests; diff --git a/crates/temper-server/src/blobs/descriptor.rs b/crates/temper-server/src/blobs/descriptor.rs new file mode 100644 index 000000000..23212f7e5 --- /dev/null +++ b/crates/temper-server/src/blobs/descriptor.rs @@ -0,0 +1,55 @@ +//! Validation for content-addressed field-overflow descriptors. + +use serde_json::Value; + +use super::{ + FIELD_OVERFLOW_BLOB_PREFIX, FIELD_OVERFLOW_ENCODING_KEY, FIELD_OVERFLOW_REF_KEY, + FIELD_OVERFLOW_SIZE_KEY, +}; + +/// Validated reference to a field-overflow JSON object. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct FieldOverflowDescriptor<'a> { + /// Content-addressed object-store key. + pub key: &'a str, + /// Lowercase SHA-256 of the serialized JSON object. + pub sha256: &'a str, + /// Serialized JSON byte length stored under `key`. + pub serialized_bytes: u64, +} + +/// Parse and validate a field-overflow descriptor. +pub fn field_overflow_descriptor(value: &Value) -> Option> { + let object = value.as_object()?; + let key = object.get(FIELD_OVERFLOW_REF_KEY)?.as_str()?; + let sha256 = field_overflow_sha256(key)?; + if object.get(FIELD_OVERFLOW_ENCODING_KEY)?.as_str()? != "json" { + return None; + } + let serialized_bytes = object.get(FIELD_OVERFLOW_SIZE_KEY)?.as_u64()?; + if serialized_bytes == 0 { + return None; + } + Some(FieldOverflowDescriptor { + key, + sha256, + serialized_bytes, + }) +} + +/// Extract the lowercase SHA-256 from a canonical field-overflow key. +pub fn field_overflow_sha256(key: &str) -> Option<&str> { + let digest = key + .strip_prefix(FIELD_OVERFLOW_BLOB_PREFIX)? + .strip_suffix(".json")?; + (digest.len() == 64 + && digest + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))) + .then_some(digest) +} + +/// Return whether `key` is a canonical field-overflow SHA-256 object key. +pub fn is_valid_field_overflow_key(key: &str) -> bool { + field_overflow_sha256(key).is_some() +} diff --git a/crates/temper-server/src/blobs/endpoint_tests.rs b/crates/temper-server/src/blobs/endpoint_tests.rs new file mode 100644 index 000000000..4b20832ef --- /dev/null +++ b/crates/temper-server/src/blobs/endpoint_tests.rs @@ -0,0 +1,201 @@ +use axum::body::{Body, to_bytes}; +use axum::http::{Method, Request, StatusCode}; +use temper_authz::{AuthenticatedRequestContext, SecurityContext}; +use temper_runtime::ActorSystem; +use temper_runtime::tenant::TenantId; +use tower::ServiceExt as _; + +use crate::registry::SpecRegistry; +use crate::state::ServerState; + +const KEY: &str = "field-overflow/sha256/shared.json"; + +fn install_blob_policy(state: &ServerState, tenant: &str) { + state + .authz + .reload_tenant_policies( + tenant, + &format!( + r#" +permit( + principal == Agent::"blob-client", + action == Action::"write_blob_object", + resource == BlobObject::"{KEY}" +); +permit( + principal == Agent::"blob-client", + action == Action::"read_blob_object", + resource == BlobObject::"{KEY}" +); +"#, + ), + ) + .expect("blob policy should parse"); +} + +fn authenticated_request(method: Method, path: &str, tenant: &str, body: Body) -> Request { + let mut request = Request::builder() + .method(method) + .uri(path) + .body(body) + .expect("internal blob request"); + request + .extensions_mut() + .insert(AuthenticatedRequestContext::new( + TenantId::new(tenant), + SecurityContext::from_resolved_identity("blob-client", "worker", None), + )); + request +} + +fn claimed_admin_request(method: Method, path: &str, tenant: &str, body: Body) -> Request { + let mut request = Request::builder() + .method(method) + .uri(path) + .body(body) + .expect("internal blob admin request"); + request + .extensions_mut() + .insert(AuthenticatedRequestContext::new( + TenantId::new(tenant), + SecurityContext { + principal: temper_authz::Principal { + id: "claimed-admin".to_string(), + kind: temper_authz::PrincipalKind::Admin, + role: None, + acting_for: None, + agent_type: None, + attributes: Default::default(), + }, + context_attrs: Default::default(), + correlation_id: "blob-admin-side-channel-test".to_string(), + }, + )); + request +} + +#[tokio::test] +async fn internal_blob_http_storage_is_bound_to_the_authenticated_tenant() { + let data_dir = tempfile::tempdir().expect("blob data directory"); + let mut state = ServerState::from_registry( + ActorSystem::new("tenant-bound-internal-blob"), + SpecRegistry::new(), + ); + state.data_dir = data_dir.path().to_path_buf(); + for tenant in ["tenant-a", "tenant-b", "default"] { + install_blob_policy(&state, tenant); + } + let app = crate::build_router(state.clone()); + let key_path = format!("/_internal/blobs/{KEY}"); + + for (tenant, value) in [("tenant-a", "alpha"), ("tenant-b", "bravo")] { + let response = app + .clone() + .oneshot(authenticated_request( + Method::PUT, + &key_path, + tenant, + Body::from(value), + )) + .await + .expect("tenant blob write"); + assert_eq!(response.status(), StatusCode::NO_CONTENT, "{tenant}"); + } + + for (tenant, expected) in [ + ("tenant-a", b"alpha".as_slice()), + ("tenant-b", b"bravo".as_slice()), + ] { + let response = app + .clone() + .oneshot(authenticated_request( + Method::GET, + &key_path, + tenant, + Body::empty(), + )) + .await + .expect("tenant blob read"); + assert_eq!(response.status(), StatusCode::OK, "{tenant}"); + let bytes = to_bytes(response.into_body(), 32) + .await + .expect("bounded blob response"); + assert_eq!(bytes.as_ref(), expected, "{tenant}"); + } + + let default_response = app + .oneshot(authenticated_request( + Method::GET, + &key_path, + "default", + Body::empty(), + )) + .await + .expect("default tenant blob read"); + assert_eq!(default_response.status(), StatusCode::NOT_FOUND); + + assert_eq!( + state + .get_blob_with_legacy_fallback(&TenantId::new("tenant-a"), KEY,) + .await + .expect("tenant A direct read"), + Some(b"alpha".to_vec()) + ); + assert_eq!( + state + .get_blob_with_legacy_fallback(&TenantId::new("tenant-b"), KEY,) + .await + .expect("tenant B direct read"), + Some(b"bravo".to_vec()) + ); +} + +#[tokio::test] +async fn internal_blob_http_requires_exact_object_authority() { + let data_dir = tempfile::tempdir().expect("blob data directory"); + let mut state = ServerState::from_registry( + ActorSystem::new("exact-internal-blob-auth"), + SpecRegistry::new(), + ); + state.data_dir = data_dir.path().to_path_buf(); + install_blob_policy(&state, "tenant-a"); + let app = crate::build_router(state); + + let allowed = app + .clone() + .oneshot(authenticated_request( + Method::PUT, + &format!("/_internal/blobs/{KEY}"), + "tenant-a", + Body::from("allowed"), + )) + .await + .expect("allowed blob request should run"); + assert_eq!(allowed.status(), StatusCode::NO_CONTENT); + + let claimed_admin = app + .clone() + .oneshot(claimed_admin_request( + Method::PUT, + &format!("/_internal/blobs/{KEY}"), + "tenant-a", + Body::from("must-not-write"), + )) + .await + .expect("claimed-admin blob request should run"); + assert_eq!(claimed_admin.status(), StatusCode::FORBIDDEN); + + for method in [Method::GET, Method::PUT] { + let denied = app + .clone() + .oneshot(authenticated_request( + method.clone(), + "/_internal/blobs/field-overflow/sha256/sibling.json", + "tenant-a", + Body::from("denied"), + )) + .await + .expect("denied blob request should run"); + assert_eq!(denied.status(), StatusCode::FORBIDDEN, "{method}"); + } +} diff --git a/crates/temper-server/src/blobs/hydration.rs b/crates/temper-server/src/blobs/hydration.rs new file mode 100644 index 000000000..1d6a52e89 --- /dev/null +++ b/crates/temper-server/src/blobs/hydration.rs @@ -0,0 +1,415 @@ +//! Aggregate-bounded field-overflow hydration. + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::{Arc, Mutex}; + +use serde_json::Value; +use sha2::{Digest as _, Sha256}; +use temper_runtime::tenant::TenantId; + +use super::{field_overflow_descriptor, field_overflow_sha256}; +#[cfg(test)] +use crate::blob_store::BlobStore; +use crate::blob_store::{BlobReadBounded, hex_lower}; +use crate::state::ServerState; + +const GENERIC_INLINE_HYDRATION_BUDGET_BYTES: usize = 1024 * 1024; +const GENERIC_MAX_INLINE_FIELD_BYTES: usize = 128 * 1024; +const WASM_DEFERRED_BLOB_BUDGET_BYTES: usize = 16 * 1024 * 1024; +const WASM_MAX_DEFERRED_BLOB_BYTES: usize = 8 * 1024 * 1024; +const MAX_BLOB_REFS_PER_VALUE: usize = 1024; +const MAX_BLOB_READ_ATTEMPTS_PER_RESPONSE: usize = 64; + +#[derive(Clone, Debug)] +pub(crate) struct BlobHydrationBudget { + inner: Arc>, + max_inline_field_bytes: usize, + max_deferred_field_bytes: usize, +} + +#[derive(Debug)] +struct BlobHydrationBudgetState { + inline_remaining: usize, + deferred_remaining: usize, + read_attempts_remaining: usize, + failed_keys: BTreeSet, +} + +#[derive(Clone, Copy, Debug)] +enum HydrationBudgetKind { + Inline, + Deferred, +} + +#[derive(Debug)] +struct HydrationReservation { + budget: BlobHydrationBudget, + kind: HydrationBudgetKind, + reserved: usize, + committed: bool, +} + +impl BlobHydrationBudget { + pub(crate) fn generic_response() -> Self { + Self::new( + GENERIC_INLINE_HYDRATION_BUDGET_BYTES, + GENERIC_MAX_INLINE_FIELD_BYTES, + 0, + 0, + ) + } + + pub(crate) fn wasm_dispatch() -> Self { + Self::new( + GENERIC_INLINE_HYDRATION_BUDGET_BYTES, + GENERIC_MAX_INLINE_FIELD_BYTES, + WASM_DEFERRED_BLOB_BUDGET_BYTES, + WASM_MAX_DEFERRED_BLOB_BYTES, + ) + } + + pub(crate) fn new( + inline_bytes: usize, + max_inline_field_bytes: usize, + deferred_bytes: usize, + max_deferred_field_bytes: usize, + ) -> Self { + Self { + inner: Arc::new(Mutex::new(BlobHydrationBudgetState { + inline_remaining: inline_bytes, + deferred_remaining: deferred_bytes, + read_attempts_remaining: MAX_BLOB_READ_ATTEMPTS_PER_RESPONSE, + failed_keys: BTreeSet::new(), + })), + max_inline_field_bytes, + max_deferred_field_bytes, + } + } + + fn try_reserve_inline(&self, declared_size: usize) -> Option { + self.try_reserve( + HydrationBudgetKind::Inline, + declared_size, + self.max_inline_field_bytes, + ) + } + + fn try_reserve_deferred(&self, declared_size: usize) -> Option { + self.try_reserve( + HydrationBudgetKind::Deferred, + declared_size, + self.max_deferred_field_bytes, + ) + } + + fn try_reserve( + &self, + kind: HydrationBudgetKind, + requested: usize, + max_field_bytes: usize, + ) -> Option { + if requested == 0 || requested > max_field_bytes { + return None; + } + let mut state = self.inner.lock().unwrap_or_else(|error| error.into_inner()); + let remaining = match kind { + HydrationBudgetKind::Inline => &mut state.inline_remaining, + HydrationBudgetKind::Deferred => &mut state.deferred_remaining, + }; + if requested > *remaining { + return None; + } + *remaining -= requested; + Some(HydrationReservation { + budget: self.clone(), + kind, + reserved: requested, + committed: false, + }) + } + + #[cfg(test)] + pub(super) fn remaining(&self) -> (usize, usize) { + let state = self.inner.lock().unwrap_or_else(|error| error.into_inner()); + (state.inline_remaining, state.deferred_remaining) + } + + fn refund(&self, kind: HydrationBudgetKind, bytes: usize) { + let mut state = self.inner.lock().unwrap_or_else(|error| error.into_inner()); + let remaining = match kind { + HydrationBudgetKind::Inline => &mut state.inline_remaining, + HydrationBudgetKind::Deferred => &mut state.deferred_remaining, + }; + *remaining = remaining.saturating_add(bytes); + } + + fn try_begin_read(&self) -> bool { + let mut state = self.inner.lock().unwrap_or_else(|error| error.into_inner()); + if state.read_attempts_remaining == 0 { + return false; + } + state.read_attempts_remaining -= 1; + true + } + + fn is_known_failed(&self, key: &str) -> bool { + self.inner + .lock() + .unwrap_or_else(|error| error.into_inner()) + .failed_keys + .contains(key) + } + + fn mark_failed(&self, key: &str) { + self.inner + .lock() + .unwrap_or_else(|error| error.into_inner()) + .failed_keys + .insert(key.to_string()); + } + + #[cfg(test)] + pub(crate) fn read_attempts_remaining(&self) -> usize { + self.inner + .lock() + .unwrap_or_else(|error| error.into_inner()) + .read_attempts_remaining + } +} + +impl HydrationReservation { + fn max_bytes(&self) -> usize { + self.reserved + } + + fn commit(mut self, actual_bytes: usize) { + debug_assert!(actual_bytes <= self.reserved); + let refund = self.reserved.saturating_sub(actual_bytes); + if refund > 0 { + self.budget.refund(self.kind, refund); + } + self.committed = true; + } +} + +impl Drop for HydrationReservation { + fn drop(&mut self) { + if !self.committed { + self.budget.refund(self.kind, self.reserved); + } + } +} + +fn collect_blob_ref_pointers(value: &Value, pointer: &str, out: &mut Vec) { + if out.len() >= MAX_BLOB_REFS_PER_VALUE { + return; + } + if field_overflow_descriptor(value).is_some() { + out.push(pointer.to_string()); + return; + } + + match value { + Value::Array(values) => { + for (index, child) in values.iter().enumerate() { + collect_blob_ref_pointers(child, &format!("{pointer}/{index}"), out); + } + } + Value::Object(map) => { + for (key, child) in map { + let escaped = key.replace('~', "~0").replace('/', "~1"); + collect_blob_ref_pointers(child, &format!("{pointer}/{escaped}"), out); + } + } + Value::Null | Value::Bool(_) | Value::Number(_) | Value::String(_) => {} + } +} + +pub(super) enum BlobReadSource<'a> { + #[cfg(test)] + Store(&'a BlobStore), + Tenant { + state: &'a ServerState, + tenant: &'a TenantId, + }, +} + +async fn read_blob_ref_bytes( + source: &BlobReadSource<'_>, + key: &str, + max_bytes: usize, +) -> Result { + match source { + #[cfg(test)] + BlobReadSource::Store(store) => store.get_bounded(key, max_bytes).await, + BlobReadSource::Tenant { state, tenant } => { + state + .get_blob_with_legacy_fallback_bounded(tenant, key, max_bytes) + .await + } + } +} + +fn blob_bytes_match_key(key: &str, bytes: &[u8]) -> bool { + let Some(expected) = field_overflow_sha256(key) else { + return false; + }; + hex_lower(&Sha256::digest(bytes)) == expected +} + +#[cfg(test)] +pub(crate) async fn hydrate_blob_refs_in_value(store: &BlobStore, value: &mut Value) { + let budget = BlobHydrationBudget::new(16 * 1024 * 1024, 16 * 1024 * 1024, 0, 0); + let _deferred = + hydrate_blob_refs_with_source(&BlobReadSource::Store(store), value, &budget).await; +} + +/// Hydrate refs below `max_inline_bytes`; return larger values for WASM streaming. +#[cfg(test)] +pub(crate) async fn hydrate_blob_refs_in_value_with_ceiling( + store: &BlobStore, + value: &mut Value, + max_inline_bytes: usize, +) -> BTreeMap> { + let budget = BlobHydrationBudget::new( + max_inline_bytes, + max_inline_bytes, + 16 * 1024 * 1024, + 16 * 1024 * 1024, + ); + hydrate_blob_refs_with_source(&BlobReadSource::Store(store), value, &budget).await +} + +pub(super) async fn hydrate_blob_refs_with_source( + source: &BlobReadSource<'_>, + value: &mut Value, + budget: &BlobHydrationBudget, +) -> BTreeMap> { + let mut deferred_blobs = BTreeMap::new(); + let mut pointers = Vec::new(); + collect_blob_ref_pointers(value, "", &mut pointers); + pointers.sort(); + + for pointer in pointers { + let Some((key, declared_size)) = (|| { + let slot = if pointer.is_empty() { + Some(&*value) + } else { + value.pointer(&pointer) + }?; + let descriptor = field_overflow_descriptor(slot)?; + let size = usize::try_from(descriptor.serialized_bytes).ok()?; + Some((descriptor.key.to_owned(), size)) + })() else { + continue; + }; + if budget.is_known_failed(&key) { + continue; + } + + if let Some(reservation) = budget.try_reserve_inline(declared_size) { + if !budget.try_begin_read() { + continue; + } + match read_blob_ref_bytes(source, &key, reservation.max_bytes()).await { + Ok(BlobReadBounded::Found(bytes)) => { + if bytes.len() != declared_size { + tracing::warn!(%key, actual_bytes = bytes.len(), declared_size, "field-overflow blob length did not match its descriptor"); + budget.mark_failed(&key); + continue; + } + if !blob_bytes_match_key(&key, &bytes) { + tracing::warn!(%key, "field-overflow blob failed SHA-256 verification"); + budget.mark_failed(&key); + continue; + } + match serde_json::from_slice::(&bytes) { + Ok(restored) => { + reservation.commit(bytes.len()); + if pointer.is_empty() { + *value = restored; + } else if let Some(slot) = value.pointer_mut(&pointer) { + *slot = restored; + } + } + Err(error) => { + tracing::warn!(%key, %error, "failed to decode hydrated field-overflow blob"); + budget.mark_failed(&key); + } + } + } + Ok(BlobReadBounded::Missing) => { + tracing::warn!(%key, "field-overflow blob missing during hydration"); + budget.mark_failed(&key); + } + Ok(BlobReadBounded::TooLarge { actual_bytes }) => { + tracing::warn!(%key, ?actual_bytes, "field-overflow blob exceeded inline hydration reservation"); + budget.mark_failed(&key); + } + Err(error) => { + tracing::warn!(%key, %error, "failed to hydrate field-overflow blob"); + budget.mark_failed(&key); + } + } + continue; + } + + if deferred_blobs.contains_key(&key) { + continue; + } + let Some(reservation) = budget.try_reserve_deferred(declared_size) else { + continue; + }; + if !budget.try_begin_read() { + continue; + } + match read_blob_ref_bytes(source, &key, reservation.max_bytes()).await { + Ok(BlobReadBounded::Found(bytes)) => { + if bytes.len() != declared_size { + tracing::warn!(%key, actual_bytes = bytes.len(), declared_size, "deferred field-overflow blob length did not match its descriptor"); + budget.mark_failed(&key); + continue; + } + if !blob_bytes_match_key(&key, &bytes) { + tracing::warn!(%key, "deferred field-overflow blob failed SHA-256 verification"); + budget.mark_failed(&key); + continue; + } + reservation.commit(bytes.len()); + deferred_blobs.insert(key, bytes); + } + Ok(BlobReadBounded::Missing) => { + tracing::warn!(%key, "deferred field-overflow blob missing"); + budget.mark_failed(&key); + } + Ok(BlobReadBounded::TooLarge { actual_bytes }) => { + tracing::warn!(%key, ?actual_bytes, "deferred field-overflow blob exceeded cache reservation"); + budget.mark_failed(&key); + } + Err(error) => { + tracing::warn!(%key, %error, "failed to fetch deferred field-overflow blob"); + budget.mark_failed(&key); + } + } + } + + deferred_blobs +} + +pub(crate) async fn hydrate_blob_refs_for_tenant( + state: &ServerState, + tenant: &TenantId, + value: &mut Value, +) { + let budget = BlobHydrationBudget::generic_response(); + let _deferred = hydrate_blob_refs_for_tenant_with_budget(state, tenant, value, &budget).await; +} + +pub(crate) async fn hydrate_blob_refs_for_tenant_with_budget( + state: &ServerState, + tenant: &TenantId, + value: &mut Value, + budget: &BlobHydrationBudget, +) -> BTreeMap> { + hydrate_blob_refs_with_source(&BlobReadSource::Tenant { state, tenant }, value, budget).await +} diff --git a/crates/temper-server/src/blobs/read_budget_tests.rs b/crates/temper-server/src/blobs/read_budget_tests.rs new file mode 100644 index 000000000..739498b35 --- /dev/null +++ b/crates/temper-server/src/blobs/read_budget_tests.rs @@ -0,0 +1,185 @@ +use serde_json::json; +use sha2::{Digest as _, Sha256}; + +use super::hydration::{BlobReadSource, hydrate_blob_refs_with_source}; +use super::{ + BlobHydrationBudget, FIELD_OVERFLOW_REF_KEY, blob_ref_value, field_overflow_descriptor, +}; +use crate::blob_store::BlobStore; + +async fn store_json(store: &BlobStore, value: &serde_json::Value) -> (String, usize) { + let bytes = serde_json::to_vec(value).expect("serialize overflow JSON"); + let key = format!("field-overflow/sha256/{:x}.json", Sha256::digest(&bytes)); + store + .put_if_absent(&key, &bytes, None) + .await + .expect("store overflow JSON"); + (key, bytes.len()) +} + +#[test] +fn descriptor_requires_the_canonical_json_encoding() { + let key = format!("field-overflow/sha256/{}.json", "0".repeat(64)); + let missing_encoding = json!({ + "__temper_blob_ref": key, + "__temper_blob_size": 4, + }); + let wrong_encoding = json!({ + "__temper_blob_ref": key, + "__temper_blob_size": 4, + "__temper_blob_encoding": "raw", + }); + + assert!(field_overflow_descriptor(&missing_encoding).is_none()); + assert!(field_overflow_descriptor(&wrong_encoding).is_none()); +} + +#[tokio::test] +async fn aggregate_inline_budget_is_shared_across_all_refs() { + let dir = tempfile::tempdir().expect("blob dir"); + let store = BlobStore::local_fs(dir.path()); + let (first_key, first_len) = store_json(&store, &json!("a".repeat(400))).await; + let (second_key, second_len) = store_json(&store, &json!("b".repeat(400))).await; + assert_eq!(first_len, second_len); + let mut value = json!({ + "a": blob_ref_value(&first_key, first_len), + "b": blob_ref_value(&second_key, second_len), + }); + let budget = BlobHydrationBudget::new(700, 500, 0, 0); + + let deferred = + hydrate_blob_refs_with_source(&BlobReadSource::Store(&store), &mut value, &budget).await; + + assert_eq!(value["a"].as_str().map(str::len), Some(400)); + assert!(value["b"].get(FIELD_OVERFLOW_REF_KEY).is_some()); + assert!(deferred.is_empty()); + assert_eq!(budget.remaining(), (700 - first_len, 0)); +} + +#[tokio::test] +async fn lying_small_descriptor_cannot_trigger_oversized_buffered_read() { + let dir = tempfile::tempdir().expect("blob dir"); + let store = BlobStore::local_fs(dir.path()); + let (key, _) = store_json(&store, &json!("x".repeat(64 * 1024))).await; + let mut value = blob_ref_value(&key, 4); + let budget = BlobHydrationBudget::new(1024, 1024, 0, 0); + + hydrate_blob_refs_with_source(&BlobReadSource::Store(&store), &mut value, &budget).await; + + assert_eq!(value[FIELD_OVERFLOW_REF_KEY], key); + assert_eq!( + budget.remaining(), + (1024, 0), + "failed bounded reads refund admission" + ); +} + +#[tokio::test] +async fn noncanonical_overflow_key_is_never_read() { + let dir = tempfile::tempdir().expect("blob dir"); + let store = BlobStore::local_fs(dir.path()); + let key = "wasm-modules/not-a-field-overflow-object"; + store + .put_if_absent(key, br#"{"secret":true}"#, None) + .await + .expect("store noncanonical object"); + let mut value = blob_ref_value(key, 15); + let budget = BlobHydrationBudget::new(1024, 1024, 0, 0); + + hydrate_blob_refs_with_source(&BlobReadSource::Store(&store), &mut value, &budget).await; + + assert_eq!(value[FIELD_OVERFLOW_REF_KEY], key); + assert_eq!(budget.remaining(), (1024, 0)); +} + +#[tokio::test] +async fn descriptor_length_must_match_the_stored_object() { + let dir = tempfile::tempdir().expect("blob dir"); + let store = BlobStore::local_fs(dir.path()); + let (key, actual) = store_json(&store, &json!("short")).await; + let mut value = blob_ref_value(&key, actual + 10); + let budget = BlobHydrationBudget::new(1024, 1024, 0, 0); + + hydrate_blob_refs_with_source(&BlobReadSource::Store(&store), &mut value, &budget).await; + + assert_eq!(value[FIELD_OVERFLOW_REF_KEY], key); + assert_eq!(budget.remaining(), (1024, 0)); +} + +#[tokio::test] +async fn missing_blob_reads_have_an_aggregate_attempt_budget() { + let dir = tempfile::tempdir().expect("blob dir"); + let store = BlobStore::local_fs(dir.path()); + let mut values = serde_json::Map::new(); + for index in 0..100u64 { + let key = format!("field-overflow/sha256/{index:064x}.json"); + values.insert(index.to_string(), blob_ref_value(&key, 4)); + } + let mut value = serde_json::Value::Object(values); + let budget = BlobHydrationBudget::new(1024, 16, 0, 0); + + hydrate_blob_refs_with_source(&BlobReadSource::Store(&store), &mut value, &budget).await; + + assert_eq!(budget.read_attempts_remaining(), 0); + assert!( + value + .as_object() + .expect("object") + .values() + .all(|value| value.get(FIELD_OVERFLOW_REF_KEY).is_some()) + ); +} + +#[tokio::test] +async fn a_repeated_missing_key_is_read_only_once() { + let dir = tempfile::tempdir().expect("blob dir"); + let store = BlobStore::local_fs(dir.path()); + let key = format!("field-overflow/sha256/{}.json", "5".repeat(64)); + let mut value = serde_json::Value::Array((0..10).map(|_| blob_ref_value(&key, 4)).collect()); + let budget = BlobHydrationBudget::new(1024, 16, 0, 0); + + hydrate_blob_refs_with_source(&BlobReadSource::Store(&store), &mut value, &budget).await; + + assert_eq!(budget.read_attempts_remaining(), 63); +} + +#[tokio::test] +async fn object_bytes_must_match_the_content_addressed_key() { + let dir = tempfile::tempdir().expect("blob dir"); + let store = BlobStore::local_fs(dir.path()); + let bytes = serde_json::to_vec(&json!("tampered")).expect("serialize value"); + let key = format!("field-overflow/sha256/{}.json", "6".repeat(64)); + store + .put_if_absent(&key, &bytes, None) + .await + .expect("store corrupted content-addressed object"); + let mut value = blob_ref_value(&key, bytes.len()); + let budget = BlobHydrationBudget::new(1024, 1024, 0, 0); + + hydrate_blob_refs_with_source(&BlobReadSource::Store(&store), &mut value, &budget).await; + + assert_eq!(value[FIELD_OVERFLOW_REF_KEY], key); + assert_eq!(budget.remaining(), (1024, 0)); +} + +#[tokio::test] +async fn wasm_deferred_cache_has_a_hard_aggregate_budget() { + let dir = tempfile::tempdir().expect("blob dir"); + let store = BlobStore::local_fs(dir.path()); + let (first_key, first_len) = store_json(&store, &json!("a".repeat(300))).await; + let (second_key, second_len) = store_json(&store, &json!("b".repeat(300))).await; + let mut value = json!({ + "a": blob_ref_value(&first_key, first_len), + "b": blob_ref_value(&second_key, second_len), + }); + let budget = BlobHydrationBudget::new(0, 0, 500, 400); + + let deferred = + hydrate_blob_refs_with_source(&BlobReadSource::Store(&store), &mut value, &budget).await; + + assert_eq!(deferred.len(), 1); + assert!(deferred.contains_key(&first_key)); + assert!(value["a"].get(FIELD_OVERFLOW_REF_KEY).is_some()); + assert!(value["b"].get(FIELD_OVERFLOW_REF_KEY).is_some()); + assert_eq!(budget.remaining(), (0, 500 - first_len)); +} diff --git a/crates/temper-server/src/conformance/conformance_test.rs b/crates/temper-server/src/conformance/conformance_test.rs new file mode 100644 index 000000000..f611f3ef8 --- /dev/null +++ b/crates/temper-server/src/conformance/conformance_test.rs @@ -0,0 +1,977 @@ +//! Conformance checker tests. +//! +//! Every violation kind gets a case asserting the exact index and kind, so a +//! change in walk order or in what counts as a violation shows up as a failing +//! assertion rather than a silently different report. + +use super::*; +use temper_ots::models::{ + DecisionType, OTSChoice, OTSConsequence, OTSDecision, OTSMetadata, OTSTrajectory, OTSTurn, + OutcomeType, +}; +use temper_spec::automaton::parse_automaton; + +const ORDER_IOA: &str = include_str!("../../../../test-fixtures/specs/order.ioa.toml"); + +/// An actor whose only action constrains its source states through a +/// `state_in` guard rather than a `from` list. +const GUARD_ONLY_IOA: &str = r#" +[automaton] +name = "Ticket" +states = ["Open", "Working", "Closed"] +initial = "Open" + +[[action]] +name = "Work" +kind = "input" +guard = [{ type = "state_in", values = ["Open"] }] +to = "Working" + +[[action]] +name = "Close" +kind = "input" +from = ["Working"] +to = "Closed" +"#; + +/// An actor with an always-enabled input action: no `from`, no `state_in`. +const ALWAYS_ENABLED_IOA: &str = r#" +[automaton] +name = "Beacon" +states = ["Active", "Closed"] +initial = "Active" + +[[action]] +name = "Close" +kind = "input" +from = ["Active"] +to = "Closed" + +[[action]] +name = "Heartbeat" +kind = "input" +"#; + +fn order_automaton() -> temper_spec::automaton::Automaton { + parse_automaton(ORDER_IOA).expect("order fixture parses") +} + +/// A successful entity-sourced row for `action`, moving `from` -> `to`. +fn row(action: &str, from: Option<&str>, to: Option<&str>) -> TursoTrajectoryRow { + TursoTrajectoryRow { + tenant: "default".to_string(), + entity_type: "Order".to_string(), + entity_id: "order-1".to_string(), + action: action.to_string(), + success: true, + from_status: from.map(str::to_string), + to_status: to.map(str::to_string), + error: None, + agent_id: Some("agent-1".to_string()), + session_id: Some("session-1".to_string()), + authz_denied: None, + denied_resource: None, + denied_module: None, + source: Some("Entity".to_string()), + spec_governed: Some(true), + created_at: "2026-01-01T00:00:00Z".to_string(), + request_body: None, + intent: None, + matched_policy_ids: None, + capture_seq: None, + } +} + +fn denied(action: &str, from: Option<&str>) -> TursoTrajectoryRow { + TursoTrajectoryRow { + success: false, + to_status: None, + authz_denied: Some(true), + error: Some("Cedar denied".to_string()), + source: Some("Authz".to_string()), + ..row(action, from, None) + } +} + +fn failed(action: &str, from: Option<&str>) -> TursoTrajectoryRow { + TursoTrajectoryRow { + success: false, + to_status: None, + error: Some("dispatch failed".to_string()), + ..row(action, from, None) + } +} + +/// Run the checker over a complete read of a run whose governing spec is +/// known: the tests that care about truncation or about an unresolved +/// governing spec build their own [`ConformanceInput`]. +fn check( + automaton: &temper_spec::automaton::Automaton, + kernel_rows: &[TursoTrajectoryRow], + ots_trajectory: Option<&OTSTrajectory>, +) -> ConformanceReport { + check_conformance(ConformanceInput { + automaton, + kernel_rows, + ots_trajectory, + rows_truncated: false, + spec_resolution: SpecResolution::Pinned, + capture_degraded: false, + }) +} + +fn only_violation(report: &ConformanceReport) -> &Violation { + assert_eq!( + report.violations.len(), + 1, + "expected exactly one violation, got {:?}", + report.violations + ); + &report.violations[0] +} + +#[test] +fn a_legal_run_passes_with_no_violations() { + let rows = vec![ + row("AddItem", Some("Draft"), Some("Draft")), + row("SubmitOrder", Some("Draft"), Some("Submitted")), + row("ConfirmOrder", Some("Submitted"), Some("Confirmed")), + row("ProcessOrder", Some("Confirmed"), Some("Processing")), + row("ShipOrder", Some("Processing"), Some("Shipped")), + row("DeliverOrder", Some("Shipped"), Some("Delivered")), + ]; + + let report = check(&order_automaton(), &rows, None); + + assert!(report.passed, "violations: {:?}", report.violations); + assert!(report.violations.is_empty()); + assert_eq!(report.stats.stream_length, 6); + assert_eq!(report.stats.actor_rows, 6); + assert_eq!(report.stats.transitions_unchecked, 0); + assert_eq!( + report.stats.terminal_entities, 0, + "Delivered still has InitiateReturn leaving it, so it is not terminal" + ); + assert!(report.stats.violations_by_kind.is_empty()); +} + +#[test] +fn illegal_transition_reports_the_offending_index() { + let rows = vec![ + row("AddItem", Some("Draft"), Some("Draft")), + // ShipOrder is legal only from Processing. + row("ShipOrder", Some("Draft"), Some("Shipped")), + ]; + + let report = check(&order_automaton(), &rows, None); + + assert!(!report.passed); + let violation = only_violation(&report); + assert_eq!(violation.index, 1); + assert_eq!(violation.kind, ViolationKind::IllegalTransition); + assert_eq!(violation.action, "ShipOrder"); + assert_eq!(violation.entity_type, "Order"); + assert!( + violation.detail.contains("Draft") && violation.detail.contains("Processing"), + "detail must name both the observed and the legal states: {}", + violation.detail + ); + assert_eq!(report.stats.violations_by_kind["illegal_transition"], 1); +} + +#[test] +fn a_state_in_guard_stands_in_for_a_missing_from_list() { + let automaton = parse_automaton(GUARD_ONLY_IOA).expect("guard fixture parses"); + // `Working` is neither terminal nor in Work's guard, so the only thing + // that can flag this row is the transition check reading the guard. + let rows = vec![TursoTrajectoryRow { + entity_type: "Ticket".to_string(), + ..row("Work", Some("Working"), Some("Working")) + }]; + + let report = check(&automaton, &rows, None); + + let violation = only_violation(&report); + assert_eq!(violation.index, 0); + assert_eq!(violation.kind, ViolationKind::IllegalTransition); + assert!( + violation.detail.contains("Open"), + "the guard's values are the legal source set: {}", + violation.detail + ); +} + +#[test] +fn a_guarded_action_keeps_its_source_states_out_of_the_terminal_set() { + let automaton = parse_automaton(GUARD_ONLY_IOA).expect("guard fixture parses"); + // `Open` is listed in no action's `from`; only Work's `state_in` guard + // names it. Reading `from` alone would call it terminal. + let rows = vec![TursoTrajectoryRow { + entity_type: "Ticket".to_string(), + ..row("Work", Some("Open"), Some("Working")) + }]; + + let report = check(&automaton, &rows, None); + + assert!(report.passed, "violations: {:?}", report.violations); + assert_eq!(report.stats.terminal_entities, 0); +} + +#[test] +fn forbidden_action_flags_a_platform_action_the_actor_does_not_declare() { + let rows = vec![ + row("AddItem", Some("Draft"), Some("Draft")), + // `Created` is a name the kernel emits; the Order spec never declares + // it, and this row came through the entity dispatch path. + row("Created", Some("Draft"), Some("Draft")), + ]; + + let report = check(&order_automaton(), &rows, None); + + let violation = only_violation(&report); + assert_eq!(violation.index, 1); + assert_eq!(violation.kind, ViolationKind::ForbiddenAction); + assert_eq!(violation.action, "Created"); + assert!(violation.detail.contains("platform defines")); +} + +#[test] +fn unknown_action_flags_a_name_no_spec_defines() { + let rows = vec![ + row("AddItem", Some("Draft"), Some("Draft")), + row("Frobnicate", Some("Draft"), None), + ]; + + let report = check(&order_automaton(), &rows, None); + + let violation = only_violation(&report); + assert_eq!(violation.index, 1); + assert_eq!(violation.kind, ViolationKind::UnknownAction); + assert_eq!(violation.action, "Frobnicate"); + assert!(violation.detail.contains("no `[[action]]`")); +} + +#[test] +fn post_terminal_flags_every_action_after_the_terminal_transition() { + let rows = vec![ + row("CancelOrder", Some("Draft"), Some("Cancelled")), + row("AddItem", Some("Cancelled"), Some("Cancelled")), + ]; + + let report = check(&order_automaton(), &rows, None); + + let violation = only_violation(&report); + assert_eq!(violation.index, 1); + assert_eq!(violation.kind, ViolationKind::PostTerminal); + assert_eq!(violation.action, "AddItem"); + assert!( + violation.detail.contains("index 0"), + "detail must point back at the terminal transition: {}", + violation.detail + ); + assert_eq!(report.stats.terminal_entities, 1); + assert!( + !report + .stats + .violations_by_kind + .contains_key("illegal_transition"), + "post_terminal suppresses the redundant illegal_transition on the same row" + ); +} + +#[test] +fn post_terminal_is_caught_when_the_session_starts_after_the_terminal_transition() { + // No row in this session drove the entity into Cancelled; the source state + // alone has to be enough. + let rows = vec![row("AddItem", Some("Cancelled"), Some("Cancelled"))]; + + let report = check(&order_automaton(), &rows, None); + + let violation = only_violation(&report); + assert_eq!(violation.index, 0); + assert_eq!(violation.kind, ViolationKind::PostTerminal); + assert!(violation.detail.contains("Cancelled")); +} + +#[test] +fn denied_then_retried_flags_a_blind_retry() { + let rows = vec![ + denied("SubmitOrder", Some("Draft")), + failed("SubmitOrder", Some("Draft")), + ]; + + let report = check(&order_automaton(), &rows, None); + + let violation = only_violation(&report); + assert_eq!(violation.index, 1); + assert_eq!(violation.kind, ViolationKind::DeniedThenRetried); + assert_eq!(violation.action, "SubmitOrder"); + assert!( + violation.detail.contains("index 0"), + "detail must point back at the denial: {}", + violation.detail + ); +} + +#[test] +fn a_retry_after_a_state_change_is_not_a_violation() { + let rows = vec![ + denied("CancelOrder", Some("Draft")), + row("SubmitOrder", Some("Draft"), Some("Submitted")), + failed("CancelOrder", Some("Submitted")), + ]; + + let report = check(&order_automaton(), &rows, None); + + assert!( + report.passed, + "the entity moved between denial and retry: {:?}", + report.violations + ); +} + +#[test] +fn a_retry_that_succeeds_is_not_a_violation() { + let rows = vec![ + denied("SubmitOrder", Some("Draft")), + row("SubmitOrder", Some("Draft"), Some("Submitted")), + ]; + + let report = check(&order_automaton(), &rows, None); + + assert!( + report.passed, + "authorization allowed the retry, so an approval landed: {:?}", + report.violations + ); +} + +#[test] +fn a_denial_on_a_different_entity_does_not_arm_the_retry_check() { + let rows = vec![ + denied("SubmitOrder", Some("Draft")), + TursoTrajectoryRow { + entity_id: "order-2".to_string(), + ..failed("SubmitOrder", Some("Draft")) + }, + ]; + + let report = check(&order_automaton(), &rows, None); + + assert!(report.passed, "violations: {:?}", report.violations); +} + +#[test] +fn three_denials_report_each_blind_retry() { + let rows = vec![ + denied("SubmitOrder", Some("Draft")), + denied("SubmitOrder", Some("Draft")), + denied("SubmitOrder", Some("Draft")), + ]; + + let report = check(&order_automaton(), &rows, None); + + let indices: Vec = report.violations.iter().map(|v| v.index).collect(); + assert_eq!(indices, vec![1, 2]); + assert!( + report + .violations + .iter() + .all(|v| v.kind == ViolationKind::DeniedThenRetried) + ); + assert_eq!(report.stats.violations_by_kind["denied_then_retried"], 2); +} + +#[test] +fn platform_rows_and_other_entities_are_counted_but_not_judged() { + let rows = vec![ + // A platform-sourced bookkeeping row naming an action Order never + // declares: kernel bookkeeping, not an actor action. + TursoTrajectoryRow { + source: Some("Platform".to_string()), + ..row("EntitySetNotFound", None, None) + }, + // Another actor's row; this checker was given only the Order spec. + TursoTrajectoryRow { + entity_type: "Invoice".to_string(), + ..row("Frobnicate", Some("Nowhere"), None) + }, + row("AddItem", Some("Draft"), Some("Draft")), + ]; + + let report = check(&order_automaton(), &rows, None); + + assert!(report.passed, "violations: {:?}", report.violations); + assert_eq!(report.stats.stream_length, 3); + assert_eq!(report.stats.actor_rows, 1); + assert_eq!(report.stats.platform_rows_skipped, 1); + assert_eq!(report.stats.other_entity_rows_skipped, 1); +} + +#[test] +fn a_row_without_a_source_state_is_counted_as_unchecked() { + let rows = vec![row("ShipOrder", None, Some("Shipped"))]; + + let report = check(&order_automaton(), &rows, None); + + assert!(report.violations.is_empty(), "{:?}", report.violations); + assert_eq!(report.stats.transitions_unchecked, 1); + assert_eq!( + report.verdict, + Verdict::Indeterminate, + "a transition that could not be checked is not a transition that passed" + ); + assert!(!report.passed); + assert!( + report + .evidence_gaps + .iter() + .any(|gap| gap.contains("source state")), + "the gap must say what was missing: {:?}", + report.evidence_gaps + ); +} + +fn ots_with_decisions(actions: &[&str]) -> OTSTrajectory { + let now = "2026-01-01T00:00:00Z" + .parse::>() + .expect("timestamp parses"); + let mut turn = OTSTurn::new(1, now); + for action in actions { + turn = turn.with_decision(OTSDecision::new( + DecisionType::ToolSelection, + OTSChoice::new(*action), + OTSConsequence::failure(), + )); + } + OTSTrajectory::new(OTSMetadata::new( + "task", + "agent-1", + OutcomeType::Failure, + now, + )) + .with_turn(turn) +} + +#[test] +fn a_declared_decision_over_an_empty_session_cannot_pass() { + // The fail-open this test exists for: a decision naming an action the spec + // declares raises no violation, and it used to be enough to suppress the + // "nothing was checked" gap. A caller able to upload a trajectory could + // then get `passed: true` for a session the kernel has no row for — a run + // nobody has any evidence ever happened. + let ots = ots_with_decisions(&["AddItem"]); + + let report = check(&order_automaton(), &[], Some(&ots)); + + assert!(report.violations.is_empty(), "AddItem is declared"); + assert_eq!(report.stats.actor_rows, 0); + assert_eq!(report.stats.ots_decisions_checked, 1); + assert_eq!( + report.verdict, + Verdict::Indeterminate, + "a decision proves an action name is in the spec, not that the run followed it" + ); + assert!(!report.passed); + assert!(!report.evidence_complete); +} + +#[test] +fn decisions_never_substitute_for_the_rows_a_run_did_not_leave() { + // The same shape with several declared decisions: quantity of agent-side + // claims must not add up to evidence the kernel never recorded. + let ots = ots_with_decisions(&["AddItem", "SubmitOrder", "ConfirmOrder"]); + + let report = check(&order_automaton(), &[], Some(&ots)); + + assert!(report.violations.is_empty()); + assert_eq!(report.stats.ots_decisions_checked, 3); + assert!(!report.passed); + assert!( + report + .evidence_gaps + .iter() + .any(|gap| gap.contains("no governed action")), + "{:?}", + report.evidence_gaps + ); +} + +#[test] +fn a_run_with_no_actor_rows_still_fails_on_a_disagreeing_decision() { + // The other half of the rule: decisions still *fail* a run. Only the path + // to Pass is closed to them. + let ots = ots_with_decisions(&["Frobnicate"]); + + let report = check(&order_automaton(), &[], Some(&ots)); + + assert_eq!(report.verdict, Verdict::Fail); + assert_eq!(only_violation(&report).kind, ViolationKind::UnknownAction); +} + +#[test] +fn ots_decisions_the_kernel_never_recorded_are_checked_after_the_rows() { + let rows = vec![row("AddItem", Some("Draft"), Some("Draft"))]; + // `AddItem` already has a row and must not be double-counted; `Frobnicate` + // never reached the kernel at all. + let ots = ots_with_decisions(&["AddItem", "Frobnicate"]); + + let report = check(&order_automaton(), &rows, Some(&ots)); + + let violation = only_violation(&report); + assert_eq!( + violation.index, 1, + "OTS decisions are indexed after the kernel rows" + ); + assert_eq!(violation.kind, ViolationKind::UnknownAction); + assert_eq!(violation.action, "Frobnicate"); + assert!(violation.detail.contains("never recorded a row")); + assert_eq!(report.stats.ots_decisions_checked, 1); + assert_eq!(report.stats.stream_length, 2); +} + +#[test] +fn ots_decisions_on_declared_actions_raise_nothing() { + let rows = vec![row("AddItem", Some("Draft"), Some("Draft"))]; + let ots = ots_with_decisions(&["CancelOrder"]); + + let report = check(&order_automaton(), &rows, Some(&ots)); + + assert!(report.passed, "violations: {:?}", report.violations); + assert_eq!(report.stats.ots_decisions_checked, 1); +} + +#[test] +fn an_empty_session_is_indeterminate_rather_than_passing() { + let report = check(&order_automaton(), &[], None); + + assert_eq!( + report.verdict, + Verdict::Indeterminate, + "a session with no rows is no evidence of conformance" + ); + assert!( + !report.passed, + "a consumer gating on `passed` must not accept a run nobody checked" + ); + assert!(report.violations.is_empty()); + assert_eq!(report.stats.stream_length, 0); + assert_eq!(report.stats.actor_rows, 0); + assert!( + report + .evidence_gaps + .iter() + .any(|gap| gap.contains("no governed action")), + "{:?}", + report.evidence_gaps + ); +} + +#[test] +fn a_truncated_read_is_indeterminate_even_with_no_violations() { + let rows = vec![row("AddItem", Some("Draft"), Some("Draft"))]; + + let report = check_conformance(ConformanceInput { + automaton: &order_automaton(), + kernel_rows: &rows, + ots_trajectory: None, + rows_truncated: true, + spec_resolution: SpecResolution::Pinned, + capture_degraded: false, + }); + + assert!(report.violations.is_empty()); + assert_eq!( + report.verdict, + Verdict::Indeterminate, + "the unread tail of the session could hold anything" + ); + assert!(!report.evidence_complete); + assert!(!report.passed); + assert!( + report + .evidence_gaps + .iter() + .any(|gap| gap.contains("row cap")), + "{:?}", + report.evidence_gaps + ); +} + +#[test] +fn a_violation_in_a_truncated_prefix_still_fails() { + let rows = vec![ + row("AddItem", Some("Draft"), Some("Draft")), + row("ShipOrder", Some("Draft"), Some("Shipped")), + ]; + + let report = check_conformance(ConformanceInput { + automaton: &order_automaton(), + kernel_rows: &rows, + ots_trajectory: None, + rows_truncated: true, + spec_resolution: SpecResolution::Pinned, + capture_degraded: false, + }); + + assert_eq!( + report.verdict, + Verdict::Fail, + "a disagreement found in a prefix is still a disagreement" + ); + assert!(!report.passed); +} + +#[test] +fn a_gap_between_two_individually_legal_rows_is_a_violation() { + // Both source states are legal for their own action. Nothing recorded ever + // moved the entity from Submitted to Processing. + let rows = vec![ + row("SubmitOrder", Some("Draft"), Some("Submitted")), + row("ShipOrder", Some("Processing"), Some("Shipped")), + ]; + + let report = check(&order_automaton(), &rows, None); + + let violation = only_violation(&report); + assert_eq!(violation.index, 1); + assert_eq!(violation.kind, ViolationKind::StateDiscontinuity); + assert!( + violation.detail.contains("Submitted") && violation.detail.contains("Processing"), + "detail must name both ends of the gap: {}", + violation.detail + ); +} + +#[test] +fn a_success_landing_somewhere_the_action_does_not_go_is_a_violation() { + // SubmitOrder is legal from Draft and lands in Submitted, never Cancelled. + let rows = vec![row("SubmitOrder", Some("Draft"), Some("Cancelled"))]; + + let report = check(&order_automaton(), &rows, None); + + let violation = only_violation(&report); + assert_eq!(violation.index, 0); + assert_eq!(violation.kind, ViolationKind::UnexpectedTargetState); + assert!( + violation.detail.contains("Cancelled") && violation.detail.contains("Submitted"), + "detail must name both the observed and the declared target: {}", + violation.detail + ); +} + +#[test] +fn an_action_with_no_target_must_leave_the_state_where_it_was() { + // AddItem declares no `to`, so the state holds. + let rows = vec![row("AddItem", Some("Draft"), Some("Submitted"))]; + + let report = check(&order_automaton(), &rows, None); + + let violation = only_violation(&report); + assert_eq!(violation.kind, ViolationKind::UnexpectedTargetState); + assert!( + violation.detail.contains("declares no `to`"), + "{}", + violation.detail + ); +} + +#[test] +fn an_always_enabled_action_empties_the_terminal_set() { + // `Heartbeat` is an input action with neither `from` nor a state_in guard, + // so the kernel enables it from every state — including `Closed`, which no + // other action lists as a source. + let automaton = parse_automaton(ALWAYS_ENABLED_IOA).expect("fixture parses"); + let rows = vec![TursoTrajectoryRow { + entity_type: "Beacon".to_string(), + ..row("Heartbeat", Some("Closed"), Some("Closed")) + }]; + + let report = check(&automaton, &rows, None); + + assert!( + report.passed, + "an always-enabled action is legal from a state nothing else leaves: {:?}", + report.violations + ); + assert_eq!(report.stats.terminal_entities, 0); +} + +#[test] +fn a_caller_supplied_audit_row_cannot_inject_a_violation() { + // POST /api/audit writes rows with a caller-chosen session, entity type, + // and action name, marked spec_governed = false. + let rows = vec![ + row("AddItem", Some("Draft"), Some("Draft")), + TursoTrajectoryRow { + spec_governed: Some(false), + ..row("Frobnicate", Some("Draft"), Some("Draft")) + }, + ]; + + let report = check(&order_automaton(), &rows, None); + + assert!( + report.passed, + "a non-governed audit record is not this actor executing its spec: {:?}", + report.violations + ); + assert_eq!(report.stats.non_governed_rows_skipped, 1); + assert_eq!(report.stats.actor_rows, 1); +} + +#[test] +fn a_row_on_another_entity_does_not_account_for_a_decision_on_this_one() { + // The kernel recorded PayInvoice against Invoice. That says nothing about + // whether the agent's PayInvoice decision against Order ever reached the + // governed path. + let rows = vec![ + row("AddItem", Some("Draft"), Some("Draft")), + TursoTrajectoryRow { + entity_type: "Invoice".to_string(), + ..row("PayInvoice", Some("Due"), Some("Paid")) + }, + ]; + let ots = ots_with_decisions(&["PayInvoice"]); + + let report = check(&order_automaton(), &rows, Some(&ots)); + + let violation = only_violation(&report); + assert_eq!(violation.kind, ViolationKind::UnknownAction); + assert_eq!(violation.action, "PayInvoice"); + assert_eq!(violation.entity_type, "Order"); +} + +#[test] +fn a_thinking_decision_is_not_reported_as_an_action() { + let rows = vec![row("AddItem", Some("Draft"), Some("Draft"))]; + let now = "2026-01-01T00:00:00Z" + .parse::>() + .expect("timestamp parses"); + let ots = OTSTrajectory::new(OTSMetadata::new( + "task", + "agent-1", + OutcomeType::Success, + now, + )) + .with_turn(OTSTurn::new(1, now).with_decision(OTSDecision::new( + DecisionType::ReasoningStep, + OTSChoice::new("compare shipping options"), + OTSConsequence::success(), + ))); + + let report = check(&order_automaton(), &rows, Some(&ots)); + + assert!( + report.passed, + "a reasoning step names a thought, not a callable: {:?}", + report.violations + ); + assert_eq!(report.stats.ots_decisions_checked, 0); + assert_eq!(report.stats.ots_decisions_skipped_as_thinking, 1); +} + +/// The row the capture path writes when it fails to store an entry for a +/// session (`crate::trajectory_outbox::capture_loss_marker`). +fn capture_loss_marker_row() -> TursoTrajectoryRow { + TursoTrajectoryRow { + entity_type: CAPTURE_LOSS_ENTITY_TYPE.to_string(), + entity_id: "session-1".to_string(), + success: false, + to_status: None, + from_status: None, + source: Some("Platform".to_string()), + spec_governed: Some(false), + error: Some("trajectory capture lost at least one entry for this session".to_string()), + ..row(CAPTURE_LOSS_ACTION, None, None) + } +} + +#[test] +fn a_run_missing_captured_rows_cannot_pass() { + // Every row the checker can see agrees with the spec. The marker says the + // ones it cannot see were never stored, so the run is unproven rather than + // conforming. + let rows = vec![ + row("AddItem", Some("Draft"), Some("Draft")), + capture_loss_marker_row(), + row("SubmitOrder", Some("Draft"), Some("Submitted")), + ]; + + let report = check(&order_automaton(), &rows, None); + + assert!(report.violations.is_empty()); + assert_eq!(report.stats.capture_loss_markers, 1); + assert_eq!( + report.stats.actor_rows, 2, + "the marker is not an action and must not be judged as one" + ); + assert_eq!(report.verdict, Verdict::Indeterminate); + assert!( + !report.passed, + "a run whose record is known to have holes in it cannot pass" + ); + assert!(!report.evidence_complete); + assert!( + report + .evidence_gaps + .iter() + .any(|gap| gap.contains("loss marker")), + "{:?}", + report.evidence_gaps + ); +} + +#[test] +fn a_capture_loss_marker_is_not_read_as_another_entity_s_row() { + // The marker carries the capture path's own entity type. Classified by the + // entity comparison alone it would be counted as another actor's row and + // the evidence gap would vanish. + let rows = vec![capture_loss_marker_row()]; + + let report = check(&order_automaton(), &rows, None); + + assert_eq!(report.stats.capture_loss_markers, 1); + assert_eq!(report.stats.other_entity_rows_skipped, 0); +} + +#[test] +fn a_run_whose_governing_spec_is_unresolved_cannot_pass() { + let rows = vec![row("AddItem", Some("Draft"), Some("Draft"))]; + + let report = check_conformance(ConformanceInput { + automaton: &order_automaton(), + kernel_rows: &rows, + ots_trajectory: None, + rows_truncated: false, + spec_resolution: SpecResolution::Unresolved, + capture_degraded: false, + }); + + assert!(report.violations.is_empty()); + assert_eq!(report.spec_resolution, SpecResolution::Unresolved); + assert_eq!( + report.verdict, + Verdict::Indeterminate, + "agreeing with a spec that may not be the one in force proves nothing" + ); + assert!(!report.passed); + assert!(!report.evidence_complete); + assert!( + report + .evidence_gaps + .iter() + .any(|gap| gap.contains("spec version")), + "{:?}", + report.evidence_gaps + ); +} + +#[test] +fn a_resolved_spec_and_a_complete_read_report_complete_evidence() { + let rows = vec![row("AddItem", Some("Draft"), Some("Draft"))]; + + let report = check(&order_automaton(), &rows, None); + + assert_eq!(report.spec_resolution, SpecResolution::Pinned); + assert!(report.evidence_complete); + assert!(report.passed); +} + +/// One MCP `execute` turn: the decision names the submitted code, and the +/// governed actions the code called are recorded inside it +/// (`temper_mcp::runtime::record_execute_turn`). +fn mcp_execute_trajectory(code: &str, nested_actions: &[&str]) -> OTSTrajectory { + let now = "2026-01-01T00:00:00Z" + .parse::>() + .expect("timestamp parses"); + let mut choice = OTSChoice::new(format!("execute: {code}")); + if !nested_actions.is_empty() { + choice = choice.with_arguments(serde_json::json!({ + "trajectory_actions": nested_actions + .iter() + .map(|action| serde_json::json!({ "action": action, "params": {} })) + .collect::>(), + })); + } + OTSTrajectory::new(OTSMetadata::new( + "task", + "agent-1", + OutcomeType::Success, + now, + )) + .with_turn(OTSTurn::new(1, now).with_decision(OTSDecision::new( + DecisionType::ToolSelection, + choice, + OTSConsequence::success(), + ))) +} + +#[test] +fn an_mcp_execute_decision_is_not_reported_as_an_action() { + // Every MCP-produced decision would otherwise surface as one + // `unknown_action` violation naming a hundred characters of Python. + let rows = vec![row("AddItem", Some("Draft"), Some("Draft"))]; + let ots = mcp_execute_trajectory("print('hello')", &[]); + + let report = check(&order_automaton(), &rows, Some(&ots)); + + assert!( + report.passed, + "an execute envelope names the harness's tool, not this actor: {:?}", + report.violations + ); + assert_eq!(report.stats.ots_decisions_checked, 0); + assert_eq!(report.stats.ots_decisions_skipped_as_harness_tool, 1); +} + +#[test] +fn the_governed_actions_inside_an_execute_decision_are_checked() { + // The envelope is not an action; the actions the code called are, and the + // kernel recorded a row for neither. + let rows = vec![row("AddItem", Some("Draft"), Some("Draft"))]; + let ots = mcp_execute_trajectory( + "temper.action('default', 'Order', 'Frobnicate', {})", + &["Frobnicate"], + ); + + let report = check(&order_automaton(), &rows, Some(&ots)); + + let violation = only_violation(&report); + assert_eq!(violation.kind, ViolationKind::UnknownAction); + assert_eq!(violation.action, "Frobnicate"); + assert_eq!(report.stats.ots_decisions_checked, 1); + assert_eq!(report.stats.ots_decisions_skipped_as_harness_tool, 0); +} + +#[test] +fn a_governed_action_inside_an_execute_decision_that_reached_the_kernel_is_not_double_counted() { + let rows = vec![row("AddItem", Some("Draft"), Some("Draft"))]; + let ots = mcp_execute_trajectory( + "temper.action('default', 'Order', 'AddItem', {})", + &["AddItem"], + ); + + let report = check(&order_automaton(), &rows, Some(&ots)); + + assert!(report.passed, "{:?}", report.violations); + assert_eq!(report.stats.ots_decisions_checked, 0); +} + +#[test] +fn an_undeclared_action_that_moved_the_entity_is_reported_once() { + // The undeclared row is the fault. If the walk forgot where that row left + // the entity, the next legal row would look like a discontinuity and the + // same fault would be reported a second time under another kind. + let rows = vec![ + row("AddItem", Some("Draft"), Some("Draft")), + row("Frobnicate", Some("Draft"), Some("Submitted")), + row("ConfirmOrder", Some("Submitted"), Some("Confirmed")), + ]; + + let report = check(&order_automaton(), &rows, None); + + let violation = only_violation(&report); + assert_eq!(violation.index, 1); + assert_eq!(violation.kind, ViolationKind::UnknownAction); +} diff --git a/crates/temper-server/src/conformance/decisions.rs b/crates/temper-server/src/conformance/decisions.rs new file mode 100644 index 000000000..09f425486 --- /dev/null +++ b/crates/temper-server/src/conformance/decisions.rs @@ -0,0 +1,165 @@ +//! What one OTS decision contributes to the action walk. +//! +//! A decision's `choice.action` is not always an action name. Three shapes +//! reach the checker: +//! +//! - a thought — a reasoning step or a response formulation, whose `action` is +//! free-form prose ([`temper_ots::models::DecisionType::is_invocation`]); +//! - a **harness envelope** — an invocation of the harness's own tool rather +//! than of a governed action. The MCP server records every `execute` tool +//! call as one decision whose `choice.action` is `"execute: "` (`temper_mcp::runtime::record_execute_turn`), with the governed +//! actions the code calls listed under `choice.arguments.trajectory_actions`; +//! - an action name, which is what the checker can judge. +//! +//! Reading the first two as action names reports attempts the agent never made: +//! every MCP-produced decision would surface as one `unknown_action` violation +//! naming a hundred characters of Python. + +use temper_ots::models::OTSDecision; + +/// Key under `choice.arguments` where a harness envelope lists the governed +/// actions the submitted code calls. +const TRAJECTORY_ACTIONS_KEY: &str = "trajectory_actions"; + +/// Field naming the action inside one entry of that list. +const NESTED_ACTION_KEY: &str = "action"; + +/// What a decision names. +pub(super) enum DecisionActions<'a> { + /// A thought rather than a callable: nothing was invoked. + Thinking, + /// A harness tool, naming no governed action the checker can judge. + HarnessTool, + /// Governed action names the agent decided on, in the order given. + Actions(Vec<&'a str>), +} + +/// Read the governed actions one decision names. +pub(super) fn decision_actions(decision: &OTSDecision) -> DecisionActions<'_> { + if !decision.decision_type.is_invocation() { + return DecisionActions::Thinking; + } + let action = decision.choice.action.as_str(); + if is_action_name(action) { + return DecisionActions::Actions(vec![action]); + } + // The envelope itself names no governed action, but the actions the code + // inside it calls are recorded alongside it and are exactly what the + // checker is looking for. + let nested = nested_actions(decision); + if nested.is_empty() { + DecisionActions::HarnessTool + } else { + DecisionActions::Actions(nested) + } +} + +/// Governed actions listed under `choice.arguments.trajectory_actions`. +fn nested_actions(decision: &OTSDecision) -> Vec<&str> { + decision + .choice + .arguments + .as_ref() + .and_then(|arguments| arguments.get(TRAJECTORY_ACTIONS_KEY)) + .and_then(|actions| actions.as_array()) + .map(|actions| { + actions + .iter() + .filter_map(|entry| entry.get(NESTED_ACTION_KEY)?.as_str()) + .filter(|action| is_action_name(action)) + .collect() + }) + .unwrap_or_default() +} + +/// Whether `action` has the shape of a declared action name. +/// +/// An `[[action]]` name in an IOA spec is a bare token. A `choice.action` +/// carrying anything else — whitespace, a colon, a newline of Python — is a +/// harness envelope around something that is not an action name, whatever the +/// harness that produced it. +fn is_action_name(action: &str) -> bool { + !action.is_empty() + && action + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '-' | '.')) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + use temper_ots::models::{DecisionType, OTSChoice, OTSConsequence}; + + fn decision(decision_type: DecisionType, choice: OTSChoice) -> OTSDecision { + OTSDecision::new(decision_type, choice, OTSConsequence::success()) + } + + fn actions(decision: &OTSDecision) -> Vec<&str> { + match decision_actions(decision) { + DecisionActions::Actions(actions) => actions, + DecisionActions::Thinking => panic!("expected actions, got a thought"), + DecisionActions::HarnessTool => panic!("expected actions, got a harness tool"), + } + } + + #[test] + fn a_bare_action_name_is_the_action() { + let decision = decision(DecisionType::ToolSelection, OTSChoice::new("ConfirmOrder")); + assert_eq!(actions(&decision), vec!["ConfirmOrder"]); + } + + #[test] + fn a_thought_names_nothing() { + let decision = decision( + DecisionType::ReasoningStep, + OTSChoice::new("compare shipping options"), + ); + assert!(matches!( + decision_actions(&decision), + DecisionActions::Thinking + )); + } + + #[test] + fn an_execute_envelope_yields_the_actions_the_code_called() { + let decision = decision( + DecisionType::ToolSelection, + OTSChoice::new("execute: temper.action('default', 'Order', 'ConfirmOrder', {})") + .with_arguments(json!({ + "trajectory_actions": [ + { "action": "ConfirmOrder", "params": {} }, + { "action": "ShipOrder", "params": {} }, + ], + })), + ); + assert_eq!(actions(&decision), vec!["ConfirmOrder", "ShipOrder"]); + } + + #[test] + fn an_execute_envelope_that_called_nothing_governed_is_a_harness_tool() { + let decision = decision( + DecisionType::ToolSelection, + OTSChoice::new("execute: print('hello')"), + ); + assert!( + matches!(decision_actions(&decision), DecisionActions::HarnessTool), + "a code envelope names no governed action, so it must not be judged as one" + ); + } + + #[test] + fn a_nested_list_of_non_names_is_a_harness_tool() { + let decision = decision( + DecisionType::ToolSelection, + OTSChoice::new("execute: temper.action(...)").with_arguments(json!({ + "trajectory_actions": [{ "action": "not an action name" }], + })), + ); + assert!(matches!( + decision_actions(&decision), + DecisionActions::HarnessTool + )); + } +} diff --git a/crates/temper-server/src/conformance/mod.rs b/crates/temper-server/src/conformance/mod.rs new file mode 100644 index 000000000..27e5b11fd --- /dev/null +++ b/crates/temper-server/src/conformance/mod.rs @@ -0,0 +1,305 @@ +//! Deterministic conformance checking of a recorded run against its spec. +//! +//! A trajectory says what an agent did. The IOA spec says what it was allowed +//! to do. [`check_conformance`] compares the two and reports every place they +//! disagree, with no sampling, no scoring, and no model in the loop: the same +//! inputs always produce the same report. +//! +//! # Inputs +//! +//! - **The automaton** — the actor spec the run executed under, as the kernel +//! already parsed it. The checker reads the declared action set, each +//! action's legal source states, and the terminal states. +//! - **The kernel rows** — one session's trajectory rows, oldest first, in the +//! order the kernel wrote them. `query_trajectories_by_session` returns them +//! in exactly that order; passing them in any other order gives a wrong +//! answer, because the state-machine checks are order-dependent. +//! - **The OTS trajectory** (optional) — the agent-side record of the same +//! run. It contributes the decisions the kernel never recorded a row for: +//! actions the agent attempted that never reached the governed path. +//! - **The spec resolution** — whether the run named the spec version it +//! executed under. See [`SpecResolution`]. +//! +//! # The row stream and violation indices +//! +//! [`Violation::index`] is the position in the ordered stream the checker +//! walked. Positions `0..kernel_rows.len()` are kernel rows and index directly +//! into the slice that was passed in. Positions after that are the actions +//! named by OTS decisions, appended in trajectory order. +//! +//! # What is checked, and what is skipped +//! +//! Only rows that belong to this actor are checked: +//! +//! - Rows whose `entity_type` is not this automaton's entity belong to another +//! actor, whose spec the checker was not given. They are counted in +//! [`ConformanceStats::other_entity_rows_skipped`] and not judged. +//! - Rows whose `source` is `Platform` are kernel bookkeeping — an entity-set +//! miss, a spec submission, a progress marker — not actions the actor took. +//! They are counted in [`ConformanceStats::platform_rows_skipped`]. +//! - Rows marked `spec_governed = false` are caller-supplied audit records +//! rather than governed dispatches. They are counted in +//! [`ConformanceStats::non_governed_rows_skipped`]. +//! - Capture-loss markers ([`CAPTURE_LOSS_ENTITY_TYPE`]) are not actions at +//! all. They are the capture path saying it failed to store a row for this +//! session, and they are counted in +//! [`ConformanceStats::capture_loss_markers`]. +//! +//! An OTS decision is checked only when it names a governed action (see +//! [`decisions`]) and no kernel row **for this actor** carries that action +//! name. A decision the kernel did record is already covered by its row, and +//! checking both would report the same fault twice — but a row belonging to +//! another entity proves nothing about this one, so it does not suppress the +//! decision either. Decisions carry no observed state, so only the action-set +//! checks apply to them. +//! +//! # Verdict, and why `passed` is not just "no violations" +//! +//! A report that saw nothing found no violations, and so did a report that +//! saw the first 5,000 rows of a 5,001-row run. Neither is evidence of +//! conformance. [`ConformanceReport::verdict`] separates the three answers: +//! [`Verdict::Fail`] when the run disagreed with the spec, [`Verdict::Pass`] +//! when a complete run agreed with it, and [`Verdict::Indeterminate`] when the +//! evidence could not settle the question. [`ConformanceReport::evidence_gaps`] +//! names every reason and [`ConformanceReport::evidence_complete`] is the +//! one-field form of the same answer. `passed` is true only for +//! [`Verdict::Pass`], so a consumer that gates on it cannot accept an unchecked +//! run. +//! +//! # Violation kinds +//! +//! - [`ViolationKind::UnknownAction`] — the action name appears in no +//! `[[action]]` of the spec and is not a name the kernel itself emits. No +//! spec defines it anywhere. +//! - [`ViolationKind::ForbiddenAction`] — the action is one the platform +//! defines (see [`KERNEL_PLATFORM_ACTIONS`]) but this actor's spec does not +//! declare, recorded against the actor from the entity dispatch path. The +//! name is defined; it is just not part of this actor's surface. +//! - [`ViolationKind::PostTerminal`] — an action on an entity that had already +//! reached a terminal state. Terminal states are those no action lists in +//! its `from`, so nothing may follow them. Detected both from an earlier row +//! in the stream that drove the entity into a terminal state and from a row +//! whose own `from_status` is terminal, so a session that begins after the +//! terminal transition is still caught. +//! - [`ViolationKind::IllegalTransition`] — the row's `from_status` is not a +//! legal source state for that action. The legal set is the action's `from` +//! list, or, for an action declared without one, the values of its +//! `state_in` guard. An `input` or `Composite` action with neither is always +//! enabled — the I/O-automata property the kernel implements by giving it +//! every state as a source — so nothing about its source state can disagree. +//! - [`ViolationKind::StateDiscontinuity`] — the row's `from_status` is not +//! where the previous row for the same entity left it. Each row's source +//! state can be legal on its own while the sequence skips a state no +//! recorded action reached. +//! - [`ViolationKind::UnexpectedTargetState`] — a successful row landed +//! somewhere other than the action's declared `to` (or, for an action with +//! no `to`, somewhere other than where it started). +//! - [`ViolationKind::DeniedThenRetried`] — the same action was re-attempted +//! on the same entity after an authorization denial with nothing having +//! changed in between. A retry is justified when the entity's state changed +//! between denial and retry (the precondition the denial evaluated no longer +//! holds), or when the retry itself succeeded (authorization allowed it, so +//! an approval landed). A retry that is refused again after neither is the +//! agent hammering a closed door. +//! +//! Rows are judged independently, so one row can raise more than one +//! violation; `post_terminal` suppresses `illegal_transition` on the same row, +//! because a terminal source state is illegal for every action and reporting +//! it twice says nothing new. + +mod decisions; +mod report; +mod spec_view; +mod walk; + +use std::collections::BTreeSet; + +use temper_ots::models::OTSTrajectory; +use temper_spec::automaton::Automaton; +use temper_store_turso::TursoTrajectoryRow; + +use decisions::{DecisionActions, decision_actions}; +use report::{EvidenceContext, evidence_gaps}; +use spec_view::SpecView; +use walk::{RowDisposition, Walk, check_row, row_disposition, undeclared_detail}; + +pub use report::{ + ConformanceReport, ConformanceStats, SpecResolution, Verdict, Violation, ViolationKind, +}; + +/// Action names the kernel itself writes into the trajectory stream. +/// +/// These are defined by the platform rather than by any actor spec: OData +/// write verbs, entity lifecycle markers, and management operations. A row +/// carrying one of these names against an actor that does not declare it is a +/// [`ViolationKind::ForbiddenAction`] — the name is defined, just not by that +/// actor — as opposed to a name no spec defines at all. +pub const KERNEL_PLATFORM_ACTIONS: &[&str] = &[ + "ContextReady", + "Create", + "Created", + "Delete", + "EntitySetNotFound", + "Patch", + "ProgressMade", + "Put", + "StreamUpdated", + "SubmitSpec", + "__Created", + "manage_wasm", + "submit_specs", +]; + +/// Entity type of the row the capture path writes when it loses a trajectory +/// entry for a session. +/// +/// The trajectory outbox is bounded and its writes can fail. Either way an +/// action the kernel captured never reaches storage, and a session read that +/// silently returns the rows that survived would let a run with holes in it +/// pass a conformance check. The capture path writes one marker per session it +/// loses a row for (`crate::trajectory_outbox`); this checker reads the marker +/// as an evidence gap. +/// +/// Not an actor's entity type and not an action: markers are never judged. +pub const CAPTURE_LOSS_ENTITY_TYPE: &str = "TrajectoryCapture"; + +/// Action name on a capture-loss marker row. +pub const CAPTURE_LOSS_ACTION: &str = "CaptureLost"; + +/// One run, and everything the checker needs to judge it. +pub struct ConformanceInput<'a> { + /// The actor spec the run executed under. + pub automaton: &'a Automaton, + /// One session's rows, in the order the kernel wrote them, oldest first. + pub kernel_rows: &'a [TursoTrajectoryRow], + /// The agent-side record of the same run, when one was supplied. + pub ots_trajectory: Option<&'a OTSTrajectory>, + /// Whether the row read stopped at its cap instead of at the end of the + /// session. A checked prefix is not a checked run, so this makes the + /// verdict indeterminate rather than passing. + pub rows_truncated: bool, + /// Whether `automaton` is provably the spec that governed the run. + pub spec_resolution: SpecResolution, + /// Whether the server holding these rows has lost captured rows it could + /// not record against any session. When true, no session read from it can + /// be assumed whole, so the report says so and cannot pass. + pub capture_degraded: bool, +} + +/// Check one recorded run against the spec that governed it. +/// +/// See the module docs for what each violation kind means, which rows are +/// judged, and how the verdict follows from the violations and the evidence. +pub fn check_conformance(input: ConformanceInput<'_>) -> ConformanceReport { + let ConformanceInput { + automaton, + kernel_rows, + ots_trajectory, + rows_truncated, + spec_resolution, + capture_degraded, + } = input; + let spec = SpecView::new(automaton); + let mut walk = Walk::default(); + let mut violations: Vec = Vec::new(); + + for (index, row) in kernel_rows.iter().enumerate() { + check_row(&spec, &mut walk, index, row, &mut violations); + } + + let mut stats = walk.into_stats(kernel_rows.len()); + + if let Some(trajectory) = ots_trajectory { + check_decisions(&spec, trajectory, kernel_rows, &mut stats, &mut violations); + } + + for violation in &violations { + *stats + .violations_by_kind + .entry(violation.kind.as_str().to_string()) + .or_insert(0) += 1; + } + + let gaps = evidence_gaps( + &stats, + &EvidenceContext { + kernel_rows_read: kernel_rows.len(), + rows_truncated, + spec_resolution, + capture_degraded, + }, + ); + ConformanceReport::new(violations, gaps, spec_resolution, stats) +} + +/// Judge the actions the agent decided on that no kernel row accounts for. +/// +/// Decisions are a one-way input. They can raise violations, and so fail a run; +/// they never count toward passing one. A decision is the agent's own account +/// of what it chose — it names an action and carries no observed state, so it +/// cannot show that anything happened, let alone that it happened legally. +/// [`ConformanceStats::ots_decisions_checked`] is a record of what was looked +/// at, not evidence, which is why the "nothing was checked" gap in +/// [`report`] keys on kernel rows alone. +fn check_decisions( + spec: &SpecView<'_>, + trajectory: &OTSTrajectory, + kernel_rows: &[TursoTrajectoryRow], + stats: &mut ConformanceStats, + violations: &mut Vec, +) { + // Only this actor's rows can account for a decision. A `PayInvoice` row on + // `Invoice` says nothing about whether the agent's `PayInvoice` decision + // against `Order` ever reached the governed path, so it must not suppress + // it. + let recorded_actions: BTreeSet<&str> = kernel_rows + .iter() + .filter(|row| row_disposition(spec, row) == RowDisposition::ActorExecution) + .map(|row| row.action.as_str()) + .collect(); + let mut index = kernel_rows.len(); + for decision in trajectory.turns.iter().flat_map(|turn| &turn.decisions) { + let actions = match decision_actions(decision) { + // A reasoning step or a response formulation names a thought, not + // a callable; reporting it as an action invents an attempt the + // agent never made. + DecisionActions::Thinking => { + stats.ots_decisions_skipped_as_thinking += 1; + continue; + } + // The agent invoked its harness, not this actor. The governed + // actions it reached through that call, if any, are recorded + // alongside it and are judged as `Actions` instead. + DecisionActions::HarnessTool => { + stats.ots_decisions_skipped_as_harness_tool += 1; + continue; + } + DecisionActions::Actions(actions) => actions, + }; + for action in actions { + if recorded_actions.contains(action) { + continue; + } + stats.ots_decisions_checked += 1; + if !spec.declared.contains_key(action) { + let kind = spec.classify_undeclared(action); + violations.push(Violation { + index, + kind, + action: action.to_string(), + entity_type: spec.entity_name.to_string(), + detail: format!( + "agent decided on `{action}`, which the kernel never recorded a row for; \ + {}", + undeclared_detail(kind, spec.entity_name) + ), + }); + } + index += 1; + } + } + stats.stream_length = index; +} + +#[cfg(test)] +mod conformance_test; diff --git a/crates/temper-server/src/conformance/report.rs b/crates/temper-server/src/conformance/report.rs new file mode 100644 index 000000000..b65e333f4 --- /dev/null +++ b/crates/temper-server/src/conformance/report.rs @@ -0,0 +1,278 @@ +//! What a conformance check answers with. +//! +//! [`check_conformance`](super::check_conformance) owns the walk; this module +//! owns the shape of the answer and the rule that turns violations and evidence +//! into a verdict. + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; + +/// The kind of disagreement between a recorded run and its spec. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ViolationKind { + /// The action's source state is not legal for that action. + IllegalTransition, + /// The action's source state is not where the previous row left the entity. + StateDiscontinuity, + /// A successful action landed somewhere the spec does not send it. + UnexpectedTargetState, + /// The action is defined by the platform but not by this actor's spec. + ForbiddenAction, + /// The action followed a terminal state. + PostTerminal, + /// The action was retried after a denial with nothing changed in between. + DeniedThenRetried, + /// No spec defines the action name. + UnknownAction, +} + +impl ViolationKind { + /// Stable snake_case name, used as the key in + /// [`ConformanceStats::violations_by_kind`]. + pub fn as_str(self) -> &'static str { + match self { + Self::IllegalTransition => "illegal_transition", + Self::StateDiscontinuity => "state_discontinuity", + Self::UnexpectedTargetState => "unexpected_target_state", + Self::ForbiddenAction => "forbidden_action", + Self::PostTerminal => "post_terminal", + Self::DeniedThenRetried => "denied_then_retried", + Self::UnknownAction => "unknown_action", + } + } +} + +/// What the checker was able to conclude. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Verdict { + /// A complete run, and it agreed with its spec. + Pass, + /// The run disagreed with its spec. + Fail, + /// The evidence could not settle the question. + Indeterminate, +} + +/// Whether the spec the run was judged against is the spec that governed it. +/// +/// A conformance report only means something against the spec the run executed +/// under. Temper keeps one spec per (tenant, entity type), so the registered +/// spec is whatever was submitted last — not necessarily the one in force when +/// the run happened. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SpecResolution { + /// The run named the version it executed under, and it is the registered + /// spec the check ran against. + Pinned, + /// Nothing named the governing version. The check ran against whatever is + /// registered now, which may or may not be what the run executed under, so + /// the report cannot claim conformance. + Unresolved, +} + +/// One place where the run disagreed with the spec. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Violation { + /// Position in the ordered row stream the checker walked. + pub index: usize, + /// What kind of disagreement this is. + pub kind: ViolationKind, + /// The action that raised it. + pub action: String, + /// The entity type the action was taken on. + pub entity_type: String, + /// Why it is a violation, naming the states or indices involved. + pub detail: String, +} + +/// What the checker looked at. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct ConformanceStats { + /// Rows and decisions in the ordered stream. + pub stream_length: usize, + /// Kernel rows attributed to this actor and judged. + pub actor_rows: usize, + /// Kernel rows skipped as kernel bookkeeping (`source = Platform`). + pub platform_rows_skipped: usize, + /// Kernel rows skipped as belonging to another actor's spec. + pub other_entity_rows_skipped: usize, + /// Kernel rows skipped as caller-supplied audit records + /// (`spec_governed = false`) rather than governed dispatches. + pub non_governed_rows_skipped: usize, + /// Capture-loss markers found in the session + /// (see [`CAPTURE_LOSS_ENTITY_TYPE`](super::CAPTURE_LOSS_ENTITY_TYPE)). + /// Each one says the capture path failed to store at least one row for + /// this session, so the record the checker walked has holes in it. + pub capture_loss_markers: usize, + /// Judged rows whose source state could not be checked — `from_status` + /// was absent, or the spec declares no source set to check it against. A + /// high count means the capture path, not the run, is the thing to look + /// at. + pub transitions_unchecked: usize, + /// Judged rows whose target state could not be checked, because the row + /// reported no end state. + pub targets_unchecked: usize, + /// OTS decisions judged because no kernel row for this actor carried their + /// action. + pub ots_decisions_checked: usize, + /// OTS decisions passed over because their type names a thought rather + /// than a callable (see + /// [`temper_ots::models::DecisionType::is_invocation`]). + pub ots_decisions_skipped_as_thinking: usize, + /// OTS decisions passed over because they name the harness's own tool + /// rather than a governed action, and carry no governed action inside + /// them (see [`decisions`](super::decisions)). + pub ots_decisions_skipped_as_harness_tool: usize, + /// Entities observed reaching a terminal state. + pub terminal_entities: usize, + /// Violation count per kind, keyed by [`ViolationKind::as_str`]. + pub violations_by_kind: BTreeMap, +} + +/// The result of checking one run against one spec. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ConformanceReport { + /// What the checker concluded. + pub verdict: Verdict, + /// True only for [`Verdict::Pass`]: a complete run that agreed with its + /// spec. False for a failure and false for evidence that could not settle + /// the question, so a consumer gating on this field cannot accept an + /// unchecked run. + pub passed: bool, + /// Whether the spec the run was judged against is provably the one that + /// governed it. + pub spec_resolution: SpecResolution, + /// True when the checker saw the whole run. False when any of the evidence + /// it needed was missing — the read stopped at its cap, the session was + /// empty, the capture path recorded a loss, or the governing spec could not + /// be resolved. Always the negation of "[`evidence_gaps`] is non-empty", + /// as a field a consumer can gate on without reading prose. + pub evidence_complete: bool, + /// Every disagreement, in stream order. + pub violations: Vec, + /// Every reason the evidence was incomplete, in the order found. Empty + /// when the checker saw the whole run. + /// + /// [`evidence_gaps`]: ConformanceReport::evidence_gaps + pub evidence_gaps: Vec, + /// What the checker looked at to get there. + pub stats: ConformanceStats, +} + +impl ConformanceReport { + /// Assemble the answer from the walk's findings. + /// + /// One place decides the verdict, so `passed` and `evidence_complete` + /// cannot drift from the gaps that produced them. + pub(super) fn new( + violations: Vec, + evidence_gaps: Vec, + spec_resolution: SpecResolution, + stats: ConformanceStats, + ) -> Self { + let evidence_complete = evidence_gaps.is_empty(); + let verdict = if !violations.is_empty() { + // A disagreement found in a prefix is still a disagreement. + Verdict::Fail + } else if evidence_complete { + Verdict::Pass + } else { + Verdict::Indeterminate + }; + Self { + verdict, + passed: verdict == Verdict::Pass, + spec_resolution, + evidence_complete, + violations, + evidence_gaps, + stats, + } + } +} + +/// What the checker knows about its inputs, beyond the rows themselves. +pub(super) struct EvidenceContext { + /// Kernel rows the read returned. + pub kernel_rows_read: usize, + /// Whether the read stopped at its cap rather than the end of the session. + pub rows_truncated: bool, + /// Whether the spec judged is provably the one that governed the run. + pub spec_resolution: SpecResolution, + /// Whether this server has lost captured rows it could not attribute to + /// any session. + pub capture_degraded: bool, +} + +/// Every reason this report is not evidence of a conforming run. +pub(super) fn evidence_gaps(stats: &ConformanceStats, context: &EvidenceContext) -> Vec { + let EvidenceContext { + kernel_rows_read, + rows_truncated, + spec_resolution, + capture_degraded, + } = *context; + let mut gaps = Vec::new(); + if spec_resolution == SpecResolution::Unresolved { + gaps.push( + "neither the request nor the trajectory named the spec version this run executed \ + under, so it was checked against whatever is registered now; Temper keeps one spec \ + per entity type, and a submit replaces it, so that may not be the spec that governed \ + the run" + .to_string(), + ); + } + if rows_truncated { + gaps.push(format!( + "the read stopped at its row cap after {kernel_rows_read} rows, so the rest of the \ + session was never checked" + )); + } + if stats.capture_loss_markers > 0 { + gaps.push(format!( + "the capture path recorded {} loss marker(s) for this session, so at least one row it \ + captured was never stored and the run is missing from the record in unknown places", + stats.capture_loss_markers + )); + } + // A marker says which session lost a row. This says a row was lost and the + // marker for it could not be written either, so no session's record can be + // trusted to be whole — including this one, whether or not it has a marker. + if capture_degraded { + gaps.push( + "this server has lost captured rows it could not record against any session, so any \ + session read from it may be missing rows with nothing to say so" + .to_string(), + ); + } + // Keyed on the kernel's rows alone. An agent-side decision names an action + // and carries no observed state, so it can contradict a spec but can never + // show that a run followed one. Counting decisions here would let a + // trajectory naming a single declared action stand in as the whole + // evidence for a session the kernel recorded nothing about. + if stats.actor_rows == 0 { + gaps.push( + "the kernel recorded no governed action for this actor, so nothing about what the \ + run did was checked; agent-side decisions can disagree with a spec but never \ + establish that a run followed one, because they carry no observed state" + .to_string(), + ); + } + if stats.transitions_unchecked > 0 { + gaps.push(format!( + "{} row(s) carried no source state the spec could be checked against", + stats.transitions_unchecked + )); + } + if stats.targets_unchecked > 0 { + gaps.push(format!( + "{} successful row(s) reported no end state, so where they landed is unknown", + stats.targets_unchecked + )); + } + gaps +} diff --git a/crates/temper-server/src/conformance/spec_view.rs b/crates/temper-server/src/conformance/spec_view.rs new file mode 100644 index 000000000..1d3cc3ed2 --- /dev/null +++ b/crates/temper-server/src/conformance/spec_view.rs @@ -0,0 +1,138 @@ +//! What the checker reads out of the spec. +//! +//! One place decides how a spec's actions are read: which source states an +//! action may fire from, which states end the run, and which actions are +//! emissions rather than transitions. [`walk`](super::walk) asks; it does not +//! interpret the spec itself. + +use std::collections::{BTreeMap, BTreeSet}; + +use temper_spec::automaton::{Action, Automaton, Guard}; + +use super::{KERNEL_PLATFORM_ACTIONS, ViolationKind}; + +/// The source states an action may fire from. +pub(super) enum SourceStates<'a> { + /// Exactly these states. + Declared(BTreeSet<&'a str>), + /// Every state in the automaton. + AnyState, + /// Nothing this checker can evaluate. + Unevaluable, +} + +/// Whether the action never transitions the entity. +/// +/// `kind = "output"` actions are emissions to the environment; the kernel +/// leaves them out of the transition table entirely +/// (`temper_spec::automaton::to_state_machine`), so they neither fire from a +/// state nor land in one. +fn is_emitted_event(action: &Action) -> bool { + action.kind == "output" +} + +/// Legal source states for an action. +/// +/// Mirrors the kernel's own reading of a spec +/// (`temper_spec::automaton::to_state_machine`): a `from` list is the source +/// set; an action written with a `state_in` guard instead still restricts its +/// sources; and an `input` or `Composite` action with neither is always +/// enabled, which is the I/O-automata property the kernel implements by giving +/// it every state as a source. +fn legal_sources(action: &Action) -> SourceStates<'_> { + if !action.from.is_empty() { + return SourceStates::Declared(action.from.iter().map(String::as_str).collect()); + } + // An action written with a `state_in` guard instead of a `from` list still + // restricts its source states; read the guard rather than treating the + // action as unconstrained. + let guarded: BTreeSet<&str> = action + .guard + .iter() + .filter_map(|guard| match guard { + Guard::StateIn { values } => Some(values.iter().map(String::as_str)), + _ => None, + }) + .flatten() + .collect(); + if !guarded.is_empty() { + return SourceStates::Declared(guarded); + } + if is_emitted_event(action) { + return SourceStates::Unevaluable; + } + if action.kind == "input" || action.kind.eq_ignore_ascii_case("composite") { + return SourceStates::AnyState; + } + // An `internal` action with no source at all fires from nowhere in the + // kernel's transition table. That is a spec-authoring fault rather than a + // run fault, so the row is reported as unchecked instead of condemned. + SourceStates::Unevaluable +} + +/// States no action can fire from. +/// +/// This is the kernel's own terminal-state rule — a state no action lists as a +/// source — extended to read `state_in` guards as well as `from` lists, and to +/// honour the always-enabled property of unconstrained input actions. +/// `Automaton::extract_metadata` reads only `from`, so an action written with +/// a guard instead would leave its own source states looking terminal and turn +/// every legal action out of them into a false `post_terminal`. For the same +/// reason a single always-enabled action empties the terminal set: it may fire +/// from every state, so no state ends the run. +fn terminal_states(automaton: &Automaton) -> BTreeSet { + let mut sources: BTreeSet<&str> = BTreeSet::new(); + for action in &automaton.actions { + match legal_sources(action) { + SourceStates::Declared(action_sources) => sources.extend(action_sources), + SourceStates::AnyState => return BTreeSet::new(), + SourceStates::Unevaluable => {} + } + } + automaton + .automaton + .states + .iter() + .filter(|state| !sources.contains(state.as_str())) + .cloned() + .collect() +} + +/// The parts of the automaton the checker consults, resolved once. +pub(super) struct SpecView<'a> { + pub(super) entity_name: &'a str, + pub(super) declared: BTreeMap<&'a str, &'a Action>, + pub(super) terminal_states: BTreeSet, +} + +impl<'a> SpecView<'a> { + pub(super) fn new(automaton: &'a Automaton) -> Self { + Self { + entity_name: automaton.automaton.name.as_str(), + declared: automaton + .actions + .iter() + .map(|action| (action.name.as_str(), action)) + .collect(), + terminal_states: terminal_states(automaton), + } + } + + /// Legal source states for an action. + pub(super) fn legal_sources(&self, action: &'a Action) -> SourceStates<'a> { + legal_sources(action) + } + + pub(super) fn is_emitted_event(&self, action: &Action) -> bool { + is_emitted_event(action) + } + + /// Classify an action name this actor's spec does not declare. + pub(super) fn classify_undeclared(&self, action: &str) -> ViolationKind { + if KERNEL_PLATFORM_ACTIONS.contains(&action) { + ViolationKind::ForbiddenAction + } else { + ViolationKind::UnknownAction + } + } +} diff --git a/crates/temper-server/src/conformance/walk.rs b/crates/temper-server/src/conformance/walk.rs new file mode 100644 index 000000000..b7c296abb --- /dev/null +++ b/crates/temper-server/src/conformance/walk.rs @@ -0,0 +1,395 @@ +//! The ordered walk over one session's rows. +//! +//! [`check_conformance`](super::check_conformance) owns the entry point and +//! the report shape; this module owns the per-row judgements and the state +//! carried between them. + +use std::collections::BTreeMap; + +use temper_spec::automaton::Action; +use temper_store_turso::TursoTrajectoryRow; + +use super::spec_view::{SourceStates, SpecView}; +use super::{CAPTURE_LOSS_ACTION, CAPTURE_LOSS_ENTITY_TYPE}; +use super::{ConformanceStats, Violation, ViolationKind}; + +/// Whether a row records something this actor's spec governs. +/// +/// Four kinds of row reach the checker and only one of them is the actor +/// executing its spec: +/// +/// - a capture-loss marker, which is the capture path reporting that a row for +/// this session never reached storage; +/// - another actor's row, whose spec the checker was not given; +/// - kernel bookkeeping (`source = Platform`); +/// - a row explicitly marked `spec_governed = false`: a report about a run +/// rather than a record of one. The caller-supplied endpoints (`POST +/// /api/audit`, `POST /api/evolution/trajectories/unmet`, the load-inline +/// SubmitSpec note) and the management-plane / pre-flight authorization +/// denials (`POST /api/authorize`, policy auth, WASM and spec management) +/// all write these — their session, entity type, or action can be +/// caller-chosen, so judging them would let any caller inject violations +/// into another session's report. +/// +/// A row with `spec_governed` absent is a governed dispatch: the kernel's +/// entity-dispatch capture sites (including the OData dispatch-guard denials) +/// leave the column unset, and every caller-influenced writer sets it false. +pub(super) fn row_disposition(spec: &SpecView<'_>, row: &TursoTrajectoryRow) -> RowDisposition { + // Checked before the entity comparison: a marker's entity type is the + // capture path's own, so the `OtherEntity` arm would otherwise swallow it + // and the evidence gap it reports would be lost. + if row.entity_type == CAPTURE_LOSS_ENTITY_TYPE && row.action == CAPTURE_LOSS_ACTION { + return RowDisposition::CaptureLoss; + } + if row.entity_type != spec.entity_name { + return RowDisposition::OtherEntity; + } + if row.source.as_deref() == Some("Platform") { + return RowDisposition::PlatformBookkeeping; + } + if row.spec_governed == Some(false) { + return RowDisposition::NotSpecGoverned; + } + RowDisposition::ActorExecution +} + +/// What a row is, for the checker's purposes. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum RowDisposition { + /// This actor executing its spec: judged. + ActorExecution, + /// The capture path reporting a row it failed to store. + CaptureLoss, + /// Another actor's row. + OtherEntity, + /// Kernel bookkeeping. + PlatformBookkeeping, + /// A caller-supplied audit record, not a governed dispatch. + NotSpecGoverned, +} + +pub(super) fn check_row( + spec: &SpecView<'_>, + walk: &mut Walk, + index: usize, + row: &TursoTrajectoryRow, + violations: &mut Vec, +) { + match row_disposition(spec, row) { + RowDisposition::CaptureLoss => { + walk.capture_loss_markers += 1; + return; + } + RowDisposition::OtherEntity => { + walk.other_entity_rows_skipped += 1; + return; + } + RowDisposition::PlatformBookkeeping => { + walk.platform_rows_skipped += 1; + return; + } + RowDisposition::NotSpecGoverned => { + walk.non_governed_rows_skipped += 1; + return; + } + RowDisposition::ActorExecution => {} + } + walk.actor_rows += 1; + + let action_name = row.action.as_str(); + let Some(action) = spec.declared.get(action_name) else { + let kind = spec.classify_undeclared(action_name); + violations.push(Violation { + index, + kind, + action: action_name.to_string(), + entity_type: row.entity_type.clone(), + detail: undeclared_detail(kind, spec.entity_name), + }); + // The row is not judged further — there is no declared action to judge + // it against — but it still moved the entity, and the walk has to know + // where to. Leaving the state behind would make the next legal row + // look like a discontinuity, reporting one fault twice. + record_state_progress(spec, walk, index, row); + return; + }; + + let entity_key = row.entity_id.as_str(); + let post_terminal = match walk.terminal_at.get(entity_key) { + Some(terminal_index) => Some(format!( + "entity reached terminal state at index {terminal_index}; no action may follow" + )), + None => row + .from_status + .as_deref() + .filter(|status| spec.terminal_states.contains(*status)) + .map(|status| { + format!("acted from terminal state `{status}`, which no action lists as a source") + }), + }; + + if let Some(detail) = post_terminal { + violations.push(Violation { + index, + kind: ViolationKind::PostTerminal, + action: action_name.to_string(), + entity_type: row.entity_type.clone(), + detail, + }); + } else { + check_transition(spec, walk, index, row, action, violations); + } + + check_continuity(walk, index, row, violations); + check_target_state(spec, walk, index, row, action, violations); + check_retry(walk, index, row, violations); + record_state_progress(spec, walk, index, row); +} + +fn check_transition( + spec: &SpecView<'_>, + walk: &mut Walk, + index: usize, + row: &TursoTrajectoryRow, + action: &Action, + violations: &mut Vec, +) { + let Some(from_status) = row.from_status.as_deref() else { + walk.transitions_unchecked += 1; + return; + }; + let legal_sources = match spec.legal_sources(action) { + SourceStates::Declared(states) => states, + // Legal from every state, so nothing to disagree with. + SourceStates::AnyState => return, + SourceStates::Unevaluable => { + // The spec declares no source set this checker can evaluate, so + // the run's source state was not checked. Counted rather than + // passed over, because an unchecked transition is missing + // evidence and the report says so. + walk.transitions_unchecked += 1; + return; + } + }; + if legal_sources.contains(from_status) { + return; + } + let legal = legal_sources.iter().copied().collect::>().join(", "); + violations.push(Violation { + index, + kind: ViolationKind::IllegalTransition, + action: row.action.clone(), + entity_type: row.entity_type.clone(), + detail: format!( + "acted from `{from_status}`, but `{}` is legal only from: {legal}", + row.action + ), + }); +} + +/// Check that the row starts where the previous row for the same entity left +/// off. +/// +/// Judging each row's `from_status` on its own accepts an impossible run: +/// `Submit: Draft -> Submitted` followed by `Ship: Processing -> Shipped` has +/// two individually legal source states and a gap between them that no +/// recorded action crossed. The state machine is a walk, so the checker walks +/// it. +fn check_continuity( + walk: &mut Walk, + index: usize, + row: &TursoTrajectoryRow, + violations: &mut Vec, +) { + let (Some(from_status), Some((previous_index, previous_status))) = ( + row.from_status.as_deref(), + walk.last_observed_status.get(row.entity_id.as_str()), + ) else { + return; + }; + if from_status == previous_status { + return; + } + violations.push(Violation { + index, + kind: ViolationKind::StateDiscontinuity, + action: row.action.clone(), + entity_type: row.entity_type.clone(), + detail: format!( + "acted from `{from_status}`, but the entity was last recorded in `{previous_status}` \ + at index {previous_index}; no recorded action crossed the gap" + ), + }); +} + +/// Check that a successful row landed where the spec says the action lands. +/// +/// A stored row claims both ends of the transition. Reading only the source +/// state accepts `Submit: Draft -> Cancelled` against a spec whose `Submit` +/// goes to `Submitted`. +fn check_target_state( + spec: &SpecView<'_>, + walk: &mut Walk, + index: usize, + row: &TursoTrajectoryRow, + action: &Action, + violations: &mut Vec, +) { + if !row.success { + // A failed action changes nothing, so there is no target to check. + return; + } + if spec.is_emitted_event(action) { + // An output action is emitted to the environment; it carries no + // transition, so it has no target state to agree with. + return; + } + let Some(to_status) = row.to_status.as_deref() else { + walk.targets_unchecked += 1; + return; + }; + // An action without a `to` leaves the state where it was; that is the + // target it must land on. + let (expected, source) = match action.to.as_deref() { + Some(expected) => (expected, "the action's `to`"), + None => match row.from_status.as_deref() { + Some(from_status) => ( + from_status, + "the action declares no `to`, so the state holds", + ), + None => { + walk.targets_unchecked += 1; + return; + } + }, + }; + if to_status == expected { + return; + } + violations.push(Violation { + index, + kind: ViolationKind::UnexpectedTargetState, + action: row.action.clone(), + entity_type: row.entity_type.clone(), + detail: format!( + "succeeded into `{to_status}`, but `{}` lands in `{expected}` ({source})", + row.action + ), + }); +} + +fn check_retry( + walk: &mut Walk, + index: usize, + row: &TursoTrajectoryRow, + violations: &mut Vec, +) { + let key = (row.entity_id.clone(), row.action.clone()); + if let Some(&denial_index) = walk.pending_denial.get(&key) { + let state_changed = walk + .last_state_change + .get(row.entity_id.as_str()) + .is_some_and(|change_index| *change_index > denial_index); + if !row.success && !state_changed { + violations.push(Violation { + index, + kind: ViolationKind::DeniedThenRetried, + action: row.action.clone(), + entity_type: row.entity_type.clone(), + detail: format!( + "retried `{}` after the denial at index {denial_index} with no intervening \ + approval or state change", + row.action + ), + }); + } + } + + if row.authz_denied == Some(true) { + walk.pending_denial.insert(key, index); + } else if row.success { + // The action went through, so whatever blocked it no longer does. + walk.pending_denial.remove(&key); + } +} + +fn record_state_progress( + spec: &SpecView<'_>, + walk: &mut Walk, + index: usize, + row: &TursoTrajectoryRow, +) { + // A row that reports where the entity ended up moves the walk's idea of + // where the entity is, whether or not the action succeeded: a failed + // action reports the unchanged state, and that is still the state the next + // row must start from. A denial reports no state at all and leaves the + // last observation standing. + if let Some(to_status) = row.to_status.as_deref() { + walk.last_observed_status + .insert(row.entity_id.clone(), (index, to_status.to_string())); + } + + if !row.success { + return; + } + let Some(to_status) = row.to_status.as_deref() else { + return; + }; + if Some(to_status) != row.from_status.as_deref() { + walk.last_state_change.insert(row.entity_id.clone(), index); + } + if spec.terminal_states.contains(to_status) { + walk.terminal_at + .entry(row.entity_id.clone()) + .or_insert(index); + } +} + +pub(super) fn undeclared_detail(kind: ViolationKind, entity_name: &str) -> String { + match kind { + ViolationKind::ForbiddenAction => format!( + "the platform defines this action, but the `{entity_name}` spec does not declare it" + ), + _ => format!("no `[[action]]` in the `{entity_name}` spec defines this name"), + } +} + +/// State carried across the row stream. +#[derive(Default)] +pub(super) struct Walk { + actor_rows: usize, + platform_rows_skipped: usize, + other_entity_rows_skipped: usize, + non_governed_rows_skipped: usize, + capture_loss_markers: usize, + transitions_unchecked: usize, + targets_unchecked: usize, + /// Index at which each entity first reached a terminal state. + terminal_at: BTreeMap, + /// Index of each entity's most recent successful status change. + last_state_change: BTreeMap, + /// Index and status of each entity's most recent reported end state. + last_observed_status: BTreeMap, + /// Index of the most recent unresolved denial per (entity, action). + pending_denial: BTreeMap<(String, String), usize>, +} + +impl Walk { + pub(super) fn into_stats(self, stream_length: usize) -> ConformanceStats { + ConformanceStats { + stream_length, + actor_rows: self.actor_rows, + platform_rows_skipped: self.platform_rows_skipped, + other_entity_rows_skipped: self.other_entity_rows_skipped, + non_governed_rows_skipped: self.non_governed_rows_skipped, + capture_loss_markers: self.capture_loss_markers, + transitions_unchecked: self.transitions_unchecked, + targets_unchecked: self.targets_unchecked, + ots_decisions_checked: 0, + ots_decisions_skipped_as_thinking: 0, + ots_decisions_skipped_as_harness_tool: 0, + terminal_entities: self.terminal_at.len(), + violations_by_kind: BTreeMap::new(), + } + } +} diff --git a/crates/temper-server/src/entity_actor/actor.rs b/crates/temper-server/src/entity_actor/actor.rs index cbf46c9a7..4c89e6f78 100644 --- a/crates/temper-server/src/entity_actor/actor.rs +++ b/crates/temper-server/src/entity_actor/actor.rs @@ -32,6 +32,7 @@ use temper_runtime::persistence::{ COMPOSITE_EVENT_TYPE, EventMetadata, PersistenceEnvelope, PersistenceError, }; use temper_runtime::scheduler::{sim_now, sim_uuid}; +pub(super) use tokio::time::sleep as sleep_persistence_retry; // determinism-ok: production persistence retry backoff use crate::storage::{BackendLabel, BoxedEventStore}; @@ -45,7 +46,28 @@ use super::types::{ MAX_ITEMS_PER_ENTITY, }; -fn event_budget_workspace_id(state: &EntityState) -> String { +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum ReplayPolicy { + LenientSnapshot, + StrictSnapshot, + StrictFullJournal, +} + +impl ReplayPolicy { + fn loads_snapshot(self) -> bool { + self != Self::StrictFullJournal + } + + fn strict_journal_read(self) -> bool { + self != Self::LenientSnapshot + } + + fn strict_event_validation(self) -> bool { + self == Self::StrictFullJournal + } +} + +pub(super) fn event_budget_workspace_id(state: &EntityState) -> String { if state.entity_type == "Workspace" { return state.entity_id.clone(); } @@ -95,20 +117,20 @@ fn duplicate_idempotency_custom_effects( /// Optionally persists events to the configured backend. Wide events are emitted /// via the OTEL SDK (no-op when OTEL is not initialised). pub struct EntityActor { - tenant: String, - entity_type: String, + pub(super) tenant: String, + pub(super) entity_type: String, entity_id: String, /// Live reference to the transition table. Reads through `RwLock` so that /// hot-swapped tables are visible on the next action dispatch without /// restarting the actor. - table: Arc>, - initial_fields: serde_json::Value, + pub(super) table: Arc>, + pub(super) initial_fields: serde_json::Value, /// Optional event journal for persistence. None = in-memory only. - event_journal: Option, + pub(super) event_journal: Option, /// Optional async snapshot writer. Event appends remain synchronous. - snapshot_queue: Option>, + pub(super) snapshot_queue: Option>, /// Persistence backend label used for metrics and backend-specific field sync. - event_backend: Option, + pub(super) event_backend: Option, /// Trace ID for correlating all events from this actor. trace_id: String, /// Shared idempotency cache (ADR-0048 sub-decision 5). Consulted before @@ -116,23 +138,18 @@ pub struct EntityActor { /// retries that race past the caller's timeout cannot double-execute. idempotency_cache: Option>, /// Object store for field-overflow blob bytes. SQL stores only refs. - blob_store: Option, + pub(super) blob_store: Option, } impl EntityActor { - fn build_initial_state( + pub(super) fn build_initial_state( entity_type: &str, entity_id: &str, table: &TransitionTable, initial_fields: &serde_json::Value, ) -> EntityState { let mut fields = initial_fields.clone(); - if let Some(obj) = fields.as_object_mut() { - obj.entry("Id".to_string()) - .or_insert(serde_json::Value::String(entity_id.to_string())); - obj.entry("Status".to_string()) - .or_insert(serde_json::Value::String(table.initial_state.clone())); - } + super::effects::canonicalize_entity_fields(&mut fields, entity_id, &table.initial_state); EntityState { entity_type: entity_type.to_string(), @@ -210,6 +227,16 @@ impl EntityActor { match serde_json::from_value::(value) { Ok(mut restored) => { + if restored.entity_type != state.entity_type + || restored.entity_id != state.entity_id + { + return false; + } + super::effects::canonicalize_entity_fields( + &mut restored.fields, + &state.entity_id, + &restored.status, + ); restored.sequence_nr = sequence_nr; restored.events_since_snapshot = 0; restored.last_snapshot_sequence_nr = sequence_nr; @@ -308,7 +335,7 @@ impl EntityActor { } /// Persistence ID for this entity: "tenant:EntityType:EntityId". - fn persistence_id(&self) -> String { + pub(super) fn persistence_id(&self) -> String { format!("{}:{}:{}", self.tenant, self.entity_type, self.entity_id) } @@ -326,7 +353,7 @@ impl EntityActor { } /// Persist an event to the configured event store. - async fn persist_event( + pub(super) async fn persist_event( &self, store: &BoxedEventStore, backend: BackendLabel, @@ -405,7 +432,7 @@ impl EntityActor { } (key_rows, vector_rows, reconcile_vectors) }; - let append_start = Instant::now(); + let append_start = Instant::now(); // determinism-ok: production-only event-store wait metric let result = store .append_with_index_rows( persistence_id, @@ -438,7 +465,7 @@ impl EntityActor { } /// Save a snapshot when the configured interval is reached. - async fn maybe_save_snapshot( + pub(super) async fn maybe_save_snapshot( store: &BoxedEventStore, snapshot_queue: Option<&Arc>, persistence_id: &str, @@ -497,19 +524,14 @@ impl EntityActor { /// all state variables (status, counters, booleans). This is option 2 from /// the replay design: the TransitionTable is the authoritative source of /// effects, so replay produces the same state as the original execution. - async fn replay_events( + pub(super) async fn replay_events( table: &TransitionTable, store: &BoxedEventStore, backend: BackendLabel, state: &mut EntityState, tenant: &str, blob_store: Option<&crate::blob_store::BlobStore>, - // When true, a journal read failure PROPAGATES as an error instead of being - // swallowed ("start fresh"). The key-index backfill needs this: it must - // distinguish "entity genuinely has no events" from "could not read the - // journal", or it would watermark a type while a present entity is unkeyed - // (a wrong-absent bug). Actor hydration keeps the lenient default (false). - strict_journal_read: bool, + replay_policy: ReplayPolicy, ) -> Result<(), ActorError> { let replay_start = Instant::now(); // determinism-ok: wall-clock for production replay duration metric only let persistence_id = format!("{tenant}:{}:{}", state.entity_type, state.entity_id); @@ -517,32 +539,34 @@ impl EntityActor { let mut from_sequence = 0; let mut loaded_snapshot = false; - match store.load_snapshot(persistence_id).await { - Ok(Some((snapshot_seq, snapshot_bytes))) => { - if Self::apply_snapshot_bytes(state, snapshot_seq, &snapshot_bytes) { - from_sequence = snapshot_seq; - loaded_snapshot = true; - tracing::info!( - entity = %state.entity_id, - seq = snapshot_seq, - "loaded snapshot before replay" - ); - } else { + if replay_policy.loads_snapshot() { + match store.load_snapshot(persistence_id).await { + Ok(Some((snapshot_seq, snapshot_bytes))) => { + if Self::apply_snapshot_bytes(state, snapshot_seq, &snapshot_bytes) { + from_sequence = snapshot_seq; + loaded_snapshot = true; + tracing::info!( + entity = %state.entity_id, + seq = snapshot_seq, + "loaded snapshot before replay" + ); + } else { + tracing::warn!( + entity = %state.entity_id, + seq = snapshot_seq, + "failed to deserialize snapshot, falling back to full replay" + ); + } + } + Ok(None) => {} + Err(e) => { tracing::warn!( entity = %state.entity_id, - seq = snapshot_seq, - "failed to deserialize snapshot, falling back to full replay" + error = %e, + "failed to load snapshot, falling back to full replay" ); } } - Ok(None) => {} - Err(e) => { - tracing::warn!( - entity = %state.entity_id, - error = %e, - "failed to load snapshot, falling back to full replay" - ); - } } match store.read_events(persistence_id, from_sequence).await { @@ -556,8 +580,41 @@ impl EntityActor { MAX_EVENTS_SINCE_SNAPSHOT ))); } - for env in &envelopes { + let mut expected_sequence = from_sequence.saturating_add(1); + for (index, env) in envelopes.iter().enumerate() { + if replay_policy.strict_event_validation() { + if env.sequence_nr != expected_sequence { + return Err(ActorError::custom(format!( + "non-contiguous journal for {}:{}: expected sequence {}, found {}", + state.entity_type, + state.entity_id, + expected_sequence, + env.sequence_nr + ))); + } + expected_sequence = env.sequence_nr.checked_add(1).ok_or_else(|| { + ActorError::custom(format!( + "journal sequence overflow for {}:{}", + state.entity_type, state.entity_id + )) + })?; + if env.metadata.actor_id != persistence_id { + return Err(ActorError::custom(format!( + "journal event for {}:{} at sequence {} is bound to actor '{}'", + state.entity_type, + state.entity_id, + env.sequence_nr, + env.metadata.actor_id + ))); + } + } + if env.event_type == COMPOSITE_EVENT_TYPE { + if replay_policy.strict_event_validation() { + super::replay_validation::validate_strict_composite_event( + tenant, state, env, + )?; + } state.sequence_nr = env.sequence_nr; continue; } @@ -567,14 +624,33 @@ impl EntityActor { // Tombstone is terminal: once deleted, entity must not replay // into a live state. Stop at the first Deleted event. if env.event_type == "Deleted" { - let tombstone = parsed_event.unwrap_or_else(|_| EntityEvent { - action: "Deleted".to_string(), - from_status: state.status.clone(), - to_status: "Deleted".to_string(), - timestamp: env.metadata.timestamp, - params: serde_json::json!({}), - idempotency_key: None, - }); + let tombstone = match parsed_event { + Ok(mut event) => { + if replay_policy.strict_event_validation() { + super::replay_validation::validate_strict_entity_event( + table, state, env, &event, + )?; + } + event.params = + super::effects::sanitize_action_params(&event.params) + .into_owned(); + event + } + Err(error) if replay_policy.strict_event_validation() => { + return Err(ActorError::custom(format!( + "invalid tombstone event for {}:{} at sequence {}: {error}", + state.entity_type, state.entity_id, env.sequence_nr + ))); + } + Err(_) => EntityEvent { + action: "Deleted".to_string(), + from_status: state.status.clone(), + to_status: "Deleted".to_string(), + timestamp: env.metadata.timestamp, + params: serde_json::json!({}), + idempotency_key: None, + }, + }; state.status = tombstone.to_status.clone(); if let Some(obj) = state.fields.as_object_mut() { obj.insert( @@ -584,11 +660,92 @@ impl EntityActor { } state.push_event_bounded(tombstone); state.sequence_nr = env.sequence_nr; + if replay_policy.strict_event_validation() && index + 1 != envelopes.len() { + return Err(ActorError::custom(format!( + "journal for {}:{} contains events after terminal tombstone at sequence {}", + state.entity_type, state.entity_id, env.sequence_nr + ))); + } break; } + // PATCH/PUT field updates are journaled outside the spec's + // action vocabulary (ARN-189). Re-apply them through the + // same helper the live handler uses so a rehydrated entity + // reaches exactly the live post-update state — including + // PUT's replace semantics, which the generic param-sync + // path below cannot express (it only merges). + if env.event_type == super::effects::FIELDS_UPDATED_EVENT + || env.event_type == super::effects::FIELDS_REPLACED_EVENT + { + match parsed_event { + Ok(event) => { + let applied = super::effects::apply_field_update( + state, + &event.params, + env.event_type == super::effects::FIELDS_REPLACED_EVENT, + ); + if !applied { + // A journaled field update whose payload is not + // an object — only reachable from a build that + // predates the live guard. It is as dropped as + // one that failed to deserialize, so it fails + // or counts the same way. + if replay_policy.strict_event_validation() { + return Err(ActorError::custom(format!( + "non-object field-update event for {}:{} at sequence {}", + state.entity_type, state.entity_id, env.sequence_nr + ))); + } + crate::event_budget_metrics::record_field_update_replay_skip( + tenant, + &state.entity_type, + &state.entity_id, + ); + } + state.push_event_bounded(event); + } + Err(e) => { + // Honor the replay policy, like the tombstone and + // generic arms do. Under a strict policy the caller + // is resolving authoritative state — identity and + // authority decisions read from it — so silently + // dropping a field update there can preserve + // exactly the authority a `FieldsReplaced` was + // meant to revoke. Fail instead of skipping. + if replay_policy.strict_event_validation() { + return Err(ActorError::custom(format!( + "invalid field-update event for {}:{} at sequence {}: {e}", + state.entity_type, state.entity_id, env.sequence_nr + ))); + } + crate::event_budget_metrics::record_field_update_replay_skip( + tenant, + &state.entity_type, + &state.entity_id, + ); + tracing::warn!( + entity = %state.entity_id, + sequence_nr = env.sequence_nr, + event_type = %env.event_type, + error = %e, + "skipping field-update event with incompatible schema during replay" + ); + } + } + state.sequence_nr = env.sequence_nr; + continue; + } + match parsed_event { - Ok(event) => { + Ok(mut event) => { + if replay_policy.strict_event_validation() { + super::replay_validation::validate_strict_entity_event( + table, state, env, &event, + )?; + } + event.params = + super::effects::sanitize_action_params(&event.params).into_owned(); // A persisted event is a historical fact: its guard // already passed at commit time and its `to_status` // is authoritative. Replay therefore re-derives the @@ -657,6 +814,12 @@ impl EntityActor { state.push_event_bounded(event); } Err(e) => { + if replay_policy.strict_event_validation() { + return Err(ActorError::custom(format!( + "invalid event for {}:{} at sequence {}: {e}", + state.entity_type, state.entity_id, env.sequence_nr + ))); + } // Schema-mismatched event: log and skip rather than panic. // This preserves entity hydration across spec evolution — // the last valid state is used and replay continues. @@ -710,7 +873,7 @@ impl EntityActor { } } Err(e) => { - if strict_journal_read { + if replay_policy.strict_journal_read() { return Err(ActorError::custom(format!( "failed to read events for replay of {}:{}: {e}", state.entity_type, state.entity_id @@ -758,7 +921,42 @@ pub(crate) async fn recover_entity_state_from_store( &mut state, tenant, blob_store, - strict_journal_read, + if strict_journal_read { + ReplayPolicy::StrictSnapshot + } else { + ReplayPolicy::LenientSnapshot + }, + ) + .await?; + Ok(state) +} + +/// Rebuild security-sensitive state from the complete durable journal. +/// +/// This intentionally ignores snapshots and fails closed on read errors, +/// sequence gaps, incompatible events, or history after a terminal tombstone. +/// Identity resolution uses this path so a stale or corrupt snapshot cannot +/// preserve revoked authority. +#[allow(clippy::too_many_arguments)] +pub(crate) async fn recover_authoritative_entity_state_from_store( + tenant: &str, + entity_type: &str, + entity_id: &str, + table: &TransitionTable, + store: &BoxedEventStore, + backend: BackendLabel, + initial_fields: &serde_json::Value, + blob_store: Option<&crate::blob_store::BlobStore>, +) -> Result { + let mut state = EntityActor::build_initial_state(entity_type, entity_id, table, initial_fields); + EntityActor::replay_events( + table, + store, + backend, + &mut state, + tenant, + blob_store, + ReplayPolicy::StrictFullJournal, ) .await?; Ok(state) @@ -801,12 +999,14 @@ impl Actor for EntityActor { // Persist a bootstrap Created event for first-time entities so initial // fields are durable and replayable. if self.event_journal.is_some() && state.total_event_count == 0 { + let initial_params = + super::effects::sanitize_action_params(&self.initial_fields).into_owned(); let created = EntityEvent { action: "Created".to_string(), from_status: String::new(), to_status: state.status.clone(), timestamp: sim_now(), - params: self.initial_fields.clone(), + params: initial_params, idempotency_key: None, }; @@ -839,6 +1039,7 @@ impl Actor for EntityActor { params, cross_entity_booleans, idempotency_key, + expected_authorization_precondition, } => { // Capture start time for span duration (DST-safe: sim_now() // returns logical clock in simulation, wall clock in production). @@ -848,6 +1049,30 @@ impl Actor for EntityActor { // outside the DST boundary; using Instant here is safe. let ask_reply_start = Instant::now(); // determinism-ok: observability only + // ARN-189: the field-update event names are reserved. Replay + // dispatches them to `apply_field_update` before the generic + // action path, so a spec action of the same name would be + // hijacked on rehydration — its params would be merged into + // fields and its transition never replayed. Reserving them "by + // convention" is not a guarantee; refuse the collision here, + // where a domain action first enters the actor. + if name == super::effects::FIELDS_UPDATED_EVENT + || name == super::effects::FIELDS_REPLACED_EVENT + { + ctx.reply(EntityResponse { + success: false, + state: state.clone(), + error: Some(format!( + "action name `{name}` is reserved for journaled field updates" + )), + custom_effects: vec![], + scheduled_actions: vec![], + spawn_requests: vec![], + spec_governed: true, + }); + return Ok(()); + } + // Snapshot the current table for this action dispatch. // On the next action, any hot-swapped table will be picked up. let table = self.table.read().expect("table lock poisoned").clone(); @@ -892,6 +1117,24 @@ impl Actor for EntityActor { return Ok(()); } + if let Some(expected) = expected_authorization_precondition + && super::effects::entity_authorization_precondition(state) != expected + { + ctx.reply(EntityResponse { + success: false, + state: state.clone(), + error: Some( + "action authorization became stale; retry against current state" + .to_string(), + ), + custom_effects: vec![], + scheduled_actions: vec![], + spawn_requests: vec![], + spec_governed: true, + }); + return Ok(()); + } + // TigerStyle: Assert preconditions before every transition. // These run in production, not just tests. debug_assert!( @@ -1073,7 +1316,7 @@ impl Actor for EntityActor { self.blob_store.as_ref(), // Actor hydration keeps the lenient "start // fresh on read error" behavior (unchanged). - false, + ReplayPolicy::LenientSnapshot, ) .await?; @@ -1147,10 +1390,10 @@ impl Actor for EntityActor { // Backoff: retry 1 → 10ms, retry 2 → 50ms. let backoff_ms = if retry_idx == 1 { 10 } else { 50 }; - tokio::time::sleep(std::time::Duration::from_millis( + sleep_persistence_retry(std::time::Duration::from_millis( backoff_ms, )) - .await; // determinism-ok: rare retry backoff (ADR-0046) + .await; match self .persist_event( @@ -1419,37 +1662,53 @@ impl Actor for EntityActor { .unwrap_or(serde_json::Value::Null); ctx.reply(value); } - EntityMsg::UpdateFields { fields, replace } => { - if replace { - // PUT: replace all fields (preserve Id and Status) - let id = state.entity_id.clone(); - let status = state.status.clone(); - state.fields = fields; - if let Some(obj) = state.fields.as_object_mut() { - obj.insert("Id".to_string(), serde_json::Value::String(id)); - obj.insert("Status".to_string(), serde_json::Value::String(status)); - } - } else { - // PATCH: merge fields into existing - if let (Some(existing), Some(updates)) = - (state.fields.as_object_mut(), fields.as_object()) - { - for (k, v) in updates { - existing.insert(k.clone(), v.clone()); - } - } - } + EntityMsg::UpdateFields { + fields, + replace, + expected_precondition, + } => { + // The whole durable transaction — validation, budget, journal + // append, conflict recovery — lives in `field_updates`. This arm + // only turns its outcome into a reply. + let outcome = super::field_updates::commit_field_update( + self, + state, + fields, + replace, + expected_precondition, + ) + .await; ctx.reply(EntityResponse { - success: true, + success: outcome.is_ok(), state: state.clone(), - error: None, + error: outcome.err(), custom_effects: vec![], scheduled_actions: vec![], spawn_requests: vec![], spec_governed: true, }); } - EntityMsg::Delete => { + + EntityMsg::Delete { + expected_authorization_precondition, + } => { + if let Some(expected) = expected_authorization_precondition + && super::effects::entity_authorization_precondition(state) != expected + { + ctx.reply(EntityResponse { + success: false, + state: state.clone(), + error: Some( + "delete authorization became stale; retry against current state" + .to_string(), + ), + custom_effects: vec![], + scheduled_actions: vec![], + spawn_requests: vec![], + spec_governed: true, + }); + return Ok(()); + } let deleted = EntityEvent { action: "Deleted".to_string(), from_status: state.status.clone(), @@ -1514,3 +1773,7 @@ impl Actor for EntityActor { #[cfg(test)] #[path = "actor_test.rs"] mod tests; + +#[cfg(test)] +#[path = "authoritative_replay_test.rs"] +mod authoritative_replay_tests; diff --git a/crates/temper-server/src/entity_actor/actor_test.rs b/crates/temper-server/src/entity_actor/actor_test.rs index 0e2f53456..5a9802e2e 100644 --- a/crates/temper-server/src/entity_actor/actor_test.rs +++ b/crates/temper-server/src/entity_actor/actor_test.rs @@ -167,6 +167,310 @@ async fn dst_entity_starts_in_initial_state() { assert!(response.state.events.is_empty()); } +#[tokio::test] +async fn dst_update_fields_preserves_runtime_owned_field_authority() { + let system = ActorSystem::new("dst"); + let actor = EntityActor::new( + "Order", + "order-owned-fields", + order_table(), + serde_json::json!({ + "Id": "forged-initial", + "id": "forged-initial", + "Status": "Delivered", + "status": "Delivered", + "has_spec": false, + "ctx_owner_status": "Privileged", + "Customer": "Alice" + }), + ); + let actor_ref = system.spawn(actor, "order-owned-fields"); + + for replace in [false, true] { + let response: EntityResponse = actor_ref + .ask( + EntityMsg::UpdateFields { + fields: serde_json::json!({ + "Id": "forged-update", + "id": "forged-update", + "Status": "Delivered", + "status": "Delivered", + "has_spec": true, + "HasSpec": true, + "ctx_owner_status": "Privileged", + "Customer": "Bob" + }), + replace, + expected_precondition: None, + }, + Duration::from_secs(1), + ) + .await + .expect("field update response"); + + assert_eq!(response.state.fields["Id"], "order-owned-fields"); + assert_eq!(response.state.fields["id"], "order-owned-fields"); + assert_eq!(response.state.fields["Status"], "Draft"); + assert_eq!(response.state.fields["status"], "Draft"); + assert_eq!(response.state.fields["Customer"], "Bob"); + for reserved in ["has_spec", "HasSpec", "ctx_owner_status"] { + assert!( + response.state.fields.get(reserved).is_none(), + "persisted {reserved} during replace={replace}" + ); + } + } +} + +#[tokio::test] +async fn dst_update_fields_rejects_stale_authorization_without_a_journal() { + let system = ActorSystem::new("dst"); + let actor = EntityActor::new( + "Order", + "order-cas", + order_table(), + serde_json::json!({"Owner": "alice"}), + ); + let actor_ref = system.spawn(actor, "order-cas"); + + let authorized: EntityResponse = actor_ref + .ask(EntityMsg::GetState, Duration::from_secs(1)) + .await + .expect("authorized snapshot"); + let expected = + crate::entity_actor::effects::entity_authorization_precondition(&authorized.state); + assert_eq!(authorized.state.sequence_nr, 0); + + let concurrent: EntityResponse = actor_ref + .ask( + EntityMsg::UpdateFields { + fields: serde_json::json!({"Owner": "mallory"}), + replace: false, + expected_precondition: None, + }, + Duration::from_secs(1), + ) + .await + .expect("concurrent field update"); + assert!(concurrent.success); + assert_eq!(concurrent.state.sequence_nr, 0); + + let stale: EntityResponse = actor_ref + .ask( + EntityMsg::UpdateFields { + fields: serde_json::json!({"Secret": "forged"}), + replace: false, + expected_precondition: Some(expected), + }, + Duration::from_secs(1), + ) + .await + .expect("stale field update response"); + assert!(!stale.success); + assert_eq!(stale.state.fields["Owner"], "mallory"); + assert!(stale.state.fields.get("Secret").is_none()); + + let fresh_precondition = + crate::entity_actor::effects::entity_authorization_precondition(&stale.state); + let fresh: EntityResponse = actor_ref + .ask( + EntityMsg::UpdateFields { + fields: serde_json::json!({"Label": "authorized"}), + replace: false, + expected_precondition: Some(fresh_precondition), + }, + Duration::from_secs(1), + ) + .await + .expect("fresh field update response"); + assert!(fresh.success); + assert_eq!(fresh.state.fields["Label"], "authorized"); + + let before_status_change = + crate::entity_actor::effects::entity_authorization_precondition(&fresh.state); + let cancelled: EntityResponse = actor_ref + .ask( + EntityMsg::Action { + name: "CancelOrder".to_string(), + params: serde_json::json!({"Reason": "test"}), + cross_entity_booleans: BTreeMap::new(), + idempotency_key: None, + expected_authorization_precondition: None, + }, + Duration::from_secs(1), + ) + .await + .expect("status-changing action response"); + assert!(cancelled.success); + assert_eq!(cancelled.state.status, "Cancelled"); + assert_eq!(cancelled.state.sequence_nr, 0); + + let stale_status: EntityResponse = actor_ref + .ask( + EntityMsg::UpdateFields { + fields: serde_json::json!({"Secret": "still-forged"}), + replace: false, + expected_precondition: Some(before_status_change), + }, + Duration::from_secs(1), + ) + .await + .expect("status-stale field update response"); + assert!(!stale_status.success); + assert!(stale_status.state.fields.get("Secret").is_none()); + + let deleted: EntityResponse = actor_ref + .ask( + EntityMsg::Delete { + expected_authorization_precondition: None, + }, + Duration::from_secs(1), + ) + .await + .expect("delete response"); + assert!(deleted.success); + let after_delete: EntityResponse = actor_ref + .ask( + EntityMsg::UpdateFields { + fields: serde_json::json!({"Secret": "post-delete"}), + replace: false, + expected_precondition: None, + }, + Duration::from_secs(1), + ) + .await + .expect("post-delete field update response"); + assert!(!after_delete.success); + assert_eq!(after_delete.state.status, "Deleted"); + assert!(after_delete.state.fields.get("Secret").is_none()); +} + +#[tokio::test] +async fn dst_action_rejects_stale_authorization_but_allows_idempotent_reply() { + let system = ActorSystem::new("dst"); + let actor = EntityActor::new( + "Order", + "order-action-cas", + order_table(), + serde_json::json!({"Owner": "alice"}), + ); + let actor_ref = system.spawn(actor, "order-action-cas"); + let authorized: EntityResponse = actor_ref + .ask(EntityMsg::GetState, Duration::from_secs(1)) + .await + .expect("authorized action snapshot"); + let stale_precondition = + crate::entity_actor::effects::entity_authorization_precondition(&authorized.state); + + let concurrent: EntityResponse = actor_ref + .ask( + EntityMsg::UpdateFields { + fields: serde_json::json!({"Owner": "mallory"}), + replace: false, + expected_precondition: None, + }, + Duration::from_secs(1), + ) + .await + .expect("concurrent owner update"); + let stale_delete: EntityResponse = actor_ref + .ask( + EntityMsg::Delete { + expected_authorization_precondition: Some(stale_precondition.clone()), + }, + Duration::from_secs(1), + ) + .await + .expect("stale delete response"); + assert!(!stale_delete.success); + assert_eq!(stale_delete.state.status, "Draft"); + + let stale: EntityResponse = actor_ref + .ask( + EntityMsg::Action { + name: "CancelOrder".to_string(), + params: serde_json::json!({"Reason": "stale"}), + cross_entity_booleans: BTreeMap::new(), + idempotency_key: Some("cancel-request".to_string()), + expected_authorization_precondition: Some(stale_precondition), + }, + Duration::from_secs(1), + ) + .await + .expect("stale action response"); + assert!(!stale.success); + assert_eq!(stale.state.status, "Draft"); + + let fresh_precondition = + crate::entity_actor::effects::entity_authorization_precondition(&concurrent.state); + let applied: EntityResponse = actor_ref + .ask( + EntityMsg::Action { + name: "CancelOrder".to_string(), + params: serde_json::json!({"Reason": "fresh"}), + cross_entity_booleans: BTreeMap::new(), + idempotency_key: Some("cancel-request".to_string()), + expected_authorization_precondition: Some(fresh_precondition.clone()), + }, + Duration::from_secs(1), + ) + .await + .expect("fresh action response"); + assert!(applied.success); + assert_eq!(applied.state.status, "Cancelled"); + + let retry: EntityResponse = actor_ref + .ask( + EntityMsg::Action { + name: "CancelOrder".to_string(), + params: serde_json::json!({"Reason": "fresh"}), + cross_entity_booleans: BTreeMap::new(), + idempotency_key: Some("cancel-request".to_string()), + expected_authorization_precondition: Some(fresh_precondition), + }, + Duration::from_secs(1), + ) + .await + .expect("idempotent action retry response"); + assert!(retry.success); + assert_eq!(retry.state.status, "Cancelled"); + assert_eq!( + retry.state.total_event_count, + applied.state.total_event_count + ); +} + +#[test] +fn snapshot_restore_canonicalizes_runtime_owned_fields() { + let mut state = EntityActor::build_initial_state( + "Order", + "order-snapshot", + &order_table().read().expect("table lock"), + &serde_json::json!({}), + ); + let mut snapshot = state.clone(); + snapshot.fields = serde_json::json!({ + "Id": "forged", + "id": "forged", + "Status": "Delivered", + "status": "Delivered", + "has_spec": false, + "ctx_owner_status": "Privileged", + "Customer": "Alice" + }); + let bytes = serde_json::to_vec(&snapshot).expect("snapshot serialization"); + + assert!(EntityActor::apply_snapshot_bytes(&mut state, 7, &bytes)); + assert_eq!(state.fields["Id"], "order-snapshot"); + assert_eq!(state.fields["id"], "order-snapshot"); + assert_eq!(state.fields["Status"], "Draft"); + assert_eq!(state.fields["status"], "Draft"); + assert_eq!(state.fields["Customer"], "Alice"); + for reserved in ["has_spec", "ctx_owner_status"] { + assert!(state.fields.get(reserved).is_none(), "restored {reserved}"); + } +} + #[tokio::test] async fn dst_add_item_then_submit() { let system = ActorSystem::new("dst"); @@ -182,6 +486,7 @@ async fn dst_add_item_then_submit() { params: serde_json::json!({"ProductId": "prod-1"}), cross_entity_booleans: std::collections::BTreeMap::new(), idempotency_key: None, + expected_authorization_precondition: None, }, Duration::from_secs(1), ) @@ -199,6 +504,7 @@ async fn dst_add_item_then_submit() { params: serde_json::json!({"ShippingAddressId": "addr-1"}), cross_entity_booleans: std::collections::BTreeMap::new(), idempotency_key: None, + expected_authorization_precondition: None, }, Duration::from_secs(1), ) @@ -232,6 +538,7 @@ async fn duplicate_composite_idempotency_reemits_spec_trigger() { params: params.clone(), cross_entity_booleans: std::collections::BTreeMap::new(), idempotency_key: Some("same-pack".into()), + expected_authorization_precondition: None, }, Duration::from_secs(1), ) @@ -246,6 +553,7 @@ async fn duplicate_composite_idempotency_reemits_spec_trigger() { params, cross_entity_booleans: std::collections::BTreeMap::new(), idempotency_key: Some("same-pack".into()), + expected_authorization_precondition: None, }, Duration::from_secs(1), ) @@ -278,6 +586,7 @@ async fn dst_cannot_submit_without_items() { params: serde_json::json!({}), cross_entity_booleans: std::collections::BTreeMap::new(), idempotency_key: None, + expected_authorization_precondition: None, }, Duration::from_secs(1), ) @@ -321,6 +630,7 @@ async fn dst_full_order_lifecycle() { params, cross_entity_booleans: std::collections::BTreeMap::new(), idempotency_key: None, + expected_authorization_precondition: None, }, Duration::from_secs(1), ) @@ -356,6 +666,7 @@ async fn dst_cancel_from_draft() { params: serde_json::json!({"Reason": "changed mind"}), cross_entity_booleans: std::collections::BTreeMap::new(), idempotency_key: None, + expected_authorization_precondition: None, }, Duration::from_secs(1), ) @@ -387,6 +698,7 @@ async fn dst_cannot_cancel_shipped_order() { params: serde_json::json!({}), cross_entity_booleans: std::collections::BTreeMap::new(), idempotency_key: None, + expected_authorization_precondition: None, }, Duration::from_secs(1), ) @@ -402,6 +714,7 @@ async fn dst_cannot_cancel_shipped_order() { params: serde_json::json!({}), cross_entity_booleans: std::collections::BTreeMap::new(), idempotency_key: None, + expected_authorization_precondition: None, }, Duration::from_secs(1), ) @@ -434,6 +747,7 @@ async fn dst_multiple_actors_independent() { params: serde_json::json!({}), cross_entity_booleans: std::collections::BTreeMap::new(), idempotency_key: None, + expected_authorization_precondition: None, }, Duration::from_secs(1), ) @@ -448,6 +762,7 @@ async fn dst_multiple_actors_independent() { params: serde_json::json!({}), cross_entity_booleans: std::collections::BTreeMap::new(), idempotency_key: None, + expected_authorization_precondition: None, }, Duration::from_secs(1), ) @@ -550,125 +865,1750 @@ async fn replay_skips_schema_mismatched_events() { assert_eq!(response.state.total_event_count, 1); } -/// A committed cross-entity-guarded transition must survive replay. -/// -/// Regression: `File.StreamUpdated` carries a `cross_entity_state` guard on the -/// owning Workspace. The guard's boolean is pre-resolved at dispatch time and -/// injected into the eval context, but replay rebuilds the context *without* -/// the related entity in scope — so re-evaluating the guard during replay sees -/// the cross-entity precondition as unsatisfied. Replay must NOT re-gate a -/// durably-stored event: it must honor the stored `to_status` and re-apply the -/// transition's effects, or a File that committed `Created -> Ready` would -/// silently rehydrate back to `Created` (losing `has_content` and the version -/// bump). This proves the stored history wins over a replay-time guard miss. +/// ARN-189: PATCH-style field updates must be journaled — a merge applied via +/// `EntityMsg::UpdateFields` has to survive actor eviction/restart, or any +/// OData PATCH is silently lost the moment the actor rehydrates from the +/// event store. +#[cfg(feature = "sim")] #[tokio::test] -async fn replay_honors_committed_cross_entity_guarded_transition() { - use temper_runtime::persistence::EventStore; +async fn patched_fields_survive_actor_restart() { use temper_store_sim::SimEventStore; - // A minimal File-shaped automaton whose advancing action is gated on a - // cross-entity Workspace status that replay cannot reconstruct. - let file_table = Arc::new(RwLock::new(TransitionTable::from_ioa_source( - r#" -[automaton] -name = "File" -states = ["Created", "Ready"] -initial = "Created" - -[[state]] -name = "version_count" -type = "counter" -initial = "0" - -[[state]] -name = "has_content" -type = "bool" -initial = "false" - -[[action]] -name = "Create" -kind = "input" -from = ["Created"] -to = "Created" -params = ["workspace_id"] - -[[action]] -name = "StreamUpdated" -kind = "input" -from = ["Created", "Ready"] -to = "Ready" -params = ["size_bytes"] -guard = [ - { type = "cross_entity_state", entity_type = "Workspace", entity_id_source = "workspace_id", forbidden_status = ["Frozen", "Archived"] }, -] -effect = [ - { type = "increment", var = "version_count" }, - { type = "set_bool", var = "has_content", value = "true" }, -] -"#, - ))); - let store = Arc::new(SimEventStore::no_faults(7)); - let pid = "default:File:fl-replay-1"; - - let event = |action: &str, from: &str, to: &str| PersistenceEnvelope { - sequence_nr: 0, - event_type: action.to_string(), - payload: serde_json::json!({ - "action": action, - "from_status": from, - "to_status": to, - "timestamp": "2024-01-01T00:00:00Z", - "params": {} - }), - metadata: EventMetadata { - event_id: sim_uuid(), - causation_id: sim_uuid(), - correlation_id: sim_uuid(), - timestamp: sim_now(), - actor_id: pid.to_string(), - }, - }; + let entity_id = "arn189-patch-1"; + let pid = format!("default:Order:{entity_id}"); - // The committed history: a File that was created, then advanced to Ready by - // a guarded StreamUpdated. No Workspace entity is in scope at replay time. - store - .append(pid, 0, &[event("Created", "", "Created")]) - .await - .unwrap(); - store - .append(pid, 1, &[event("Create", "Created", "Created")]) + // Generation 1: live actor accepts the PATCH merge. + let system = ActorSystem::new("sim-arn189-patch-a"); + let actor = EntityActor::with_persistence( + "Order", + entity_id, + order_table(), + serde_json::json!({}), + crate::storage::BoxedEventStore::from_arc(store.clone()), + crate::storage::BackendLabel::Sim, + ); + let actor_ref = system.spawn(actor, entity_id); + let response: EntityResponse = actor_ref + .ask( + EntityMsg::UpdateFields { + fields: serde_json::json!({"Title": "durable title", "Priority": 3}), + replace: false, + expected_precondition: None, + }, + Duration::from_secs(5), + ) .await .unwrap(); - store - .append(pid, 2, &[event("StreamUpdated", "Created", "Ready")]) + assert!(response.success); + assert_eq!( + response.state.fields.get("Title").and_then(|v| v.as_str()), + Some("durable title"), + "live merge must apply" + ); + + // Generation 2: fresh actor over the same store — replay is the only input. + let system2 = ActorSystem::new("sim-arn189-patch-b"); + let actor2 = EntityActor::with_persistence( + "Order", + entity_id, + order_table(), + serde_json::json!({}), + crate::storage::BoxedEventStore::from_arc(store.clone()), + crate::storage::BackendLabel::Sim, + ); + let actor_ref2 = system2.spawn(actor2, entity_id); + let rehydrated: EntityResponse = actor_ref2 + .ask(EntityMsg::GetState, Duration::from_secs(5)) .await .unwrap(); + assert_eq!( + rehydrated + .state + .fields + .get("Title") + .and_then(|v| v.as_str()), + Some("durable title"), + "PATCHed field must survive restart via journal replay (pid {pid})" + ); + assert_eq!( + rehydrated + .state + .fields + .get("Priority") + .and_then(|v| v.as_i64()), + Some(3), + "all merged fields must survive restart" + ); +} - let system = ActorSystem::new("sim-replay-cross-entity"); +/// ARN-189: PUT-style replacement must also be journaled with REPLACE +/// semantics — after restart the replaced field set must match the live +/// result, including the absence of keys the replacement dropped. +#[cfg(feature = "sim")] +#[tokio::test] +async fn replaced_fields_survive_actor_restart_with_replace_semantics() { + use temper_store_sim::SimEventStore; + + let store = Arc::new(SimEventStore::no_faults(11)); + let entity_id = "arn189-put-1"; + + let system = ActorSystem::new("sim-arn189-put-a"); let actor = EntityActor::with_persistence( - "File", - "fl-replay-1", - file_table, + "Order", + entity_id, + order_table(), serde_json::json!({}), - crate::storage::BoxedEventStore::from_arc(store), + crate::storage::BoxedEventStore::from_arc(store.clone()), crate::storage::BackendLabel::Sim, ); - let actor_ref = system.spawn(actor, "fl-replay-1"); + let actor_ref = system.spawn(actor, entity_id); - let response: EntityResponse = actor_ref - .ask(EntityMsg::GetState, Duration::from_secs(5)) + // First a merge that introduces a key the later replacement drops. + let merged: EntityResponse = actor_ref + .ask( + EntityMsg::UpdateFields { + fields: serde_json::json!({"Title": "before", "Legacy": "drop me"}), + replace: false, + expected_precondition: None, + }, + Duration::from_secs(5), + ) .await .unwrap(); + assert!(merged.success); - assert!(response.success); - // The committed StreamUpdated transition survives replay despite the guard - // being unsatisfiable without the Workspace in scope. - assert_eq!( - response.state.status, "Ready", - "a committed cross-entity-guarded transition must not be dropped on replay" + // PUT: full replacement (Id/Status are preserved by the live path). + let replaced: EntityResponse = actor_ref + .ask( + EntityMsg::UpdateFields { + fields: serde_json::json!({"Title": "after"}), + replace: true, + expected_precondition: None, + }, + Duration::from_secs(5), + ) + .await + .unwrap(); + assert!(replaced.success); + assert!( + replaced.state.fields.get("Legacy").is_none(), + "live replacement must drop absent keys" + ); + + let system2 = ActorSystem::new("sim-arn189-put-b"); + let actor2 = EntityActor::with_persistence( + "Order", + entity_id, + order_table(), + serde_json::json!({}), + crate::storage::BoxedEventStore::from_arc(store.clone()), + crate::storage::BackendLabel::Sim, + ); + let actor_ref2 = system2.spawn(actor2, entity_id); + let rehydrated: EntityResponse = actor_ref2 + .ask(EntityMsg::GetState, Duration::from_secs(5)) + .await + .unwrap(); + assert_eq!( + rehydrated + .state + .fields + .get("Title") + .and_then(|v| v.as_str()), + Some("after"), + "replaced field value must survive restart" + ); + assert!( + rehydrated.state.fields.get("Legacy").is_none(), + "replacement semantics must survive restart — dropped keys must not resurrect" + ); + assert_eq!( + rehydrated.state.fields.get("Id").and_then(|v| v.as_str()), + Some(entity_id), + "Id must be preserved through replace + replay" + ); +} + +/// Delegating event store whose appends fail once `armed` is set (so a test +/// can start an actor normally and then fail exactly the append under test) +/// and whose snapshot saves fail once `fail_snapshots` is set (so a test can +/// model a stalled snapshot path while appends keep succeeding). +#[cfg(feature = "sim")] +struct AppendFuseStore { + inner: temper_store_sim::SimEventStore, + armed: std::sync::atomic::AtomicBool, + fail_snapshots: std::sync::atomic::AtomicBool, +} + +#[cfg(feature = "sim")] +impl AppendFuseStore { + fn no_faults(seed: u64) -> Self { + Self { + inner: temper_store_sim::SimEventStore::no_faults(seed), + armed: std::sync::atomic::AtomicBool::new(false), + fail_snapshots: std::sync::atomic::AtomicBool::new(false), + } + } + + fn fuse_err(&self) -> Option { + if self.armed.load(std::sync::atomic::Ordering::SeqCst) { + Some(temper_runtime::persistence::PersistenceError::Storage( + "injected append failure (fuse armed)".to_string(), + )) + } else { + None + } + } +} + +#[cfg(feature = "sim")] +impl temper_runtime::persistence::EventStore for AppendFuseStore { + async fn append( + &self, + persistence_id: &str, + expected_sequence: u64, + events: &[PersistenceEnvelope], + ) -> Result { + if let Some(e) = self.fuse_err() { + return Err(e); + } + self.inner + .append(persistence_id, expected_sequence, events) + .await + } + + async fn append_with_index_rows( + &self, + persistence_id: &str, + expected_sequence: u64, + events: &[PersistenceEnvelope], + key_rows: &[temper_runtime::persistence::EntityKeyRow], + vector_rows: &[temper_runtime::persistence::EntityVectorRow], + reconcile_vectors: bool, + ) -> Result { + if let Some(e) = self.fuse_err() { + return Err(e); + } + self.inner + .append_with_index_rows( + persistence_id, + expected_sequence, + events, + key_rows, + vector_rows, + reconcile_vectors, + ) + .await + } + + async fn append_batch( + &self, + appends: &[temper_runtime::persistence::PersistenceAppend], + ) -> Result< + Vec, + temper_runtime::persistence::PersistenceError, + > { + if let Some(e) = self.fuse_err() { + return Err(e); + } + self.inner.append_batch(appends).await + } + + async fn read_events( + &self, + persistence_id: &str, + from_sequence: u64, + ) -> Result, temper_runtime::persistence::PersistenceError> { + self.inner.read_events(persistence_id, from_sequence).await + } + + async fn save_snapshot( + &self, + persistence_id: &str, + sequence_nr: u64, + snapshot: &[u8], + ) -> Result<(), temper_runtime::persistence::PersistenceError> { + if self + .fail_snapshots + .load(std::sync::atomic::Ordering::SeqCst) + { + return Err(temper_runtime::persistence::PersistenceError::Storage( + "injected snapshot failure (stalled snapshot path)".to_string(), + )); + } + self.inner + .save_snapshot(persistence_id, sequence_nr, snapshot) + .await + } + + async fn load_snapshot( + &self, + persistence_id: &str, + ) -> Result)>, temper_runtime::persistence::PersistenceError> { + self.inner.load_snapshot(persistence_id).await + } + + async fn list_entity_ids( + &self, + tenant: &str, + ) -> Result, temper_runtime::persistence::PersistenceError> { + self.inner.list_entity_ids(tenant).await + } + + async fn list_entity_ids_by_type( + &self, + tenant: &str, + entity_type: &str, + ) -> Result, temper_runtime::persistence::PersistenceError> { + self.inner + .list_entity_ids_by_type(tenant, entity_type) + .await + } +} + +/// ARN-189 fail-closed: when the journal append fails, a field update must +/// NOT report success — otherwise the caller believes a write is durable +/// while restart will lose it. +#[cfg(feature = "sim")] +#[tokio::test] +async fn field_update_with_failed_journal_append_does_not_report_success() { + let store = Arc::new(AppendFuseStore::no_faults(13)); + let entity_id = "arn189-fail-1"; + + let system = ActorSystem::new("sim-arn189-fail"); + let actor = EntityActor::with_persistence( + "Order", + entity_id, + order_table(), + serde_json::json!({}), + crate::storage::BoxedEventStore::from_arc(store.clone()), + crate::storage::BackendLabel::Sim, + ); + let actor_ref = system.spawn(actor, entity_id); + + // Let startup (bootstrap Created append) succeed, then arm the fuse so the + // field-update append is the one that fails. + let started: EntityResponse = actor_ref + .ask(EntityMsg::GetState, Duration::from_secs(5)) + .await + .unwrap(); + assert!(started.success); + store.armed.store(true, std::sync::atomic::Ordering::SeqCst); + + let response: EntityResponse = actor_ref + .ask( + EntityMsg::UpdateFields { + fields: serde_json::json!({"Title": "never durable"}), + replace: false, + expected_precondition: None, + }, + Duration::from_secs(5), + ) + .await + .unwrap(); + assert!( + !response.success, + "a field update whose journal append failed must not claim success" + ); + assert!( + response.state.fields.get("Title").is_none(), + "failed update must not leave half-applied in-memory state" + ); + + // The actor's RETAINED state must also be unmutated — not just the + // error reply's snapshot. + let retained: EntityResponse = actor_ref + .ask(EntityMsg::GetState, Duration::from_secs(5)) + .await + .unwrap(); + assert!( + retained.state.fields.get("Title").is_none(), + "failed update must not persist in the actor's retained state" + ); +} + +/// ARN-189: field updates consume the same event budget as spec actions. +/// With the snapshot path stalled (all snapshot failures are soft), sustained +/// PATCH traffic must be REJECTED at MAX_EVENTS_SINCE_SNAPSHOT instead of +/// growing a replay tail that makes the entity permanently unhydratable. +#[cfg(feature = "sim")] +#[tokio::test] +async fn field_updates_reject_when_event_budget_exhausted() { + let store = Arc::new(AppendFuseStore::no_faults(17)); + // Snapshots fail from the start: models a stalled snapshot path while + // appends keep succeeding. + store + .fail_snapshots + .store(true, std::sync::atomic::Ordering::SeqCst); + let entity_id = "arn189-budget-1"; + + let system = ActorSystem::new("sim-arn189-budget"); + let actor = EntityActor::with_persistence( + "Order", + entity_id, + order_table(), + serde_json::json!({}), + crate::storage::BoxedEventStore::from_arc(store.clone()), + crate::storage::BackendLabel::Sim, + ); + let actor_ref = system.spawn(actor, entity_id); + + let mut rejected_at = None; + for i in 0..=MAX_EVENTS_SINCE_SNAPSHOT { + let response: EntityResponse = actor_ref + .ask( + EntityMsg::UpdateFields { + fields: serde_json::json!({"counter": i}), + replace: false, + expected_precondition: None, + }, + Duration::from_secs(5), + ) + .await + .unwrap(); + if !response.success { + assert!( + response + .error + .as_deref() + .unwrap_or_default() + .contains("Event budget exhausted"), + "rejection must be the event budget, got: {:?}", + response.error + ); + assert!( + response.state.events_since_snapshot >= MAX_EVENTS_SINCE_SNAPSHOT, + "rejection must fire only at the budget boundary" + ); + rejected_at = Some(i); + break; + } + } + assert!( + rejected_at.is_some(), + "sustained field updates with a stalled snapshot path must hit the event budget, \ + not grow the replay tail unboundedly" + ); +} + +/// A committed cross-entity-guarded transition must survive replay. +/// +/// Regression: `File.StreamUpdated` carries a `cross_entity_state` guard on the +/// owning Workspace. The guard's boolean is pre-resolved at dispatch time and +/// injected into the eval context, but replay rebuilds the context *without* +/// the related entity in scope — so re-evaluating the guard during replay sees +/// the cross-entity precondition as unsatisfied. Replay must NOT re-gate a +/// durably-stored event: it must honor the stored `to_status` and re-apply the +/// transition's effects, or a File that committed `Created -> Ready` would +/// silently rehydrate back to `Created` (losing `has_content` and the version +/// bump). This proves the stored history wins over a replay-time guard miss. +#[tokio::test] +async fn replay_honors_committed_cross_entity_guarded_transition() { + use temper_runtime::persistence::EventStore; + use temper_store_sim::SimEventStore; + + // A minimal File-shaped automaton whose advancing action is gated on a + // cross-entity Workspace status that replay cannot reconstruct. + let file_table = Arc::new(RwLock::new(TransitionTable::from_ioa_source( + r#" +[automaton] +name = "File" +states = ["Created", "Ready"] +initial = "Created" + +[[state]] +name = "version_count" +type = "counter" +initial = "0" + +[[state]] +name = "has_content" +type = "bool" +initial = "false" + +[[action]] +name = "Create" +kind = "input" +from = ["Created"] +to = "Created" +params = ["workspace_id"] + +[[action]] +name = "StreamUpdated" +kind = "input" +from = ["Created", "Ready"] +to = "Ready" +params = ["size_bytes"] +guard = [ + { type = "cross_entity_state", entity_type = "Workspace", entity_id_source = "workspace_id", forbidden_status = ["Frozen", "Archived"] }, +] +effect = [ + { type = "increment", var = "version_count" }, + { type = "set_bool", var = "has_content", value = "true" }, +] +"#, + ))); + + let store = Arc::new(SimEventStore::no_faults(7)); + let pid = "default:File:fl-replay-1"; + + let event = |action: &str, from: &str, to: &str| PersistenceEnvelope { + sequence_nr: 0, + event_type: action.to_string(), + payload: serde_json::json!({ + "action": action, + "from_status": from, + "to_status": to, + "timestamp": "2024-01-01T00:00:00Z", + "params": {} + }), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp: sim_now(), + actor_id: pid.to_string(), + }, + }; + + // The committed history: a File that was created, then advanced to Ready by + // a guarded StreamUpdated. No Workspace entity is in scope at replay time. + store + .append(pid, 0, &[event("Created", "", "Created")]) + .await + .unwrap(); + store + .append(pid, 1, &[event("Create", "Created", "Created")]) + .await + .unwrap(); + store + .append(pid, 2, &[event("StreamUpdated", "Created", "Ready")]) + .await + .unwrap(); + + let system = ActorSystem::new("sim-replay-cross-entity"); + let actor = EntityActor::with_persistence( + "File", + "fl-replay-1", + file_table, + serde_json::json!({}), + crate::storage::BoxedEventStore::from_arc(store), + crate::storage::BackendLabel::Sim, + ); + let actor_ref = system.spawn(actor, "fl-replay-1"); + + let response: EntityResponse = actor_ref + .ask(EntityMsg::GetState, Duration::from_secs(5)) + .await + .unwrap(); + + assert!(response.success); + // The committed StreamUpdated transition survives replay despite the guard + // being unsatisfiable without the Workspace in scope. + assert_eq!( + response.state.status, "Ready", + "a committed cross-entity-guarded transition must not be dropped on replay" ); // Its effects were re-applied: content flag set, version bumped. assert_eq!(response.state.booleans.get("has_content"), Some(&true)); assert_eq!(response.state.counters.get("version_count"), Some(&1)); } + +/// ARN-189. `parse_json_body_or_400` accepts any valid JSON, so a `PUT` body of +/// `[1,2,3]` reaches the actor. Once field updates are journaled, letting a +/// non-object through would make the damage permanent: `apply_field_update` +/// cannot restore `Id`/`Status` into a non-object, and `persist_event` would +/// co-commit zero key and zero vector rows, purging the entity's index. Before +/// journaling, that corruption was in-memory and healed on restart. +#[cfg(feature = "sim")] +#[tokio::test] +async fn field_update_rejects_non_object_payload_before_journaling() { + let store = Arc::new(AppendFuseStore::no_faults(23)); + let entity_id = "arn189-non-object"; + + let system = ActorSystem::new("sim-arn189-non-object"); + let actor = EntityActor::with_persistence( + "Order", + entity_id, + order_table(), + serde_json::json!({"Customer": "Alice"}), + crate::storage::BoxedEventStore::from_arc(store.clone()), + crate::storage::BackendLabel::Sim, + ); + let actor_ref = system.spawn(actor, entity_id); + + let baseline: EntityResponse = actor_ref + .ask(EntityMsg::GetState, Duration::from_secs(5)) + .await + .expect("baseline state"); + let baseline_events = baseline.state.total_event_count; + let baseline_sequence = baseline.state.sequence_nr; + + for body in [ + serde_json::json!([1, 2, 3]), + serde_json::json!("a string"), + serde_json::json!(null), + serde_json::json!(7), + ] { + for replace in [true, false] { + let response: EntityResponse = actor_ref + .ask( + EntityMsg::UpdateFields { + fields: body.clone(), + replace, + expected_precondition: None, + }, + Duration::from_secs(5), + ) + .await + .expect("field update response"); + assert!( + !response.success, + "non-object body {body:?} (replace={replace}) must be rejected" + ); + assert!( + response + .error + .as_deref() + .unwrap_or_default() + .contains("must be a JSON object"), + "unexpected error for {body:?}: {:?}", + response.error + ); + // The entity is untouched — not emptied, not stripped of identity, + // and nothing reached the journal or the index rows it co-commits. + assert_eq!(response.state.fields["Customer"], "Alice"); + assert_eq!(response.state.fields["Id"], entity_id); + assert_eq!( + response.state.total_event_count, baseline_events, + "a refused payload must not be journaled" + ); + assert_eq!( + response.state.sequence_nr, baseline_sequence, + "a refused payload must not advance the durable sequence" + ); + } + } +} + +/// ARN-189. Runtime-owned fields must survive a field update, and — because +/// `apply_field_update` is shared with journal replay — must survive rehydration +/// identically. A sanitize step applied only on the live path would silently +/// rewrite the entity the next time it replayed. +#[cfg(feature = "sim")] +#[tokio::test] +async fn field_update_sanitization_is_reproduced_by_replay() { + let store = Arc::new(AppendFuseStore::no_faults(29)); + let entity_id = "arn189-sanitize-replay"; + + let system = ActorSystem::new("sim-arn189-sanitize"); + let actor = EntityActor::with_persistence( + "Order", + entity_id, + order_table(), + serde_json::json!({"Customer": "Alice"}), + crate::storage::BoxedEventStore::from_arc(store.clone()), + crate::storage::BackendLabel::Sim, + ); + let actor_ref = system.spawn(actor, entity_id); + + let response: EntityResponse = actor_ref + .ask( + EntityMsg::UpdateFields { + fields: serde_json::json!({ + "Customer": "Bob", + "Id": "forged-id", + "Status": "Delivered", + "has_spec": true, + "ctx_owner_status": "Privileged", + }), + replace: false, + expected_precondition: None, + }, + Duration::from_secs(5), + ) + .await + .expect("field update response"); + assert!( + response.success, + "update should succeed: {:?}", + response.error + ); + + let live_fields = response.state.fields.clone(); + assert_eq!(live_fields["Id"], entity_id, "forged Id must not stick"); + assert_eq!(live_fields["Customer"], "Bob"); + for reserved in ["has_spec", "ctx_owner_status"] { + assert!( + live_fields.get(reserved).is_none(), + "runtime-owned `{reserved}` must be stripped, got {live_fields:?}" + ); + } + + // The journal itself must not carry the forged runtime-owned keys. Sanitizing + // only on the way into state would leave them in the event, where a future + // reader (an audit, a projection, a replay under different code) still sees a + // second claimed truth for identity and lifecycle. + use temper_runtime::persistence::EventStore as _; + let envelopes = store + .inner + .read_events(&format!("default:Order:{entity_id}"), 0) + .await + .expect("read journal"); + assert!(!envelopes.is_empty(), "the update must have been journaled"); + let mut saw_field_event = false; + for envelope in &envelopes { + let Ok(event) = serde_json::from_value::(envelope.payload.clone()) else { + continue; + }; + if event.action != crate::entity_actor::effects::FIELDS_UPDATED_EVENT + && event.action != crate::entity_actor::effects::FIELDS_REPLACED_EVENT + { + continue; + } + saw_field_event = true; + for reserved in ["has_spec", "ctx_owner_status"] { + assert!( + event.params.get(reserved).is_none(), + "journaled `{}` still carries runtime-owned `{reserved}`: {:?}", + event.action, + event.params + ); + } + } + assert!(saw_field_event, "expected a journaled field-update event"); + + // Rehydrate from the journal: replay must land on exactly the same fields. + let replayed = EntityActor::with_persistence( + "Order", + entity_id, + order_table(), + serde_json::json!({"Customer": "Alice"}), + crate::storage::BoxedEventStore::from_arc(store.clone()), + crate::storage::BackendLabel::Sim, + ); + let replayed_ref = system.spawn(replayed, entity_id); + let after: EntityResponse = replayed_ref + .ask(EntityMsg::GetState, Duration::from_secs(5)) + .await + .expect("state after rehydration"); + assert_eq!( + after.state.fields, live_fields, + "replay must reproduce the live result exactly" + ); +} + +/// ARN-189. Replay dispatches the field-update event names to +/// `apply_field_update` before the generic action path, so a spec action with +/// the same name would be hijacked on rehydration: its params merged into +/// fields, its transition never replayed. The ADR reserved the names by +/// convention; this makes the reservation real. +#[cfg(feature = "sim")] +#[tokio::test] +async fn reserved_field_update_event_names_are_refused_as_actions() { + let store = Arc::new(AppendFuseStore::no_faults(31)); + let entity_id = "arn189-reserved-name"; + + let system = ActorSystem::new("sim-arn189-reserved"); + let actor = EntityActor::with_persistence( + "Order", + entity_id, + order_table(), + serde_json::json!({"Customer": "Alice"}), + crate::storage::BoxedEventStore::from_arc(store.clone()), + crate::storage::BackendLabel::Sim, + ); + let actor_ref = system.spawn(actor, entity_id); + + let baseline: EntityResponse = actor_ref + .ask(EntityMsg::GetState, Duration::from_secs(5)) + .await + .expect("baseline state"); + let baseline_events = baseline.state.total_event_count; + + for reserved in [ + crate::entity_actor::effects::FIELDS_UPDATED_EVENT, + crate::entity_actor::effects::FIELDS_REPLACED_EVENT, + ] { + let response: EntityResponse = actor_ref + .ask( + EntityMsg::Action { + name: reserved.to_string(), + params: serde_json::json!({"Customer": "Mallory"}), + cross_entity_booleans: BTreeMap::new(), + idempotency_key: None, + expected_authorization_precondition: None, + }, + Duration::from_secs(5), + ) + .await + .expect("action response"); + assert!( + !response.success, + "`{reserved}` must not be dispatchable as a domain action" + ); + assert!( + response + .error + .as_deref() + .unwrap_or_default() + .contains("reserved"), + "unexpected error for `{reserved}`: {:?}", + response.error + ); + assert_eq!( + response.state.fields["Customer"], "Alice", + "a refused action must not have merged its params into fields" + ); + assert_eq!( + response.state.total_event_count, baseline_events, + "a refused action must not journal anything" + ); + } +} + +/// ARN-189. A sequence conflict on the field-update append used to wedge the +/// arm: every later update failed with an error until the actor happened to +/// rehydrate with the authoritative sequence. The Action arm right above has had +/// ADR-0046 replay-and-retry for exactly this; a field update needs the same, +/// minus the guard re-evaluation it has no guards for. +#[cfg(feature = "sim")] +#[tokio::test] +async fn field_update_recovers_from_a_concurrency_violation() { + let store = Arc::new(AppendFuseStore::no_faults(37)); + let entity_id = "arn189-concurrency"; + let persistence_id = format!("default:Order:{entity_id}"); + + let system = ActorSystem::new("sim-arn189-concurrency"); + let actor = EntityActor::with_persistence( + "Order", + entity_id, + order_table(), + serde_json::json!({"Customer": "Alice"}), + crate::storage::BoxedEventStore::from_arc(store.clone()), + crate::storage::BackendLabel::Sim, + ); + let actor_ref = system.spawn(actor, entity_id); + + // Warm up first: spawning/creating the entity itself appends, and an + // injection armed before that would be spent on the creation append instead + // of the update under test — leaving the test green for the wrong reason. + let warmup: EntityResponse = actor_ref + .ask( + EntityMsg::UpdateFields { + fields: serde_json::json!({"Customer": "Warmup"}), + replace: false, + expected_precondition: None, + }, + Duration::from_secs(5), + ) + .await + .expect("warmup response"); + assert!(warmup.success, "warmup update should succeed"); + + // One deterministic conflict on the next append: within the retry budget. + store + .inner + .inject_concurrency_violations(&persistence_id, 1); + + let response: EntityResponse = actor_ref + .ask( + EntityMsg::UpdateFields { + fields: serde_json::json!({"Customer": "Bob"}), + replace: false, + expected_precondition: None, + }, + Duration::from_secs(5), + ) + .await + .expect("field update response"); + + assert!( + response.success, + "a single conflict must be recovered, not surfaced: {:?}", + response.error + ); + assert_eq!(response.state.fields["Customer"], "Bob"); + + // And the arm is not wedged: the next update still succeeds. + let next: EntityResponse = actor_ref + .ask( + EntityMsg::UpdateFields { + fields: serde_json::json!({"Customer": "Carol"}), + replace: false, + expected_precondition: None, + }, + Duration::from_secs(5), + ) + .await + .expect("second field update response"); + assert!( + next.success, + "the arm must not be wedged after a recovered conflict: {:?}", + next.error + ); + assert_eq!(next.state.fields["Customer"], "Carol"); + + // The update is durable: rehydrate and read it back. + let replayed = EntityActor::with_persistence( + "Order", + entity_id, + order_table(), + serde_json::json!({"Customer": "Alice"}), + crate::storage::BoxedEventStore::from_arc(store.clone()), + crate::storage::BackendLabel::Sim, + ); + let replayed_ref = system.spawn(replayed, entity_id); + let after: EntityResponse = replayed_ref + .ask(EntityMsg::GetState, Duration::from_secs(5)) + .await + .expect("state after rehydration"); + assert_eq!(after.state.fields["Customer"], "Carol"); +} + +/// Beyond the retry budget the arm still fails closed rather than reporting a +/// success it did not persist. +#[cfg(feature = "sim")] +#[tokio::test] +async fn field_update_fails_closed_when_conflicts_exceed_the_retry_budget() { + let store = Arc::new(AppendFuseStore::no_faults(41)); + let entity_id = "arn189-concurrency-exhausted"; + let persistence_id = format!("default:Order:{entity_id}"); + + let system = ActorSystem::new("sim-arn189-exhausted"); + let actor = EntityActor::with_persistence( + "Order", + entity_id, + order_table(), + serde_json::json!({"Customer": "Alice"}), + crate::storage::BoxedEventStore::from_arc(store.clone()), + crate::storage::BackendLabel::Sim, + ); + let actor_ref = system.spawn(actor, entity_id); + + // Warm up first: spawning/creating the entity itself appends, and an + // injection armed before that would be spent on the creation append instead + // of the update under test — leaving the test green for the wrong reason. + let warmup: EntityResponse = actor_ref + .ask( + EntityMsg::UpdateFields { + fields: serde_json::json!({"Customer": "Warmup"}), + replace: false, + expected_precondition: None, + }, + Duration::from_secs(5), + ) + .await + .expect("warmup response"); + assert!(warmup.success, "warmup update should succeed"); + + let before: EntityResponse = actor_ref + .ask(EntityMsg::GetState, Duration::from_secs(5)) + .await + .expect("state before"); + + // 3 conflicts: 1 initial attempt + 2 retries all fail. + store + .inner + .inject_concurrency_violations(&persistence_id, 3); + + let response: EntityResponse = actor_ref + .ask( + EntityMsg::UpdateFields { + fields: serde_json::json!({"Customer": "Bob"}), + replace: false, + expected_precondition: None, + }, + Duration::from_secs(5), + ) + .await + .expect("field update response"); + + assert!( + !response.success, + "an unpersisted update must never be reported as success" + ); + assert_eq!( + response.state.fields["Customer"], "Warmup", + "the speculative merge must be rolled back to the last durable value" + ); + assert_eq!( + response.state.sequence_nr, before.state.sequence_nr, + "a failed append must not advance the durable sequence" + ); + assert_eq!( + response.state.total_event_count, before.state.total_event_count, + "a failed append must not add an event" + ); +} + +/// ARN-189. `apply_field_update` is the one place PATCH and PUT semantics are +/// decided, and it runs on both the live path and journal replay. Pinned +/// directly, without an actor system, so a change to merge/replace or to the +/// runtime-owned field handling fails here first and unambiguously. +#[test] +fn apply_field_update_merge_replace_and_runtime_owned_fields() { + fn state_with(fields: serde_json::Value) -> EntityState { + EntityState { + entity_type: "Order".to_string(), + entity_id: "order-1".to_string(), + status: "Draft".to_string(), + item_count: 0, + counters: BTreeMap::new(), + booleans: BTreeMap::new(), + lists: BTreeMap::new(), + fields, + events: std::collections::VecDeque::new(), + total_event_count: 0, + events_since_snapshot: 0, + last_snapshot_sequence_nr: 0, + sequence_nr: 0, + processed_idempotency_keys: BTreeMap::new(), + } + } + + // PATCH merges and leaves untouched keys alone. + let mut state = state_with(serde_json::json!({"Customer": "Alice", "Region": "eu"})); + assert!(crate::entity_actor::effects::apply_field_update( + &mut state, + &serde_json::json!({"Customer": "Bob"}), + false, + )); + assert_eq!(state.fields["Customer"], "Bob"); + assert_eq!( + state.fields["Region"], "eu", + "PATCH must not drop other keys" + ); + + // PUT replaces wholesale — the unmentioned key is gone. + let mut state = state_with(serde_json::json!({"Customer": "Alice", "Region": "eu"})); + assert!(crate::entity_actor::effects::apply_field_update( + &mut state, + &serde_json::json!({"Customer": "Bob"}), + true, + )); + assert_eq!(state.fields["Customer"], "Bob"); + assert!( + state.fields.get("Region").is_none(), + "PUT must replace, not merge: {:?}", + state.fields + ); + + // Identity and lifecycle stay authoritative under both, and a caller cannot + // forge them or the runtime-owned fields. + for replace in [true, false] { + let mut state = state_with(serde_json::json!({"Customer": "Alice"})); + assert!(crate::entity_actor::effects::apply_field_update( + &mut state, + &serde_json::json!({ + "Id": "forged", + "Status": "Delivered", + "has_spec": true, + "ctx_owner_status": "Privileged", + }), + replace, + )); + assert_eq!(state.fields["Id"], "order-1", "replace={replace}"); + assert_eq!(state.fields["Status"], "Draft", "replace={replace}"); + for reserved in ["has_spec", "ctx_owner_status"] { + assert!( + state.fields.get(reserved).is_none(), + "runtime-owned `{reserved}` must be stripped (replace={replace})" + ); + } + } + + // Idempotent: replay applies the same event again and must not drift. + let mut state = state_with(serde_json::json!({"Customer": "Alice"})); + let update = serde_json::json!({"Customer": "Bob"}); + assert!(crate::entity_actor::effects::apply_field_update( + &mut state, &update, false + )); + let once = state.fields.clone(); + assert!(crate::entity_actor::effects::apply_field_update( + &mut state, &update, false + )); + assert_eq!(state.fields, once, "re-applying an event must not drift"); +} + +/// ARN-189 / F1. The retry catches up by rebuilding from a fresh initial state, +/// not by replaying onto the live state. Replaying additively re-applies every +/// event on top of its own effects: the events deque grows, `total_event_count` +/// and `events_since_snapshot` climb, and non-idempotent effects fire twice — and +/// the result is returned to the caller, projected, and possibly snapshotted. +#[cfg(feature = "sim")] +#[tokio::test] +async fn field_update_retry_does_not_double_apply_the_journal() { + let store = Arc::new(AppendFuseStore::no_faults(43)); + let entity_id = "arn189-additive-replay"; + let persistence_id = format!("default:Order:{entity_id}"); + + let system = ActorSystem::new("sim-arn189-additive"); + let actor = EntityActor::with_persistence( + "Order", + entity_id, + order_table(), + serde_json::json!({"Customer": "Alice"}), + crate::storage::BoxedEventStore::from_arc(store.clone()), + crate::storage::BackendLabel::Sim, + ); + let actor_ref = system.spawn(actor, entity_id); + + // Build a little history so a replay has something to double-apply. + for customer in ["First", "Second", "Third"] { + let response: EntityResponse = actor_ref + .ask( + EntityMsg::UpdateFields { + fields: serde_json::json!({"Customer": customer}), + replace: false, + expected_precondition: None, + }, + Duration::from_secs(5), + ) + .await + .expect("seed update"); + assert!(response.success); + } + let before: EntityResponse = actor_ref + .ask(EntityMsg::GetState, Duration::from_secs(5)) + .await + .expect("state before conflict"); + let events_before = before.state.total_event_count; + let since_snapshot_before = before.state.events_since_snapshot; + + // One conflict: the retry replays. + store + .inner + .inject_concurrency_violations(&persistence_id, 1); + let recovered: EntityResponse = actor_ref + .ask( + EntityMsg::UpdateFields { + fields: serde_json::json!({"Customer": "Fourth"}), + replace: false, + expected_precondition: None, + }, + Duration::from_secs(5), + ) + .await + .expect("recovered update"); + assert!( + recovered.success, + "conflict should be recovered: {:?}", + recovered.error + ); + + // Exactly one event was added, not the whole journal over again. + assert_eq!( + recovered.state.total_event_count, + events_before + 1, + "a recovered retry must add one event, not re-apply the journal \ + (before={events_before}, after={})", + recovered.state.total_event_count + ); + assert_eq!( + recovered.state.events_since_snapshot, + since_snapshot_before + 1, + "replaying onto live state would inflate the snapshot tail and can wedge \ + the entity on the event budget" + ); + assert_eq!(recovered.state.fields["Customer"], "Fourth"); + + // And the rehydrated state agrees with what the caller was told. + let replayed = EntityActor::with_persistence( + "Order", + entity_id, + order_table(), + serde_json::json!({"Customer": "Alice"}), + crate::storage::BoxedEventStore::from_arc(store.clone()), + crate::storage::BackendLabel::Sim, + ); + let replayed_ref = system.spawn(replayed, entity_id); + let after: EntityResponse = replayed_ref + .ask(EntityMsg::GetState, Duration::from_secs(5)) + .await + .expect("state after rehydration"); + assert_eq!( + after.state.total_event_count, + recovered.state.total_event_count + ); + assert_eq!(after.state.fields["Customer"], "Fourth"); +} + +/// ARN-189 / F2. A preconditioned update is a compare-and-set. A conflict proves +/// the journal moved under it, so replaying and committing would apply the write +/// to state the caller never saw and Cedar never evaluated. It must be refused, +/// not retried — `entity_ops` caps preconditioned asks at one attempt for exactly +/// this reason. +#[cfg(feature = "sim")] +#[tokio::test] +async fn preconditioned_field_update_refuses_rather_than_retrying_a_conflict() { + let store = Arc::new(AppendFuseStore::no_faults(47)); + let entity_id = "arn189-cas-no-retry"; + let boxed = || crate::storage::BoxedEventStore::from_arc(store.clone()); + + let system = ActorSystem::new("sim-arn189-cas"); + let holder = system.spawn( + EntityActor::with_persistence( + "Order", + entity_id, + order_table(), + serde_json::json!({"Customer": "Alice"}), + boxed(), + crate::storage::BackendLabel::Sim, + ), + entity_id, + ); + let seeded: EntityResponse = holder + .ask( + EntityMsg::UpdateFields { + fields: serde_json::json!({"Customer": "Seed"}), + replace: false, + expected_precondition: None, + }, + Duration::from_secs(5), + ) + .await + .expect("seed"); + assert!(seeded.success); + + // The digest the caller authorizes against — computed from the state this + // actor can currently see. + let precondition = + crate::entity_actor::effects::entity_authorization_precondition(&seeded.state); + + // A genuine journal-advancing race: a second actor commits, so the holder's + // in-memory sequence goes stale. Its entry-time digest still matches its own + // memory, so only the append reveals the conflict — which is exactly the + // window in which a retry would commit against state the caller never saw. + let other = system.spawn( + EntityActor::with_persistence( + "Order", + entity_id, + order_table(), + serde_json::json!({"Customer": "Alice"}), + boxed(), + crate::storage::BackendLabel::Sim, + ), + format!("{entity_id}-other"), + ); + let concurrent: EntityResponse = other + .ask( + EntityMsg::UpdateFields { + fields: serde_json::json!({"Region": "eu"}), + replace: false, + expected_precondition: None, + }, + Duration::from_secs(5), + ) + .await + .expect("concurrent write"); + assert!(concurrent.success); + + let response: EntityResponse = holder + .ask( + EntityMsg::UpdateFields { + fields: serde_json::json!({"Customer": "Bob"}), + replace: false, + expected_precondition: Some(precondition), + }, + Duration::from_secs(5), + ) + .await + .expect("field update response"); + + assert!( + !response.success, + "a compare-and-set must not be retried onto state the caller never \ + authorized: {:?}", + response.error + ); + assert!( + response + .error + .as_deref() + .unwrap_or_default() + .contains("authorization became stale"), + "unexpected error: {:?}", + response.error + ); + assert_ne!( + response.state.fields["Customer"], "Bob", + "the speculative merge must be rolled back" + ); +} + +/// ARN-189 / F4.1. The race the retry loop exists for can also delete the entity. +/// The deletion check runs before the first attempt against the actor's memory; +/// after a replay the journal may hold a tombstone the actor had not seen. Without +/// rechecking, the retry appends a field update *after* the tombstone, and strict +/// replay then rejects the whole journal as events-after-terminal — the entity +/// becomes unrecoverable by the authoritative path. +/// +/// Driven by a second actor on the same store, so the conflict is a real stale +/// sequence rather than an injected one. +#[cfg(feature = "sim")] +#[tokio::test] +async fn field_update_retry_refuses_when_the_race_deleted_the_entity() { + let store = Arc::new(AppendFuseStore::no_faults(53)); + let entity_id = "arn189-deleted-in-race"; + let boxed = || crate::storage::BoxedEventStore::from_arc(store.clone()); + + let system = ActorSystem::new("sim-arn189-deleted-race"); + let writer = system.spawn( + EntityActor::with_persistence( + "Order", + entity_id, + order_table(), + serde_json::json!({"Customer": "Alice"}), + boxed(), + crate::storage::BackendLabel::Sim, + ), + entity_id, + ); + // Give the writer a durable baseline it has seen. + let seeded: EntityResponse = writer + .ask( + EntityMsg::UpdateFields { + fields: serde_json::json!({"Customer": "Seed"}), + replace: false, + expected_precondition: None, + }, + Duration::from_secs(5), + ) + .await + .expect("seed"); + assert!(seeded.success); + + // A second actor on the same journal deletes the entity. The first actor's + // in-memory sequence is now stale and knows nothing about the tombstone. + let deleter = system.spawn( + EntityActor::with_persistence( + "Order", + entity_id, + order_table(), + serde_json::json!({"Customer": "Alice"}), + boxed(), + crate::storage::BackendLabel::Sim, + ), + format!("{entity_id}-deleter"), + ); + let deleted: EntityResponse = deleter + .ask( + EntityMsg::Delete { + expected_authorization_precondition: None, + }, + Duration::from_secs(5), + ) + .await + .expect("delete"); + assert!( + deleted.success, + "delete should succeed: {:?}", + deleted.error + ); + + // The stale writer now tries to update. Its append conflicts, it replays, + // and it must see the tombstone and refuse. + let response: EntityResponse = writer + .ask( + EntityMsg::UpdateFields { + fields: serde_json::json!({"Customer": "TooLate"}), + replace: false, + expected_precondition: None, + }, + Duration::from_secs(5), + ) + .await + .expect("post-delete update response"); + + assert!( + !response.success, + "a field update must not be appended after the entity was deleted" + ); + assert!( + response + .error + .as_deref() + .unwrap_or_default() + .contains("after entity deletion"), + "unexpected error: {:?}", + response.error + ); +} + +/// ARN-189 / F4.4. The advertised budget is 1 initial attempt + 2 retries. +/// `field_update_recovers_from_a_concurrency_violation` pins 1 conflict and +/// `..._exceed_the_retry_budget` pins 3, so halving the budget to a single retry +/// is invisible to both. This pins the boundary: exactly 2 conflicts must still +/// recover. +#[cfg(feature = "sim")] +#[tokio::test] +async fn field_update_recovers_at_the_full_retry_budget() { + let store = Arc::new(AppendFuseStore::no_faults(59)); + let entity_id = "arn189-budget-boundary"; + let persistence_id = format!("default:Order:{entity_id}"); + + let system = ActorSystem::new("sim-arn189-boundary"); + let actor_ref = system.spawn( + EntityActor::with_persistence( + "Order", + entity_id, + order_table(), + serde_json::json!({"Customer": "Alice"}), + crate::storage::BoxedEventStore::from_arc(store.clone()), + crate::storage::BackendLabel::Sim, + ), + entity_id, + ); + let warmup: EntityResponse = actor_ref + .ask( + EntityMsg::UpdateFields { + fields: serde_json::json!({"Customer": "Warmup"}), + replace: false, + expected_precondition: None, + }, + Duration::from_secs(5), + ) + .await + .expect("warmup"); + assert!(warmup.success); + + // 2 conflicts: the second retry is the last one allowed. + store + .inner + .inject_concurrency_violations(&persistence_id, 2); + + let response: EntityResponse = actor_ref + .ask( + EntityMsg::UpdateFields { + fields: serde_json::json!({"Customer": "Bob"}), + replace: false, + expected_precondition: None, + }, + Duration::from_secs(5), + ) + .await + .expect("field update response"); + + assert!( + response.success, + "two conflicts are within the advertised 1 + 2 budget: {:?}", + response.error + ); + assert_eq!(response.state.fields["Customer"], "Bob"); +} + +/// ARN-189. A historical `FieldsReplaced` event carrying a non-object payload — +/// written by a build that predates the live guard — must not be able to replace +/// an entity's fields with an array during replay. The guard lives in +/// `apply_field_update`, so live and replay screen the same inputs. +#[test] +fn replaying_a_non_object_field_event_leaves_state_untouched() { + let mut state = EntityState { + entity_type: "Order".to_string(), + entity_id: "order-1".to_string(), + status: "Draft".to_string(), + item_count: 0, + counters: BTreeMap::new(), + booleans: BTreeMap::new(), + lists: BTreeMap::new(), + fields: serde_json::json!({"Customer": "Alice", "Id": "order-1"}), + events: std::collections::VecDeque::new(), + total_event_count: 0, + events_since_snapshot: 0, + last_snapshot_sequence_nr: 0, + sequence_nr: 0, + processed_idempotency_keys: BTreeMap::new(), + }; + let before = state.fields.clone(); + + for payload in [ + serde_json::json!([1, 2, 3]), + serde_json::json!("a string"), + serde_json::json!(null), + ] { + for replace in [true, false] { + assert!( + !crate::entity_actor::effects::apply_field_update(&mut state, &payload, replace), + "a non-object payload must be declined, not applied" + ); + assert_eq!( + state.fields, before, + "a non-object payload must not touch state (replace={replace}, \ + payload={payload:?})" + ); + } + } +} + +/// ARN-189 / F4.3. `previous_fields` is re-captured from the caught-up state on +/// every retry. If it were not, the terminal rollback after an exhausted retry +/// would restore the *pre-replay* fields — erasing a concurrent writer's +/// committed values from live state until the actor next rehydrates, so reads +/// would serve data the journal says is stale. +#[cfg(feature = "sim")] +#[tokio::test] +async fn exhausted_field_update_rolls_back_to_the_caught_up_state() { + let store = Arc::new(AppendFuseStore::no_faults(61)); + let entity_id = "arn189-rollback-base"; + let persistence_id = format!("default:Order:{entity_id}"); + let boxed = || crate::storage::BoxedEventStore::from_arc(store.clone()); + + let system = ActorSystem::new("sim-arn189-rollback-base"); + let writer = system.spawn( + EntityActor::with_persistence( + "Order", + entity_id, + order_table(), + serde_json::json!({"Customer": "Alice"}), + boxed(), + crate::storage::BackendLabel::Sim, + ), + entity_id, + ); + let seeded: EntityResponse = writer + .ask( + EntityMsg::UpdateFields { + fields: serde_json::json!({"Customer": "Seed"}), + replace: false, + expected_precondition: None, + }, + Duration::from_secs(5), + ) + .await + .expect("seed"); + assert!(seeded.success); + + // A second actor commits a value the first has never seen. + let other = system.spawn( + EntityActor::with_persistence( + "Order", + entity_id, + order_table(), + serde_json::json!({"Customer": "Alice"}), + boxed(), + crate::storage::BackendLabel::Sim, + ), + format!("{entity_id}-other"), + ); + let concurrent: EntityResponse = other + .ask( + EntityMsg::UpdateFields { + fields: serde_json::json!({"Customer": "Concurrent"}), + replace: false, + expected_precondition: None, + }, + Duration::from_secs(5), + ) + .await + .expect("concurrent write"); + assert!(concurrent.success); + + // The stale writer conflicts naturally on its first attempt, replays and + // picks up "Concurrent"; the injected conflicts then exhaust its retries. + store + .inner + .inject_concurrency_violations(&persistence_id, 4); + + let response: EntityResponse = writer + .ask( + EntityMsg::UpdateFields { + fields: serde_json::json!({"Customer": "Loser"}), + replace: false, + expected_precondition: None, + }, + Duration::from_secs(5), + ) + .await + .expect("exhausted update response"); + + assert!(!response.success, "the update should have failed"); + assert_eq!( + response.state.fields["Customer"], "Concurrent", + "rollback must restore the caught-up state, not the pre-replay one: {:?}", + response.state.fields + ); +} + +/// ARN-189 / F4.2. The event budget is checked before the first attempt against +/// the actor's memory. A concurrent writer can spend the budget while this update +/// is in flight, so the check is re-run after each catch-up replay. Without the +/// recheck the retry appends past `MAX_EVENTS_SINCE_SNAPSHOT`, growing the +/// snapshot tail past the hydration budget the entry check exists to protect — +/// which is how an entity becomes permanently unhydratable. +#[cfg(feature = "sim")] +#[tokio::test] +async fn field_update_retry_refuses_when_the_race_spent_the_event_budget() { + let store = Arc::new(AppendFuseStore::no_faults(67)); + // Snapshots never succeed: models the stalled snapshot path that lets the + // tail grow in the first place. + store + .fail_snapshots + .store(true, std::sync::atomic::Ordering::SeqCst); + let entity_id = "arn189-budget-race"; + let boxed = || crate::storage::BoxedEventStore::from_arc(store.clone()); + + let system = ActorSystem::new("sim-arn189-budget-race"); + let stale = system.spawn( + EntityActor::with_persistence( + "Order", + entity_id, + order_table(), + serde_json::json!({"Customer": "Alice"}), + boxed(), + crate::storage::BackendLabel::Sim, + ), + entity_id, + ); + // The stale writer sees one durable event and nothing after it. + let seeded: EntityResponse = stale + .ask( + EntityMsg::UpdateFields { + fields: serde_json::json!({"Customer": "Seed"}), + replace: false, + expected_precondition: None, + }, + Duration::from_secs(5), + ) + .await + .expect("seed"); + assert!(seeded.success); + + // A second actor on the same journal spends the whole budget. + let hog = system.spawn( + EntityActor::with_persistence( + "Order", + entity_id, + order_table(), + serde_json::json!({"Customer": "Alice"}), + boxed(), + crate::storage::BackendLabel::Sim, + ), + format!("{entity_id}-hog"), + ); + for i in 0..MAX_EVENTS_SINCE_SNAPSHOT { + let response: EntityResponse = hog + .ask( + EntityMsg::UpdateFields { + fields: serde_json::json!({"filler": i}), + replace: false, + expected_precondition: None, + }, + Duration::from_secs(5), + ) + .await + .expect("filler update"); + if !response.success { + break; + } + } + + // The stale writer now attempts an update: its append conflicts, it replays + // and finds the budget already spent. + let response: EntityResponse = stale + .ask( + EntityMsg::UpdateFields { + fields: serde_json::json!({"Customer": "TooLate"}), + replace: false, + expected_precondition: None, + }, + Duration::from_secs(5), + ) + .await + .expect("post-race update response"); + + assert!( + !response.success, + "an update must not append past the event budget after catching up" + ); + assert!( + response + .error + .as_deref() + .unwrap_or_default() + .contains("Event budget exhausted"), + "unexpected error: {:?}", + response.error + ); +} + +/// ARN-189 / F4.5. Under a lenient replay policy a field-update event whose +/// payload no longer deserializes is skipped so hydration survives spec +/// evolution — but that means an update the caller was told was durable is absent +/// from the rebuilt state. A `tracing::warn!` alone leaves that undetectable in +/// aggregate, so it is also counted. Asserted through a real meter, because a +/// counter nobody reads is indistinguishable from one that is never incremented. +#[cfg(feature = "sim")] +#[tokio::test] +async fn replay_skip_of_a_field_update_event_is_counted() { + use opentelemetry_sdk::metrics::{InMemoryMetricExporter, SdkMeterProvider}; + use temper_runtime::persistence::EventStore; + use temper_store_sim::SimEventStore; + + let exporter = InMemoryMetricExporter::default(); + let provider = SdkMeterProvider::builder() + .with_periodic_exporter(exporter.clone()) + .build(); + opentelemetry::global::set_meter_provider(provider.clone()); + + let store = Arc::new(SimEventStore::no_faults(71)); + let pid = "default:Order:arn189-replay-skip-metric"; + + // A journaled field update whose payload does not deserialize as an + // `EntityEvent` — the shape a build under a previous schema could have left. + let bad_env = PersistenceEnvelope { + sequence_nr: 0, + event_type: crate::entity_actor::effects::FIELDS_UPDATED_EVENT.to_string(), + payload: serde_json::json!({ + "action": 999, + "params": {"Customer": "Bob"} + }), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp: sim_now(), + actor_id: pid.to_string(), + }, + }; + store + .append(pid, 0, std::slice::from_ref(&bad_env)) + .await + .expect("append malformed field-update event"); + + // Rehydrate: the malformed event is skipped (lenient policy) and counted. + let system = ActorSystem::new("sim-arn189-skip-metric"); + let actor_ref = system.spawn( + EntityActor::with_persistence( + "Order", + "arn189-replay-skip-metric", + order_table(), + serde_json::json!({"Customer": "Alice"}), + crate::storage::BoxedEventStore::from_arc(store.clone()), + crate::storage::BackendLabel::Sim, + ), + "arn189-replay-skip-metric", + ); + let after: EntityResponse = actor_ref + .ask(EntityMsg::GetState, Duration::from_secs(5)) + .await + .expect("state after rehydration"); + assert_eq!( + after.state.fields["Customer"], "Alice", + "the malformed update must be skipped, not applied" + ); + + provider.force_flush().expect("flush metrics"); + let counted = exporter + .get_finished_metrics() + .expect("metrics") + .iter() + .flat_map(|rm| rm.scope_metrics.iter()) + .flat_map(|sm| sm.metrics.iter()) + .any(|m| m.name == "temper_entity_field_update_replay_skipped_total"); + assert!( + counted, + "a skipped field-update event must be counted, not only logged" + ); +} diff --git a/crates/temper-server/src/entity_actor/authoritative_replay_test.rs b/crates/temper-server/src/entity_actor/authoritative_replay_test.rs new file mode 100644 index 000000000..2092f6695 --- /dev/null +++ b/crates/temper-server/src/entity_actor/authoritative_replay_test.rs @@ -0,0 +1,368 @@ +use super::*; + +use temper_runtime::persistence::{ + EventStore, PersistenceAppend, PersistenceAppendResult, PersistenceError, +}; + +const ORDER_IOA: &str = include_str!("../../../../test-fixtures/specs/order.ioa.toml"); + +#[derive(Clone, Default)] +struct StaticEventStore { + events: Vec, + snapshot: Option<(u64, Vec)>, + read_error: Option, +} + +impl EventStore for StaticEventStore { + async fn append( + &self, + _persistence_id: &str, + _expected_sequence: u64, + _events: &[PersistenceEnvelope], + ) -> Result { + Err(PersistenceError::Storage( + "static replay store is read-only".to_string(), + )) + } + + async fn append_batch( + &self, + _appends: &[PersistenceAppend], + ) -> Result, PersistenceError> { + Err(PersistenceError::Storage( + "static replay store is read-only".to_string(), + )) + } + + async fn read_events( + &self, + _persistence_id: &str, + from_sequence: u64, + ) -> Result, PersistenceError> { + if let Some(error) = &self.read_error { + return Err(PersistenceError::Storage(error.clone())); + } + Ok(self + .events + .iter() + .filter(|event| event.sequence_nr > from_sequence) + .cloned() + .collect()) + } + + async fn save_snapshot( + &self, + _persistence_id: &str, + _sequence_nr: u64, + _snapshot: &[u8], + ) -> Result<(), PersistenceError> { + Ok(()) + } + + async fn load_snapshot( + &self, + _persistence_id: &str, + ) -> Result)>, PersistenceError> { + Ok(self.snapshot.clone()) + } + + async fn list_entity_ids( + &self, + _tenant: &str, + ) -> Result, PersistenceError> { + Ok(Vec::new()) + } + + async fn list_entity_ids_by_type( + &self, + _tenant: &str, + _entity_type: &str, + ) -> Result, PersistenceError> { + Ok(Vec::new()) + } +} + +fn order_table() -> TransitionTable { + TransitionTable::from_ioa_source(ORDER_IOA) +} + +fn envelope( + sequence_nr: u64, + action: &str, + from_status: &str, + to_status: &str, +) -> PersistenceEnvelope { + let event = EntityEvent { + action: action.to_string(), + from_status: from_status.to_string(), + to_status: to_status.to_string(), + timestamp: sim_now(), + params: serde_json::json!({}), + idempotency_key: None, + }; + PersistenceEnvelope { + sequence_nr, + event_type: action.to_string(), + payload: serde_json::to_value(event).expect("test event should serialize"), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp: sim_now(), + actor_id: "default:Order:security-replay".to_string(), + }, + } +} + +async fn authoritative_replay(store: StaticEventStore) -> Result { + let initial_fields = serde_json::json!({}); + recover_authoritative_entity_state_from_store( + "default", + "Order", + "security-replay", + &order_table(), + &BoxedEventStore::new(store), + BackendLabel::Turso, + &initial_fields, + None, + ) + .await +} + +#[tokio::test] +async fn authoritative_replay_rejects_sequence_gaps() { + let error = authoritative_replay(StaticEventStore { + events: vec![ + envelope(1, "AddItem", "Draft", "Draft"), + envelope(3, "CancelOrder", "Draft", "Cancelled"), + ], + ..StaticEventStore::default() + }) + .await + .expect_err("security replay must reject a journal gap"); + + assert!(error.to_string().contains("non-contiguous journal")); +} + +#[tokio::test] +async fn authoritative_replay_rejects_malformed_events() { + let mut malformed = envelope(1, "CancelOrder", "Draft", "Cancelled"); + malformed.payload = serde_json::json!({"action": 42}); + let error = authoritative_replay(StaticEventStore { + events: vec![malformed], + ..StaticEventStore::default() + }) + .await + .expect_err("security replay must reject malformed history"); + + assert!(error.to_string().contains("invalid event")); +} + +#[tokio::test] +async fn authoritative_replay_ignores_ahead_stale_snapshot() { + let table = order_table(); + let initial_fields = serde_json::json!({}); + let mut snapshot_state = + EntityActor::build_initial_state("Order", "security-replay", &table, &initial_fields); + snapshot_state.status = "Delivered".to_string(); + snapshot_state.fields["Status"] = serde_json::json!("Delivered"); + snapshot_state.sequence_nr = 99; + snapshot_state.total_event_count = 99; + let snapshot = EntityActor::serialize_snapshot_state(&snapshot_state) + .expect("test snapshot should serialize"); + let store = StaticEventStore { + events: vec![envelope(1, "CancelOrder", "Draft", "Cancelled")], + snapshot: Some((99, snapshot)), + read_error: None, + }; + let boxed = BoxedEventStore::new(store.clone()); + + let snapshot_recovery = recover_entity_state_from_store( + "default", + "Order", + "security-replay", + &table, + &boxed, + BackendLabel::Turso, + &initial_fields, + None, + false, + ) + .await + .expect("ordinary recovery accepts the snapshot"); + assert_eq!(snapshot_recovery.status, "Delivered"); + + let authoritative = authoritative_replay(store) + .await + .expect("complete journal is valid"); + assert_eq!(authoritative.status, "Cancelled"); + assert_eq!(authoritative.sequence_nr, 1); +} + +#[tokio::test] +async fn authoritative_replay_propagates_journal_read_failure() { + let error = authoritative_replay(StaticEventStore { + read_error: Some("injected identity journal failure".to_string()), + ..StaticEventStore::default() + }) + .await + .expect_err("security replay must fail closed on journal read failure"); + + assert!( + error + .to_string() + .contains("injected identity journal failure") + ); +} + +#[tokio::test] +async fn authoritative_replay_rejects_history_after_tombstone() { + let error = authoritative_replay(StaticEventStore { + events: vec![ + envelope(1, "Deleted", "Draft", "Deleted"), + envelope(2, "CancelOrder", "Draft", "Cancelled"), + ], + ..StaticEventStore::default() + }) + .await + .expect_err("security replay must treat tombstones as terminal"); + + assert!(error.to_string().contains("after terminal tombstone")); +} + +#[tokio::test] +async fn authoritative_replay_rejects_contradictory_tombstone() { + let error = authoritative_replay(StaticEventStore { + events: vec![envelope(1, "Deleted", "Draft", "Active")], + ..StaticEventStore::default() + }) + .await + .expect_err("security replay must reject a non-terminal tombstone payload"); + + assert!(error.to_string().contains("tombstone transition")); +} + +#[tokio::test] +async fn authoritative_replay_rejects_envelope_payload_action_mismatch() { + let mut mismatched = envelope(1, "AddItem", "Draft", "Draft"); + mismatched.event_type = "CancelOrder".to_string(); + let error = authoritative_replay(StaticEventStore { + events: vec![mismatched], + ..StaticEventStore::default() + }) + .await + .expect_err("security replay must bind envelope type to payload action"); + + assert!(error.to_string().contains("differs from payload action")); +} + +#[tokio::test] +async fn authoritative_replay_rejects_impossible_transition_history() { + for event in [ + envelope(1, "CancelOrder", "Submitted", "Cancelled"), + envelope(1, "AddItem", "Draft", "Delivered"), + ] { + let error = authoritative_replay(StaticEventStore { + events: vec![event], + ..StaticEventStore::default() + }) + .await + .expect_err("security replay must validate transition semantics"); + assert!(error.to_string().contains("incompatible event")); + } +} + +#[tokio::test] +async fn authoritative_replay_rejects_misbound_actor_metadata() { + let mut misbound = envelope(1, "AddItem", "Draft", "Draft"); + misbound.metadata.actor_id = "default:Order:another-entity".to_string(); + let error = authoritative_replay(StaticEventStore { + events: vec![misbound], + ..StaticEventStore::default() + }) + .await + .expect_err("security replay must bind every event to its actor stream"); + + assert!(error.to_string().contains("bound to actor")); +} + +/// ARN-189. Authoritative replay backs identity and authority resolution, so a +/// malformed field-update event must FAIL it, not be skipped: a `FieldsReplaced` +/// that was written to revoke authority — clearing a privileged field — would +/// otherwise be dropped, and the "authoritative" state would preserve exactly the +/// authority it was meant to remove. The lenient path skips and counts; this pins +/// the strict path. +#[tokio::test] +async fn authoritative_replay_rejects_malformed_field_update_events() { + for event_type in [ + crate::entity_actor::effects::FIELDS_UPDATED_EVENT, + crate::entity_actor::effects::FIELDS_REPLACED_EVENT, + ] { + let malformed = PersistenceEnvelope { + sequence_nr: 1, + event_type: event_type.to_string(), + // Not deserializable as an EntityEvent: `action` is a number. + payload: serde_json::json!({ + "action": 999, + "params": {"Privileged": true} + }), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp: sim_now(), + actor_id: "default:Order:security-replay".to_string(), + }, + }; + let error = authoritative_replay(StaticEventStore { + events: vec![malformed], + read_error: None, + snapshot: None, + }) + .await + .expect_err("a malformed field-update event must fail authoritative replay"); + assert!( + error.to_string().contains("field-update"), + "unexpected error for {event_type}: {error}" + ); + } +} + +/// Same property for a payload that deserializes but is not an object: a +/// journaled non-object `FieldsReplaced` (only writable by a build predating the +/// live guard) cannot be applied, so strict replay must fail rather than +/// pretending the journal was fully replayed. +#[tokio::test] +async fn authoritative_replay_rejects_non_object_field_update_payloads() { + let event = EntityEvent { + action: crate::entity_actor::effects::FIELDS_REPLACED_EVENT.to_string(), + from_status: "Draft".to_string(), + to_status: "Draft".to_string(), + timestamp: sim_now(), + params: serde_json::json!([1, 2, 3]), + idempotency_key: None, + }; + let non_object = PersistenceEnvelope { + sequence_nr: 1, + event_type: event.action.clone(), + payload: serde_json::to_value(event).expect("serialize"), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp: sim_now(), + actor_id: "default:Order:security-replay".to_string(), + }, + }; + let error = authoritative_replay(StaticEventStore { + events: vec![non_object], + read_error: None, + snapshot: None, + }) + .await + .expect_err("a non-object field-update payload must fail authoritative replay"); + assert!( + error.to_string().contains("field-update"), + "unexpected error: {error}" + ); +} diff --git a/crates/temper-server/src/entity_actor/effects.rs b/crates/temper-server/src/entity_actor/effects.rs index 1dd7aac5f..f96a3407e 100644 --- a/crates/temper-server/src/entity_actor/effects.rs +++ b/crates/temper-server/src/entity_actor/effects.rs @@ -19,6 +19,64 @@ use crate::blobs::{FIELD_OVERFLOW_BLOB_PREFIX, OverflowBlobWrite, blob_ref_value use super::types::{EntityEvent, EntityState, MAX_EVENTS_SINCE_SNAPSHOT}; +/// Journal event type for a PATCH-style field merge (ARN-189). +pub(crate) const FIELDS_UPDATED_EVENT: &str = "FieldsUpdated"; +/// Journal event type for a PUT-style field replacement (ARN-189). +pub(crate) const FIELDS_REPLACED_EVENT: &str = "FieldsReplaced"; + +/// Apply a PATCH/PUT field update to entity state (ARN-189). +/// +/// The single source of truth for field-update semantics, called by BOTH the +/// live `EntityMsg::UpdateFields` handler and journal replay, so a rehydrated +/// entity reaches exactly the state the live update produced. +/// +/// - `replace == false` (PATCH): merge `fields` into the existing object. +/// A non-object existing/incoming value leaves state unchanged, matching +/// the historical live behavior. +/// - `replace == true` (PUT): replace all fields, preserving `Id` and +/// `Status` from the entity itself. +#[must_use = "a field update that did not apply is a dropped update; count or refuse it"] +pub(crate) fn apply_field_update( + state: &mut EntityState, + fields: &serde_json::Value, + replace: bool, +) -> bool { + // One helper for both the live `UpdateFields` arm and journal replay + // (ARN-189). Replay must reproduce the live result exactly, so every + // transformation belongs here — a step applied only on the live path would + // silently rewrite the entity on the next rehydration. + // + // `canonicalize_entity_fields` is the single enforcement point for + // runtime-owned fields: it both strips the keys a caller must not set + // (`has_spec`, `ctx_owner_status`, ...) and restores the authoritative + // `Id`/`Status` (and their lowercase aliases). The live arm additionally + // sanitizes the *event payload* before journaling (see + // `field_updates::commit_field_update`) so the journal never records a forged + // key, which canonicalizing `state.fields` alone would not prevent. + // + // Guard here, not only at the live arm: replay feeds this the `params` of + // whatever is in the journal, including events written by a build that + // predates the live guard. A `FieldsReplaced` carrying `[1,2,3]` would set + // `fields` to an array, after which `canonicalize_entity_fields` cannot + // restore `Id`/`Status` — there is no object to insert into. Refusing in the + // shared helper is what makes live and replay agree on every input, not just + // the ones the live path screens. + if !fields.is_object() { + return false; + } + if replace { + state.fields = fields.clone(); + } else if let (Some(existing), Some(updates)) = + (state.fields.as_object_mut(), fields.as_object()) + { + for (k, v) in updates { + existing.insert(k.clone(), v.clone()); + } + } + canonicalize_entity_fields(&mut state.fields, &state.entity_id, &state.status); + true +} + /// A scheduled action to fire after a delay. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ScheduledAction { @@ -258,7 +316,8 @@ pub fn process_action_with_xref_and_field_mode( } let effective_params = normalize_ref_action_params(state, action, params); - let params = effective_params.as_ref(); + let sanitized_params = sanitize_action_params(effective_params.as_ref()); + let params = sanitized_params.as_ref(); let (custom_effects, scheduled_actions, spawn_requests, schedule_at_requests) = apply_effects(state, &transition_result.effects, params); @@ -325,6 +384,91 @@ pub fn process_action_with_xref_and_field_mode( } } +/// Remove caller fields whose values are derived authoritatively by the runtime. +/// +/// The returned value borrows the input when no reserved keys are present and +/// clones only when sanitization is required. Persisted events therefore never +/// acquire a second mutable identity, lifecycle, or context-status truth. +pub(crate) fn sanitize_action_params( + params: &serde_json::Value, +) -> std::borrow::Cow<'_, serde_json::Value> { + let Some(fields) = params.as_object() else { + return std::borrow::Cow::Borrowed(params); + }; + if !fields + .keys() + .any(|key| temper_spec::automaton::is_server_derived_field_name(key)) + { + return std::borrow::Cow::Borrowed(params); + } + let mut sanitized = fields.clone(); + sanitized.retain(|key, _| !temper_spec::automaton::is_server_derived_field_name(key)); + std::borrow::Cow::Owned(serde_json::Value::Object(sanitized)) +} + +/// Compute the exact actor-state precondition bound to an external field-update +/// authorization decision. +/// +/// The digest includes the durable sequence and the authorization-visible local +/// lifecycle/field state. Recursive canonical JSON hashing keeps the result +/// deterministic even when callers construct object keys in different orders. +pub(crate) fn entity_authorization_precondition(state: &EntityState) -> String { + fn update_bytes(hasher: &mut Sha256, bytes: &[u8]) { + let len = u64::try_from(bytes.len()).unwrap_or(u64::MAX); + hasher.update(len.to_be_bytes()); + hasher.update(bytes); + } + + fn update_json(hasher: &mut Sha256, value: &serde_json::Value) { + match value { + serde_json::Value::Null => hasher.update([0]), + serde_json::Value::Bool(value) => hasher.update([1, u8::from(*value)]), + serde_json::Value::Number(value) => { + hasher.update([2]); + update_bytes(hasher, value.to_string().as_bytes()); + } + serde_json::Value::String(value) => { + hasher.update([3]); + update_bytes(hasher, value.as_bytes()); + } + serde_json::Value::Array(values) => { + hasher.update([4]); + hasher.update( + u64::try_from(values.len()) + .unwrap_or(u64::MAX) + .to_be_bytes(), + ); + for value in values { + update_json(hasher, value); + } + } + serde_json::Value::Object(values) => { + hasher.update([5]); + hasher.update( + u64::try_from(values.len()) + .unwrap_or(u64::MAX) + .to_be_bytes(), + ); + let mut keys = values.keys().collect::>(); + keys.sort_unstable(); + for key in keys { + update_bytes(hasher, key.as_bytes()); + if let Some(value) = values.get(key) { + update_json(hasher, value); + } + } + } + } + } + + let mut hasher = Sha256::new(); + hasher.update(b"temper-entity-authorization-precondition-v1"); + hasher.update(state.sequence_nr.to_be_bytes()); + update_bytes(&mut hasher, state.status.as_bytes()); + update_json(&mut hasher, &state.fields); + format!("{:x}", hasher.finalize()) +} + fn validate_ref_action_contract( state: &EntityState, action: &str, @@ -763,15 +907,14 @@ pub fn sync_fields_with_metadata( let entity_type = state.entity_type.clone(); let entity_id = state.entity_id.clone(); if let Some(obj) = state.fields.as_object_mut() { - obj.insert( - "Status".to_string(), - serde_json::Value::String(state.status.clone()), - ); + canonicalize_entity_field_map(obj, &entity_id, &state.status); prune_transient_action_fields(&entity_type, obj); // Project action params into fields if let Some(p) = params.as_object() { for (k, v) in p { - if is_transient_action_field(&entity_type, k) { + if is_transient_action_field(&entity_type, k) + || temper_spec::automaton::is_server_derived_field_name(k) + { continue; } let field_meta = state_var_metadata.and_then(|m| m.get(k.as_str())); @@ -791,14 +934,23 @@ pub fn sync_fields_with_metadata( } // Sync counters into fields for (k, v) in &state.counters { + if temper_spec::automaton::is_server_derived_field_name(k) { + continue; + } obj.insert(k.clone(), serde_json::Value::Number((*v as u64).into())); } // Sync booleans into fields for (k, v) in &state.booleans { + if temper_spec::automaton::is_server_derived_field_name(k) { + continue; + } obj.insert(k.clone(), serde_json::Value::Bool(*v)); } // Sync lists into fields for (k, v) in &state.lists { + if temper_spec::automaton::is_server_derived_field_name(k) { + continue; + } let arr: Vec = v .iter() .map(|s| serde_json::Value::String(s.clone())) @@ -821,6 +973,38 @@ pub fn sync_fields_with_metadata( overflow_blobs } +/// Remove mutable aliases of runtime-owned fields and publish canonical identity/status. +pub(crate) fn canonicalize_entity_fields( + fields: &mut serde_json::Value, + entity_id: &str, + status: &str, +) { + let Some(fields) = fields.as_object_mut() else { + return; + }; + canonicalize_entity_field_map(fields, entity_id, status); +} + +fn canonicalize_entity_field_map( + fields: &mut serde_json::Map, + entity_id: &str, + status: &str, +) { + fields.retain(|key, _| !temper_spec::automaton::is_server_derived_field_name(key)); + for key in ["id", "Id"] { + fields.insert( + key.to_string(), + serde_json::Value::String(entity_id.to_string()), + ); + } + for key in ["status", "Status"] { + fields.insert( + key.to_string(), + serde_json::Value::String(status.to_string()), + ); + } +} + fn prune_transient_action_fields( entity_type: &str, fields: &mut serde_json::Map, @@ -1905,4 +2089,57 @@ params = ["NewCommitSha"] assert_eq!(overflow.len(), 1, "dedupe by content hash"); } + + #[test] + fn action_processing_strips_runtime_owned_fields_from_state_and_event() { + let _guard = temper_runtime::scheduler::install_deterministic_context(42); + let table = temper_jit::table::TransitionTable::from_ioa_source( + r#" +[automaton] +name = "Document" +states = ["Initial", "Approved"] +initial = "Initial" + +[[action]] +name = "Approve" +kind = "input" +from = ["Initial"] +to = "Approved" +params = ["Title"] +"#, + ); + let mut state = make_state("Document", "doc-1"); + state.fields = serde_json::json!({ + "Id": "forged-before", + "Status": "forged-before", + "ctx_owner_status": "Privileged" + }); + let result = process_action( + &mut state, + &table, + "Approve", + &serde_json::json!({ + "Id": "forged-after", + "id": "forged-after", + "Status": "Rejected", + "status": "Rejected", + "has_spec": false, + "HasSpec": false, + "ctx_owner_status": "Privileged", + "Title": "Trusted title" + }), + ); + + assert!(result.success); + assert_eq!(state.fields["id"], "doc-1"); + assert_eq!(state.fields["Id"], "doc-1"); + assert_eq!(state.fields["status"], "Approved"); + assert_eq!(state.fields["Status"], "Approved"); + assert_eq!(state.fields["Title"], "Trusted title"); + for reserved in ["has_spec", "HasSpec", "ctx_owner_status"] { + assert!(state.fields.get(reserved).is_none(), "persisted {reserved}"); + } + let event = result.event.expect("successful action event"); + assert_eq!(event.params, serde_json::json!({"Title": "Trusted title"})); + } } diff --git a/crates/temper-server/src/entity_actor/field_updates.rs b/crates/temper-server/src/entity_actor/field_updates.rs new file mode 100644 index 000000000..05fa2b1be --- /dev/null +++ b/crates/temper-server/src/entity_actor/field_updates.rs @@ -0,0 +1,285 @@ +//! The durable transaction behind OData PATCH/PUT (ARN-189, ADR-0157). +//! +//! Field updates bypass the spec's action vocabulary — no guards, no effects, +//! no transition — but they still change entity state, so they have to be +//! journaled or they vanish on eviction and restart. That makes this a small +//! transaction with the same obligations as the `Action` arm: refuse before +//! mutating, never acknowledge what was not persisted, and converge with what +//! replay will rebuild. +//! +//! It lives beside the actor rather than inside the message match because it is +//! ~250 lines of policy, and because the caller's only job is to turn the +//! outcome into a reply. + +use serde_json::Value; +use temper_runtime::persistence::PersistenceError; +use temper_runtime::scheduler::sim_now; + +use super::actor::{EntityActor, ReplayPolicy}; +use super::effects; +use super::types::{EntityEvent, EntityState, MAX_EVENTS_SINCE_SNAPSHOT}; + +/// Attempts allowed after the first, matching the `Action` arm's ADR-0046 budget. +const MAX_RETRIES: u32 = 2; + +/// Apply a field update and make it durable, or refuse it. +/// +/// `Ok(())` means the update is in the journal. `Err(reason)` means it is not, +/// and `state` has been restored — a refusal never leaves a partial write. The +/// caller replies with the reason verbatim. +pub(super) async fn commit_field_update( + actor: &EntityActor, + state: &mut EntityState, + fields: Value, + replace: bool, + expected_precondition: Option, +) -> Result<(), String> { + let has_precondition = expected_precondition.is_some(); + if let Some(expected) = expected_precondition + && effects::entity_authorization_precondition(state) != expected + { + return Err(STALE_AUTHORIZATION.to_string()); + } + if state.status == "Deleted" { + return Err("cannot update fields after entity deletion".to_string()); + } + // `parse_json_body_or_400` accepts any valid JSON, so a `PUT` body of + // `[1,2,3]` reaches here. With `replace`, `apply_field_update` would set + // `fields` to the array and then fail to restore `Id`/`Status` — there is no + // object to insert into — and the append would co-commit zero key and zero + // vector rows, purging the entity's index. Before field updates were + // journaled that corruption was in-memory and healed on restart; persisting + // it would make it permanent. + if !fields.is_object() { + return Err("entity field update must be a JSON object".to_string()); + } + // The same budget that gates spec actions. Field updates append journal + // events too, so ungated they could grow the snapshot replay tail past + // `MAX_EVENTS_SINCE_SNAPSHOT` while the snapshot path is stalled, after which + // the entity can never rehydrate. Rejected before mutating. + if let Some(reason) = budget_refusal(actor, state, replace) { + return Err(reason); + } + + // Sanitize once and use the same value for state and journal. + // `apply_field_update` sanitizes internally as well — it must, so that + // replaying an event written before this guard still lands on clean state — + // but the event written now must not carry a caller's forged `has_spec` or + // `ctx_owner_status` into the journal in the first place. + let fields = effects::sanitize_action_params(&fields).into_owned(); + + let action = if replace { + effects::FIELDS_REPLACED_EVENT + } else { + effects::FIELDS_UPDATED_EVENT + }; + + // Apply speculatively so the append co-commits key/vector index rows derived + // from the NEW fields, then journal fail-closed: an update that is not + // durable must not be acknowledged. Rolled back on failure. + let mut previous_fields = state.fields.clone(); + // Bind the result: `debug_assert!` does not evaluate its argument in release + // builds, so asserting the call directly would skip the update in production. + let applied = effects::apply_field_update(state, &fields, replace); + debug_assert!( + applied, + "object-ness was checked above; the apply cannot decline" + ); + let mut event = field_event(action, state, &fields); + + let (Some(store), Some(backend)) = (actor.event_journal.as_ref(), actor.event_backend) else { + // No configured persistence: memory-only, as in every other handler. + return Ok(()); + }; + + let mut attempt: u32 = 0; + loop { + match actor + .persist_event(store, backend, &actor.persistence_id(), state, &event) + .await + { + Ok(_) => break, + // A preconditioned update is a compare-and-set: the caller authorized + // this write against one exact state digest. A conflict is proof the + // journal held state the actor's memory did not, so replaying and + // re-applying would commit against state the caller never saw and + // Cedar never evaluated. `entity_ops` already caps preconditioned asks + // at a single attempt for that reason; retrying here would reintroduce + // one layer down the retry the layer above forbids. + Err(PersistenceError::ConcurrencyViolation { .. }) if has_precondition => { + state.fields = previous_fields; + return Err(STALE_AUTHORIZATION.to_string()); + } + Err(PersistenceError::ConcurrencyViolation { actual, .. }) if attempt < MAX_RETRIES => { + attempt += 1; + match catch_up(actor, state, attempt, actual, action).await { + Ok(()) => {} + Err(reason) => { + state.fields = previous_fields; + return Err(reason); + } + } + if let Some(reason) = post_replay_refusal(actor, state, replace) { + return Err(reason); + } + // Re-apply onto the caught-up state and rebuild the event against + // its (possibly new) status. + previous_fields = state.fields.clone(); + let applied = effects::apply_field_update(state, &fields, replace); + debug_assert!( + applied, + "object-ness was checked above; the apply cannot decline" + ); + event = field_event(action, state, &fields); + } + Err(e) => { + state.fields = previous_fields; + return Err(format!("persistence failed: {e}")); + } + } + } + + state.push_event_bounded(event); + + let persistence_id = actor.persistence_id(); + if let Err(e) = EntityActor::maybe_save_snapshot( + store, + actor.snapshot_queue.as_ref(), + &persistence_id, + state, + ) + .await + { + tracing::warn!( + entity = %state.entity_id, + seq = state.sequence_nr, + error = %e, + "failed to persist snapshot" + ); + } + + Ok(()) +} + +/// The refusal an update gets when the state it was authorized against moved. +/// Shared by the entry check and the conflict path so a caller cannot tell the +/// two apart — both mean "re-read and retry". +const STALE_AUTHORIZATION: &str = + "field update authorization became stale; retry against current state"; + +fn field_event(action: &str, state: &EntityState, fields: &Value) -> EntityEvent { + EntityEvent { + action: action.to_string(), + from_status: state.status.clone(), + to_status: state.status.clone(), + timestamp: sim_now(), + params: fields.clone(), + idempotency_key: None, + } +} + +fn budget_refusal(actor: &EntityActor, state: &EntityState, replace: bool) -> Option { + if state.events_since_snapshot < MAX_EVENTS_SINCE_SNAPSHOT { + return None; + } + let workspace_id = super::actor::event_budget_workspace_id(state); + crate::event_budget_metrics::record_exhausted( + &actor.tenant, + &state.entity_type, + &state.entity_id, + &workspace_id, + ); + tracing::warn!( + tenant = %actor.tenant, + entity_type = %state.entity_type, + entity_id = %state.entity_id, + workspace_id = %workspace_id, + status = %state.status, + replace, + events_since_snapshot = state.events_since_snapshot, + total_event_count = state.total_event_count, + max_events_since_snapshot = MAX_EVENTS_SINCE_SNAPSHOT, + "Event budget exhausted (field update rejected)" + ); + Some(format!( + "Event budget exhausted ({MAX_EVENTS_SINCE_SNAPSHOT} max since snapshot)" + )) +} + +/// Rebuild `state` from the journal after a sequence conflict. +/// +/// Rebuilds from a *fresh* initial state, exactly as +/// `recover_entity_state_from_store` does. `replay_events` applies onto whatever +/// state it is handed and never resets it, so replaying onto the live state +/// re-applies every event on top of its own effects: the events deque grows, +/// `total_event_count` / `events_since_snapshot` climb, and non-idempotent +/// effects (counter increments) fire twice. That corruption would be returned to +/// the caller, upserted into the query projection, and made durable by the next +/// snapshot. Rolling back `fields` alone cannot help — those other fields were +/// never part of the speculative update. +async fn catch_up( + actor: &EntityActor, + state: &mut EntityState, + attempt: u32, + actual: u64, + action: &str, +) -> Result<(), String> { + // Back off first, or against a live concurrent writer the whole budget burns + // in microseconds and the retry buys nothing. + super::actor::sleep_persistence_retry(std::time::Duration::from_millis(if attempt == 1 { + 10 + } else { + 50 + })) + .await; + tracing::warn!( + entity = %state.entity_id, + action = %action, + actual_seq = actual, + attempt, + "field update hit optimistic-concurrency violation; replaying and retrying" + ); + + let (Some(store), Some(backend)) = (actor.event_journal.as_ref(), actor.event_backend) else { + return Err("persistence unavailable during conflict recovery".to_string()); + }; + let table = actor.table.read().expect("table lock poisoned").clone(); + let mut caught_up = EntityActor::build_initial_state( + &actor.entity_type, + &state.entity_id, + &table, + &actor.initial_fields, + ); + // A replay failure is returned as a refusal rather than propagated: bailing + // out of the message handler would leave the caller's `ask` unanswered until + // it times out, with the entity mid-rollback. + EntityActor::replay_events( + &table, + store, + backend, + &mut caught_up, + &actor.tenant, + actor.blob_store.as_ref(), + ReplayPolicy::LenientSnapshot, + ) + .await + .map_err(|e| format!("conflict recovery replay failed: {e}"))?; + debug_assert!( + caught_up.sequence_nr >= actual, + "POSTCONDITION: field-update replay under-reached the authoritative sequence \ + (sequence_nr={} < actual={actual})", + caught_up.sequence_nr + ); + *state = caught_up; + Ok(()) +} + +/// Re-run the refusals that were checked before the first attempt. The race may +/// have deleted the entity or spent the budget, and both were only ever checked +/// against the actor's pre-conflict memory. +fn post_replay_refusal(actor: &EntityActor, state: &EntityState, replace: bool) -> Option { + if state.status == "Deleted" { + return Some("cannot update fields after entity deletion".to_string()); + } + budget_refusal(actor, state, replace) +} diff --git a/crates/temper-server/src/entity_actor/mod.rs b/crates/temper-server/src/entity_actor/mod.rs index a0f31f33c..2917e5155 100644 --- a/crates/temper-server/src/entity_actor/mod.rs +++ b/crates/temper-server/src/entity_actor/mod.rs @@ -6,12 +6,16 @@ mod actor; pub mod effects; +mod field_updates; +mod replay_validation; pub mod sim_handler; mod snapshot_queue; pub mod types; pub use actor::EntityActor; -pub(crate) use actor::recover_entity_state_from_store; +pub(crate) use actor::{ + recover_authoritative_entity_state_from_store, recover_entity_state_from_store, +}; pub use effects::{ ProcessResult, ScheduledAction, apply_effects, apply_new_state_fallback, build_eval_context, process_action, process_action_with_xref, sync_fields, diff --git a/crates/temper-server/src/entity_actor/replay_validation.rs b/crates/temper-server/src/entity_actor/replay_validation.rs new file mode 100644 index 000000000..e23b8e179 --- /dev/null +++ b/crates/temper-server/src/entity_actor/replay_validation.rs @@ -0,0 +1,142 @@ +//! Semantic validation for security-sensitive full-journal replay. + +use temper_jit::table::{Effect, TransitionTable}; +use temper_runtime::actor::ActorError; +use temper_runtime::persistence::{CompositeEvent, PersistenceEnvelope}; + +use super::types::{EntityEvent, EntityState}; + +pub(super) fn validate_strict_composite_event( + tenant: &str, + state: &EntityState, + envelope: &PersistenceEnvelope, +) -> Result<(), ActorError> { + let event: CompositeEvent = + serde_json::from_value(envelope.payload.clone()).map_err(|error| { + ActorError::custom(format!( + "incompatible composite audit event for {}:{} at sequence {}: {error}", + state.entity_type, state.entity_id, envelope.sequence_nr + )) + })?; + if event.tenant != tenant + || event.parent_entity_type != state.entity_type + || event.parent_entity_id != state.entity_id + { + return Err(ActorError::custom(format!( + "misbound composite audit event for {}:{} at sequence {}", + state.entity_type, state.entity_id, envelope.sequence_nr + ))); + } + Ok(()) +} + +pub(super) fn validate_strict_entity_event( + table: &TransitionTable, + state: &EntityState, + envelope: &PersistenceEnvelope, + event: &EntityEvent, +) -> Result<(), ActorError> { + if envelope.event_type != event.action { + return Err(incompatible_event( + state, + envelope, + format!( + "envelope type '{}' differs from payload action '{}'", + envelope.event_type, event.action + ), + )); + } + + if event.action == "Deleted" { + if event.from_status != state.status || event.to_status != "Deleted" { + return Err(incompatible_event( + state, + envelope, + format!( + "tombstone transition '{} -> {}' does not match current '{}' -> 'Deleted'", + event.from_status, event.to_status, state.status + ), + )); + } + return Ok(()); + } + + if event.action == "Created" && event.from_status.is_empty() { + if envelope.sequence_nr != 1 + || state.total_event_count != 0 + || event.to_status != table.initial_state + { + return Err(incompatible_event( + state, + envelope, + format!( + "bootstrap Created event must be sequence 1 and target initial state '{}'", + table.initial_state + ), + )); + } + return Ok(()); + } + + if event.from_status != state.status { + return Err(incompatible_event( + state, + envelope, + format!( + "payload from-status '{}' does not match replay state '{}'", + event.from_status, state.status + ), + )); + } + + let Some(rule) = table.rules.iter().find(|rule| { + rule.name == event.action + && (rule.from_states.is_empty() + || rule + .from_states + .iter() + .any(|status| status == &state.status)) + }) else { + return Err(incompatible_event( + state, + envelope, + format!( + "action '{}' has no transition from '{}' in the active spec", + event.action, state.status + ), + )); + }; + + let mut expected_status = state.status.clone(); + for effect in &rule.effects { + if let Effect::SetState(status) = effect { + expected_status.clone_from(status); + } + } + let fallback_status = rule.to_state.as_deref().unwrap_or(&state.status); + if expected_status == state.status && !fallback_status.is_empty() { + expected_status = fallback_status.to_string(); + } + if event.to_status != expected_status { + return Err(incompatible_event( + state, + envelope, + format!( + "payload target '{}' differs from active transition target '{}'", + event.to_status, expected_status + ), + )); + } + Ok(()) +} + +fn incompatible_event( + state: &EntityState, + envelope: &PersistenceEnvelope, + detail: String, +) -> ActorError { + ActorError::custom(format!( + "incompatible event for {}:{} at sequence {}: {detail}", + state.entity_type, state.entity_id, envelope.sequence_nr + )) +} diff --git a/crates/temper-server/src/entity_actor/sim_handler.rs b/crates/temper-server/src/entity_actor/sim_handler.rs index 7082baa71..bd87585cc 100644 --- a/crates/temper-server/src/entity_actor/sim_handler.rs +++ b/crates/temper-server/src/entity_actor/sim_handler.rs @@ -39,6 +39,8 @@ impl EntityActorHandler { ) -> Self { let entity_type = entity_type.into(); let entity_id = entity_id.into(); + let mut fields = serde_json::json!({}); + super::effects::canonicalize_entity_fields(&mut fields, &entity_id, &table.initial_state); let state = EntityState { entity_type, @@ -48,7 +50,7 @@ impl EntityActorHandler { counters: std::collections::BTreeMap::new(), booleans: std::collections::BTreeMap::new(), lists: std::collections::BTreeMap::new(), - fields: serde_json::json!({}), + fields, events: std::collections::VecDeque::new(), total_event_count: 0, events_since_snapshot: 0, diff --git a/crates/temper-server/src/entity_actor/types.rs b/crates/temper-server/src/entity_actor/types.rs index a3e23d142..2d3dcafdf 100644 --- a/crates/temper-server/src/entity_actor/types.rs +++ b/crates/temper-server/src/entity_actor/types.rs @@ -48,6 +48,9 @@ pub enum EntityMsg { /// Covers the race where a dispatch-layer retry produces a second /// in-flight ask after the first one already processed. idempotency_key: Option, + /// Digest of the exact local state used for an external Cedar + /// decision. Internal dispatches omit it. + expected_authorization_precondition: Option, }, /// Get the current entity state. GetState, @@ -57,9 +60,17 @@ pub enum EntityMsg { UpdateFields { fields: serde_json::Value, replace: bool, + /// Digest of the exact state used for an external authorization + /// decision. The actor rejects the update if that state changed before + /// this message reached its mailbox. + expected_precondition: Option, }, /// Delete this entity. - Delete, + Delete { + /// Digest of the exact local state used for an external Cedar + /// decision. Internal dispatches omit it. + expected_authorization_precondition: Option, + }, } impl Message for EntityMsg {} diff --git a/crates/temper-server/src/event_budget_metrics.rs b/crates/temper-server/src/event_budget_metrics.rs index 72681e1d8..923d5fcfd 100644 --- a/crates/temper-server/src/event_budget_metrics.rs +++ b/crates/temper-server/src/event_budget_metrics.rs @@ -33,3 +33,33 @@ pub fn record_exhausted(tenant: &str, entity_type: &str, entity_id: &str, worksp ], ); } + +fn field_update_replay_skip_counter() -> &'static Counter { + static COUNTER: OnceLock> = OnceLock::new(); + COUNTER.get_or_init(|| { + global::meter("temper.runtime") + .u64_counter("temper_entity_field_update_replay_skipped_total") + .with_description( + "Journaled field-update events dropped during replay because their payload \ + did not deserialize. Each one is a silently lost PATCH/PUT.", + ) + .build() + }) +} + +/// Record a field-update event skipped during replay (ARN-189). +/// +/// The skip is the fail-safe branch — a malformed payload must not abort +/// rehydration — but it means a field update the caller was told had been +/// durably applied is now absent from the rebuilt state. A `tracing::warn!` +/// alone leaves that undetectable in aggregate; this makes it alertable. +pub fn record_field_update_replay_skip(tenant: &str, entity_type: &str, entity_id: &str) { + field_update_replay_skip_counter().add( + 1, + &[ + KeyValue::new("tenant", tenant.to_string()), + KeyValue::new("entity_type", entity_type.to_string()), + KeyValue::new("entity_id", entity_id.to_string()), + ], + ); +} diff --git a/crates/temper-server/src/events.rs b/crates/temper-server/src/events.rs index 36b8b4aaa..02df85c7f 100644 --- a/crates/temper-server/src/events.rs +++ b/crates/temper-server/src/events.rs @@ -6,15 +6,16 @@ use std::collections::BTreeMap; use std::convert::Infallible; -use axum::extract::State; -use axum::http::{HeaderMap, StatusCode}; +use axum::extract::{Extension, State}; +use axum::http::StatusCode; use axum::response::sse::{Event, KeepAlive, Sse}; use tokio_stream::StreamExt; use tokio_stream::wrappers::BroadcastStream; +use temper_authz::AuthenticatedRequestContext; use tracing::instrument; -use crate::authz::{observe_tenant_scope, require_observe_auth}; +use crate::authz::{observe_tenant_scope, require_authenticated_context, require_observe_auth}; use crate::state::ServerState; /// A notification emitted when an entity transitions to a new state. @@ -54,19 +55,17 @@ pub struct EntityStateChange { #[instrument(skip_all, fields(otel.name = "GET /tdata/$events"))] pub async fn handle_events( State(state): State, - headers: HeaderMap, + authenticated: Option>, ) -> Result>>, StatusCode> { - require_observe_auth(&state, &headers, "read_events", "Entity")?; - let tenant_scope = observe_tenant_scope(&state, &headers)?; - let filter_tenant = tenant_scope.map(|t| t.as_str().to_string()); + let authenticated = require_authenticated_context(authenticated.as_deref())?; + require_observe_auth(&state, authenticated, "read_events", "Entity")?; + let filter_tenant = observe_tenant_scope(authenticated).as_str().to_string(); let rx = state.event_tx.subscribe(); let stream = BroadcastStream::new(rx).filter_map(move |result| { match result { Ok(change) => { // Enforce tenant scope: only emit events for the scoped tenant. - if let Some(ref tenant) = filter_tenant - && change.tenant != *tenant - { + if change.tenant != filter_tenant { return None; } let data = serde_json::to_string(&change).unwrap_or_default(); diff --git a/crates/temper-server/src/http_endpoint.rs b/crates/temper-server/src/http_endpoint.rs index 5726b41d8..365429aac 100644 --- a/crates/temper-server/src/http_endpoint.rs +++ b/crates/temper-server/src/http_endpoint.rs @@ -24,6 +24,7 @@ use std::collections::BTreeMap; use std::time::Duration; use temper_runtime::tenant::TenantId; +use tokio::spawn as spawn_http_endpoint_reconciler; // determinism-ok: production HTTP route watcher use tokio::sync::RwLock; const ACTOR_ASK_TIMEOUT: Duration = Duration::from_millis(500); @@ -82,6 +83,38 @@ pub struct MatchedRoute { pub params: BTreeMap, } +/// Immutable route admission produced by the trusted outer authentication +/// edge and consumed by the kernel fallback. +/// +/// Binding the snapshot to tenant, method, and path prevents a public-route +/// decision from being replayed onto another request. The matched route is not +/// re-read after admission, closing Public→Private configuration races. +#[derive(Debug, Clone)] +pub struct AdmittedHttpEndpoint { + tenant: TenantId, + method: String, + path: String, + matched: MatchedRoute, +} + +impl AdmittedHttpEndpoint { + /// Bind a matched endpoint to the exact inbound request. + pub fn new(tenant: TenantId, method: &str, path: &str, matched: MatchedRoute) -> Self { + Self { + tenant, + method: method.to_uppercase(), + path: path.to_string(), + matched, + } + } + + /// Consume the admission only when its request binding still matches. + pub fn into_matched(self, tenant: &TenantId, method: &str, path: &str) -> Option { + (self.tenant == *tenant && self.method == method.to_uppercase() && self.path == path) + .then_some(self.matched) + } +} + /// In-memory route table. One instance per tenant in the kernel. /// Rebuilt on entity-state change events; lookups take an async /// read lock so concurrent requests don't serialize against @@ -176,6 +209,12 @@ impl HttpEndpointTables { .clone() } + /// Look up an existing tenant table without allocating attacker-controlled + /// routing state for an unknown tenant. + pub async fn get(&self, tenant: &TenantId) -> Option> { + self.by_tenant.read().await.get(tenant).cloned() + } + pub async fn tenant_count(&self) -> usize { self.by_tenant.read().await.len() } @@ -282,7 +321,7 @@ pub async fn rebuild_tenant_table(state: &crate::state::ServerState, tenant: &Te /// row count (O(hundreds) per tenant per ADR-0069). pub fn spawn_reconciler(state: crate::state::ServerState) { let mut rx = state.event_tx.subscribe(); - tokio::spawn(async move { + spawn_http_endpoint_reconciler(async move { loop { match rx.recv().await { Ok(change) if change.entity_type == "HttpEndpoint" => { @@ -585,6 +624,52 @@ mod tests { assert_eq!(tables.tenant_count().await, 2); } + #[tokio::test] + async fn admitted_route_is_immutable_across_public_to_private_update() { + let tenant = TenantId::new("tenant-a"); + let table = HttpEndpointTable::new(); + table + .replace(vec![route("public", "/repo.git", &["GET"], "public")]) + .await; + let admitted = AdmittedHttpEndpoint::new( + tenant.clone(), + "GET", + "/repo.git/info/refs", + table + .match_request("GET", "/repo.git/info/refs") + .await + .unwrap(), + ); + + let mut private = route("private", "/repo.git", &["GET"], "private"); + private.requires_auth = true; + table.replace(vec![private]).await; + + let snapshot = admitted + .into_matched(&tenant, "GET", "/repo.git/info/refs") + .expect("exact request binding should consume admission"); + assert_eq!(snapshot.route.id, "public"); + assert!(!snapshot.route.requires_auth); + } + + #[test] + fn admitted_route_rejects_request_binding_changes() { + let tenant = TenantId::new("tenant-a"); + let matched = MatchedRoute { + route: route("public", "/repo.git", &["GET"], "public"), + params: BTreeMap::new(), + }; + for (bound_tenant, method, path) in [ + (TenantId::new("tenant-b"), "GET", "/repo.git"), + (tenant.clone(), "POST", "/repo.git"), + (tenant.clone(), "GET", "/other"), + ] { + let admitted = + AdmittedHttpEndpoint::new(tenant.clone(), "GET", "/repo.git", matched.clone()); + assert!(admitted.into_matched(&bound_tenant, method, path).is_none()); + } + } + #[tokio::test] async fn empty_param_segment_rejects() { let table = HttpEndpointTable::new(); diff --git a/crates/temper-server/src/identity/endpoint.rs b/crates/temper-server/src/identity/endpoint.rs index ec21082f4..682b26f42 100644 --- a/crates/temper-server/src/identity/endpoint.rs +++ b/crates/temper-server/src/identity/endpoint.rs @@ -1,8 +1,8 @@ //! Identity resolution HTTP endpoint. //! //! `POST /api/identity/resolve` — resolves a bearer token to a verified -//! agent identity. Used by the MCP server at startup to resolve its -//! credential once and cache the result for the session lifetime. +//! agent identity. The MCP server uses the result as session metadata, while +//! each protected request still presents and revalidates the credential. use axum::extract::State; use axum::http::{HeaderMap, StatusCode}; @@ -63,12 +63,21 @@ pub async fn handle_identity_resolve( let tenant_str = headers .get("x-tenant-id") .and_then(|v| v.to_str().ok()) - .map(String::from) - .unwrap_or(body.tenant); - let tenant = TenantId::new(&tenant_str); + .unwrap_or(&body.tenant) + .trim(); + let tenant = match TenantId::try_new(tenant_str) { + Ok(tenant) => tenant, + Err(error) => { + return ( + StatusCode::BAD_REQUEST, + axum::Json(serde_json::json!({ "error": error })), + ) + .into_response(); + } + }; - // Use the server-level identity resolver (no cache layer here — the - // MCP server caches the result for the session lifetime). + // Resolve authoritatively. Callers may retain the returned fields as + // metadata, but they do not replace per-request bearer authentication. let resolver = super::IdentityResolver::new(); match resolver.resolve(&state, &tenant, &body.bearer_token).await { Some(identity) => { diff --git a/crates/temper-server/src/identity/mod.rs b/crates/temper-server/src/identity/mod.rs index 57216adeb..25afb2587 100644 --- a/crates/temper-server/src/identity/mod.rs +++ b/crates/temper-server/src/identity/mod.rs @@ -7,4 +7,4 @@ pub mod endpoint; mod resolver; -pub use resolver::{IdentityResolver, ResolvedIdentity, hash_token}; +pub use resolver::{IdentityResolver, MAX_CREDENTIAL_BYTES, ResolvedIdentity, hash_token}; diff --git a/crates/temper-server/src/identity/resolver.rs b/crates/temper-server/src/identity/resolver.rs index 42e79fb36..669d6f902 100644 --- a/crates/temper-server/src/identity/resolver.rs +++ b/crates/temper-server/src/identity/resolver.rs @@ -4,18 +4,17 @@ //! linked `AgentType` is active, and returns a `ResolvedIdentity` that the //! security context uses as the authoritative agent identity. -use std::collections::BTreeMap; -use std::sync::{Arc, RwLock}; - +use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use temper_runtime::scheduler::sim_now; use temper_runtime::tenant::TenantId; +use crate::entity_actor::{EntityState, recover_authoritative_entity_state_from_store}; use crate::state::ServerState; -/// Cache entry TTL in seconds. -const CACHE_TTL_SECS: i64 = 60; +/// Maximum opaque credential size accepted by the identity boundary. +pub const MAX_CREDENTIAL_BYTES: usize = 8 * 1024; /// A platform-resolved agent identity. /// @@ -33,99 +32,105 @@ pub struct ResolvedIdentity { pub verified: bool, } -/// Cached resolution result with expiry. -struct CacheEntry { - identity: ResolvedIdentity, - expires_at: chrono::DateTime, -} - /// Resolves bearer tokens to platform-assigned agent identities. /// -/// Uses an in-memory cache (`BTreeMap` for DST determinism) to avoid -/// entity lookups on every request. Cache entries are scoped by `(tenant, -/// key_hash)` so a credential verified in one tenant never leaks into -/// another. Entries expire after [`CACHE_TTL_SECS`]. -pub struct IdentityResolver { - cache: Arc>>, -} - -impl Default for IdentityResolver { - fn default() -> Self { - Self::new() - } -} +/// Each protected request resolves both the credential and its linked agent +/// type from authoritative state. Persistent deployments replay the complete +/// durable journal with strict validation; in-memory deployments read their +/// sole local actor state. Successful identities are deliberately not cached, +/// so revocation and type deprecation take effect on the next request even +/// when another server replica performed the mutation. +#[derive(Clone, Copy, Debug, Default)] +pub struct IdentityResolver; impl IdentityResolver { /// Create a new identity resolver. pub fn new() -> Self { - Self { - cache: Arc::new(RwLock::new(BTreeMap::new())), - } + Self } /// Resolve a bearer token to a verified agent identity. /// /// 1. Hash the token (SHA-256) - /// 2. Check cache (hit → return immediately) - /// 3. Look up `AgentCredential` entity by using key_hash as entity ID - /// 4. Verify credential is `Active` - /// 5. Look up linked `AgentType` entity - /// 6. Verify AgentType is `Active` - /// 7. Cache and return `ResolvedIdentity` + /// 2. Read `AgentCredential` by using key_hash as entity ID + /// 3. Verify credential is `Active` and unexpired + /// 4. Read the linked `AgentType` + /// 5. Verify AgentType is `Active` + /// 6. Return the verified identity without retaining positive authority pub async fn resolve( &self, state: &ServerState, tenant: &TenantId, bearer_token: &str, ) -> Option { - let key_hash = hash_token(bearer_token); - let cache_key = cache_key(tenant, &key_hash); - - // Check cache first. - if let Some(cached) = self.get_cached(&cache_key) { - return Some(cached); + if bearer_token.is_empty() || bearer_token.len() > MAX_CREDENTIAL_BYTES { + return None; } + let key_hash = hash_token(bearer_token); // Look up AgentCredential entity. We use the key_hash as entity ID // for O(1) lookup — the Issue action must use the key_hash as the // entity ID when creating credentials. - let cred_response = state - .get_tenant_entity_state(tenant, "AgentCredential", &key_hash) - .await - .ok()?; + let credential = + authoritative_entity_state(state, tenant, "AgentCredential", &key_hash).await?; // Verify credential is Active. - if cred_response.state.status != "Active" { + if credential.status != "Active" { return None; } - let fields = &cred_response.state.fields; + let fields = &credential.fields; + let credential_expires_at = match parse_credential_expiry(fields) { + Ok(expires_at) => expires_at, + Err(error) => { + tracing::warn!(tenant = %tenant, %error, "credential has invalid expiration metadata"); + return None; + } + }; + if credential_expires_at.is_some_and(|expires_at| sim_now() >= expires_at) { + return None; + } let agent_type_id = fields.get("agent_type_id")?.as_str()?; let agent_instance_id = fields.get("agent_instance_id")?.as_str()?; + let stored_key_hash = fields.get("key_hash")?.as_str()?; - if agent_type_id.is_empty() || agent_instance_id.is_empty() { + if agent_type_id.is_empty() || agent_instance_id.is_empty() || stored_key_hash != key_hash { return None; } // Look up linked AgentType entity. - let type_response = state - .get_tenant_entity_state(tenant, "AgentType", agent_type_id) - .await - .ok()?; + let agent_type = + authoritative_entity_state(state, tenant, "AgentType", agent_type_id).await?; // Verify AgentType is Active. - if type_response.state.status != "Active" { + if agent_type.status != "Active" { return None; } - let agent_type_name = type_response - .state + let agent_type_name = agent_type .fields .get("name") .and_then(|v| v.as_str()) - .unwrap_or("") + .filter(|name| !name.is_empty())? .to_string(); + // The credential and linked type are separate actors/streams. Re-read + // the credential after observing the type and require an identical + // authority-bearing snapshot. This establishes a point during the type + // read at which both were active; without it, a revocation or link + // change between the two reads could assemble a mixed-time identity + // that never existed. + let credential_recheck = + authoritative_entity_state(state, tenant, "AgentCredential", &key_hash).await?; + if !same_credential_authority(&credential, &credential_recheck) { + tracing::warn!( + tenant = %tenant, + credential = %key_hash, + "credential authority changed during identity resolution" + ); + return None; + } + let identity = ResolvedIdentity { agent_instance_id: agent_instance_id.to_string(), agent_type_id: agent_type_id.to_string(), @@ -133,77 +138,81 @@ impl IdentityResolver { verified: true, }; - // Cache the result. - self.put_cached(cache_key, identity.clone()); + // Re-check after the linked AgentType lookup so a short-lived + // credential cannot cross its expiry while resolution is in flight. + if credential_expires_at.is_some_and(|expires_at| sim_now() >= expires_at) { + return None; + } Some(identity) } +} - /// Invalidate all cached entries (e.g., after credential rotation/revocation). - pub fn invalidate_all(&self) { - let mut cache = self.cache.write().unwrap(); // ci-ok: infallible lock - cache.clear(); - } - - /// Invalidate a specific credential by its token. - pub fn invalidate_token(&self, bearer_token: &str) { - let key_hash = hash_token(bearer_token); - self.invalidate_key_hash(&key_hash); - } - - /// Invalidate a specific credential by its hashed key (`AgentCredential` entity ID). - pub fn invalidate_key_hash(&self, key_hash: &str) { - let mut cache = self.cache.write().unwrap(); // ci-ok: infallible lock - let suffix = format!(":{key_hash}"); - let keys: Vec = cache - .keys() - .filter(|key| key.ends_with(&suffix)) - .cloned() - .collect(); - for key in keys { - cache.remove(&key); - } - } +fn same_credential_authority(first: &EntityState, second: &EntityState) -> bool { + first.entity_type == second.entity_type + && first.entity_id == second.entity_id + && first.sequence_nr == second.sequence_nr + && first.status == second.status + && first.fields == second.fields +} - fn get_cached(&self, cache_key: &str) -> Option { - let cache = self.cache.read().unwrap(); // ci-ok: infallible lock - let entry = cache.get(cache_key)?; - let now = sim_now(); - if now < entry.expires_at { - Some(entry.identity.clone()) - } else { +async fn authoritative_entity_state( + state: &ServerState, + tenant: &TenantId, + entity_type: &str, + entity_id: &str, +) -> Option { + let Some((store, backend)) = state.event_journal() else { + return state + .get_tenant_entity_state(tenant, entity_type, entity_id) + .await + .ok() + .map(|response| response.state); + }; + + let table = state.registry.read().ok()?.get_table(tenant, entity_type)?; + let initial_fields = serde_json::json!({}); + match recover_authoritative_entity_state_from_store( + tenant.as_str(), + entity_type, + entity_id, + table.as_ref(), + &store, + backend, + &initial_fields, + None, + ) + .await + { + Ok(entity) if entity.total_event_count > 0 => Some(entity), + Ok(_) => None, + Err(error) => { + tracing::warn!( + tenant = %tenant, + entity_type, + entity_id, + %error, + "authoritative identity state replay failed closed" + ); None } } - - fn put_cached(&self, cache_key: String, identity: ResolvedIdentity) { - let expires_at = sim_now() + chrono::Duration::seconds(CACHE_TTL_SECS); - let mut cache = self.cache.write().unwrap(); // ci-ok: infallible lock - - // Evict expired entries opportunistically (bounded work: max 32 per insert). - let now = sim_now(); - let expired_keys: Vec = cache - .iter() - .filter(|(_, entry)| now >= entry.expires_at) - .take(32) - .map(|(k, _)| k.clone()) - .collect(); - for k in expired_keys { - cache.remove(&k); - } - - cache.insert( - cache_key, - CacheEntry { - identity, - expires_at, - }, - ); - } } -fn cache_key(tenant: &TenantId, key_hash: &str) -> String { - format!("{}:{key_hash}", tenant.as_str()) +fn parse_credential_expiry(fields: &serde_json::Value) -> Result>, String> { + let Some(value) = fields.get("expires_at") else { + return Ok(None); + }; + let value = value + .as_str() + .ok_or_else(|| "expires_at must be an RFC3339 string".to_string())? + .trim(); + if value.is_empty() { + return Ok(None); + } + DateTime::parse_from_rfc3339(value) + .map(|expires_at| Some(expires_at.with_timezone(&Utc))) + .map_err(|error| format!("expires_at is not valid RFC3339: {error}")) } /// Hash a bearer token with SHA-256 for credential lookup. @@ -232,4 +241,59 @@ mod tests { let h2 = hash_token("token-b"); assert_ne!(h1, h2); } + + #[test] + fn credential_expiry_is_optional_but_malformed_values_fail_closed() { + assert_eq!( + parse_credential_expiry(&serde_json::json!({"expires_at": ""})), + Ok(None) + ); + assert_eq!(parse_credential_expiry(&serde_json::json!({})), Ok(None)); + assert!(parse_credential_expiry(&serde_json::json!({"expires_at": "tomorrow"})).is_err()); + assert!(parse_credential_expiry(&serde_json::json!({"expires_at": 42})).is_err()); + assert_eq!( + parse_credential_expiry(&serde_json::json!({ + "expires_at": "2030-01-02T03:04:05+02:00" + })) + .expect("valid RFC3339 expiry") + .expect("expiry present") + .to_rfc3339(), + "2030-01-02T01:04:05+00:00" + ); + } + + #[test] + fn credential_stability_check_binds_sequence_status_and_fields() { + let state = |sequence_nr, status: &str, fields: serde_json::Value| EntityState { + entity_type: "AgentCredential".to_string(), + entity_id: "hash".to_string(), + status: status.to_string(), + item_count: 0, + counters: Default::default(), + booleans: Default::default(), + lists: Default::default(), + fields, + events: Default::default(), + total_event_count: 0, + events_since_snapshot: 0, + last_snapshot_sequence_nr: 0, + sequence_nr, + processed_idempotency_keys: Default::default(), + }; + let first = state(3, "Active", serde_json::json!({"agent_type_id": "type-a"})); + + assert!(same_credential_authority(&first, &first.clone())); + assert!(!same_credential_authority( + &first, + &state(4, "Active", first.fields.clone()) + )); + assert!(!same_credential_authority( + &first, + &state(3, "Revoked", first.fields.clone()) + )); + assert!(!same_credential_authority( + &first, + &state(3, "Active", serde_json::json!({"agent_type_id": "type-b"})) + )); + } } diff --git a/crates/temper-server/src/internal_invocation.rs b/crates/temper-server/src/internal_invocation.rs new file mode 100644 index 000000000..ec5aa98d7 --- /dev/null +++ b/crates/temper-server/src/internal_invocation.rs @@ -0,0 +1,342 @@ +//! Short-lived credentials for authenticated internal HTTP re-entry. +//! +//! A credential is an opaque, single-use capability. The store retains only a +//! SHA-256 digest and binds the capability to one tenant, HTTP method, exact +//! canonical path/query, and immutable authenticated request context. + +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use axum::http::{Method, Uri}; +use chrono::{DateTime, Utc}; +use sha2::{Digest, Sha256}; +use temper_authz::{AuthenticatedRequestContext, PrincipalKind}; +use temper_runtime::scheduler::{sim_now, sim_uuid}; +use temper_runtime::tenant::TenantId; + +/// Prefix reserved for internal invocation bearer credentials. +pub const INTERNAL_INVOCATION_BEARER_PREFIX: &str = "temper-internal-v1."; + +/// Maximum number of outstanding internal invocation credentials. +pub const INTERNAL_INVOCATION_CREDENTIAL_CAPACITY: usize = 4_096; + +/// Per-tenant share of the global credential budget. +pub const INTERNAL_INVOCATION_CREDENTIAL_TENANT_CAPACITY: usize = 256; + +/// Lifetime of an internal invocation credential. +pub const INTERNAL_INVOCATION_CREDENTIAL_TTL: Duration = Duration::from_secs(30); + +const TOKEN_GENERATION_ATTEMPT_BUDGET: usize = 8; + +/// Injected clock used by the credential store. +pub type InternalInvocationNowFn = Arc DateTime + Send + Sync>; + +/// Injected source of 32 bytes of opaque token material. +pub type InternalInvocationTokenFn = Arc [u8; 32] + Send + Sync>; + +/// Errors returned while issuing or consuming an internal credential. +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +pub enum InternalInvocationCredentialError { + /// The supplied URL or HTTP method cannot be canonicalized. + #[error("invalid internal invocation request target")] + InvalidRequestTarget, + /// The token is unknown, expired, malformed, replayed, or has the wrong prefix. + #[error("invalid internal invocation credential")] + InvalidCredential, + /// The credential exists but is not bound to this request. + #[error("internal invocation credential binding mismatch")] + BindingMismatch, + /// Kernel System authority cannot cross an HTTP bearer boundary. + #[error("System authority cannot be delegated through internal HTTP")] + SystemContextNotDelegable, + /// The injected token source repeatedly generated an existing credential. + #[error("internal invocation token generation budget exhausted")] + TokenGenerationExhausted, + /// No capacity remains that can be reclaimed from the issuing tenant. + #[error("internal invocation credential capacity exhausted")] + CapacityExhausted, + /// The credential store lock was poisoned. + #[error("internal invocation credential store unavailable")] + StoreUnavailable, +} + +#[derive(Clone)] +struct CredentialEntry { + context: AuthenticatedRequestContext, + tenant: TenantId, + method: String, + target: String, + expires_at: DateTime, + issue_sequence: u64, +} + +#[derive(Default)] +struct CredentialState { + entries: BTreeMap<[u8; 32], CredentialEntry>, + next_issue_sequence: u64, +} + +/// Bounded store for short-lived, single-use internal invocation credentials. +#[derive(Clone)] +pub struct InternalInvocationCredentialStore { + state: Arc>, + capacity: usize, + tenant_capacity: usize, + ttl: Duration, + now: InternalInvocationNowFn, + token: InternalInvocationTokenFn, +} + +impl InternalInvocationCredentialStore { + /// Create the runtime store. + /// + /// The default simulation context supplies deterministic time and IDs. + /// Production's `sim_uuid()` source is UUIDv7-backed; two independent UUIDs + /// provide more than 128 random bits while still allowing deterministic + /// source injection in DST. + pub fn runtime() -> Self { + Self::with_limits_and_sources( + INTERNAL_INVOCATION_CREDENTIAL_CAPACITY, + INTERNAL_INVOCATION_CREDENTIAL_TENANT_CAPACITY, + INTERNAL_INVOCATION_CREDENTIAL_TTL, + Arc::new(sim_now), + Arc::new(|| { + let first = sim_uuid(); + let second = sim_uuid(); + let mut bytes = [0_u8; 32]; + bytes[..16].copy_from_slice(first.as_bytes()); + bytes[16..].copy_from_slice(second.as_bytes()); + bytes + }), + ) + } + + /// Create a store with explicit bounds, clock, and token source. + pub fn with_sources( + capacity: usize, + ttl: Duration, + now: InternalInvocationNowFn, + token: InternalInvocationTokenFn, + ) -> Self { + Self::with_limits_and_sources(capacity, capacity, ttl, now, token) + } + + fn with_limits_and_sources( + capacity: usize, + tenant_capacity: usize, + ttl: Duration, + now: InternalInvocationNowFn, + token: InternalInvocationTokenFn, + ) -> Self { + assert!(capacity > 0, "credential capacity must be positive"); + assert!(tenant_capacity > 0, "tenant capacity must be positive"); + assert!( + tenant_capacity <= capacity, + "tenant capacity must not exceed global capacity" + ); + assert!(!ttl.is_zero(), "credential TTL must be positive"); + Self { + state: Arc::new(Mutex::new(CredentialState::default())), + capacity, + tenant_capacity, + ttl, + now, + token, + } + } + + /// Issue one credential bound to an absolute request URL. + pub fn issue_for_url( + &self, + context: AuthenticatedRequestContext, + method: &str, + url: &str, + ) -> Result { + if context.security_context().principal.kind == PrincipalKind::System { + return Err(InternalInvocationCredentialError::SystemContextNotDelegable); + } + let method = canonical_method(method)?; + let target = canonical_request_target_from_url(url)?; + let tenant = context.tenant().clone(); + let now = (self.now)(); + let expires_at = now + + chrono::Duration::from_std(self.ttl) + .map_err(|_| InternalInvocationCredentialError::InvalidRequestTarget)?; + let mut state = self + .state + .lock() + .map_err(|_| InternalInvocationCredentialError::StoreUnavailable)?; + + purge_expired(&mut state, now); + reclaim_issuing_tenant_slot(&mut state, &tenant, self.capacity, self.tenant_capacity); + if state.entries.len() >= self.capacity { + return Err(InternalInvocationCredentialError::CapacityExhausted); + } + let generated = (0..TOKEN_GENERATION_ATTEMPT_BUDGET).find_map(|_| { + let token = encode_token((self.token)()); + let digest = credential_digest(&token); + if state.entries.contains_key(&digest) { + None + } else { + Some((token, digest)) + } + }); + let Some((token, digest)) = generated else { + return Err(InternalInvocationCredentialError::TokenGenerationExhausted); + }; + + let issue_sequence = state.next_issue_sequence; + state.next_issue_sequence = state.next_issue_sequence.wrapping_add(1); + state.entries.insert( + digest, + CredentialEntry { + context, + tenant, + method, + target, + expires_at, + issue_sequence, + }, + ); + debug_assert!(state.entries.len() <= self.capacity); + Ok(token) + } + + /// Consume a credential for the exact inbound request. + /// + /// Removal happens before binding validation, so every credential can be + /// presented at most once even when the presenting request is malformed. + pub fn consume_for_request( + &self, + token: &str, + tenant: &TenantId, + method: &Method, + uri: &Uri, + ) -> Result { + if !is_internal_invocation_bearer(token) { + return Err(InternalInvocationCredentialError::InvalidCredential); + } + let digest = credential_digest(token); + let now = (self.now)(); + let mut state = self + .state + .lock() + .map_err(|_| InternalInvocationCredentialError::StoreUnavailable)?; + let Some(entry) = state.entries.remove(&digest) else { + return Err(InternalInvocationCredentialError::InvalidCredential); + }; + + if now >= entry.expires_at { + return Err(InternalInvocationCredentialError::InvalidCredential); + } + if entry.context.security_context().principal.kind == PrincipalKind::System { + return Err(InternalInvocationCredentialError::SystemContextNotDelegable); + } + let target = canonical_request_target_from_uri(uri); + if entry.tenant != *tenant + || entry.context.tenant() != tenant + || entry.method != method.as_str() + || entry.target != target + { + return Err(InternalInvocationCredentialError::BindingMismatch); + } + + Ok(entry.context) + } + + #[cfg(test)] + fn len(&self) -> Result { + self.state + .lock() + .map(|state| state.entries.len()) + .map_err(|_| InternalInvocationCredentialError::StoreUnavailable) + } +} + +/// Return whether a bearer value uses the reserved internal prefix. +pub fn is_internal_invocation_bearer(token: &str) -> bool { + token.starts_with(INTERNAL_INVOCATION_BEARER_PREFIX) +} + +/// Canonicalize the path and query reqwest will send for an absolute URL. +pub fn canonical_request_target_from_url( + url: &str, +) -> Result { + let url = reqwest::Url::parse(url) + .map_err(|_| InternalInvocationCredentialError::InvalidRequestTarget)?; + let mut target = if url.path().is_empty() { + "/".to_string() + } else { + url.path().to_string() + }; + if let Some(query) = url.query() { + target.push('?'); + target.push_str(query); + } + Ok(target) +} + +/// Return the canonical path/query seen by the inbound HTTP router. +pub fn canonical_request_target_from_uri(uri: &Uri) -> String { + uri.path_and_query() + .map(|value| value.as_str().to_string()) + .unwrap_or_else(|| uri.path().to_string()) +} + +fn canonical_method(method: &str) -> Result { + Method::from_bytes(method.as_bytes()) + .map(|method| method.as_str().to_string()) + .map_err(|_| InternalInvocationCredentialError::InvalidRequestTarget) +} + +fn encode_token(bytes: [u8; 32]) -> String { + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut opaque_hasher = Sha256::new(); + opaque_hasher.update(b"temper-internal-invocation-token-v1\0"); + opaque_hasher.update(bytes); + let opaque: [u8; 32] = opaque_hasher.finalize().into(); + let mut token = String::with_capacity(INTERNAL_INVOCATION_BEARER_PREFIX.len() + 64); + token.push_str(INTERNAL_INVOCATION_BEARER_PREFIX); + for byte in opaque { + token.push(HEX[(byte >> 4) as usize] as char); + token.push(HEX[(byte & 0x0f) as usize] as char); + } + token +} + +fn credential_digest(token: &str) -> [u8; 32] { + Sha256::digest(token.as_bytes()).into() +} + +fn purge_expired(state: &mut CredentialState, now: DateTime) { + state.entries.retain(|_, entry| entry.expires_at > now); +} + +fn reclaim_issuing_tenant_slot( + state: &mut CredentialState, + tenant: &TenantId, + global_capacity: usize, + tenant_capacity: usize, +) { + let tenant_entries = state + .entries + .values() + .filter(|entry| &entry.tenant == tenant) + .count(); + if tenant_entries < tenant_capacity && state.entries.len() < global_capacity { + return; + } + let oldest = state + .entries + .iter() + .filter(|(_, entry)| &entry.tenant == tenant) + .min_by_key(|(digest, entry)| (entry.expires_at, entry.issue_sequence, **digest)) + .map(|(digest, _)| *digest); + if let Some(oldest) = oldest { + state.entries.remove(&oldest); + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/temper-server/src/internal_invocation/tests.rs b/crates/temper-server/src/internal_invocation/tests.rs new file mode 100644 index 000000000..7a7f80eb0 --- /dev/null +++ b/crates/temper-server/src/internal_invocation/tests.rs @@ -0,0 +1,245 @@ +use super::*; +use std::sync::atomic::{AtomicI64, AtomicU64, Ordering}; +use temper_authz::SecurityContext; + +fn test_store(capacity: usize) -> (InternalInvocationCredentialStore, Arc) { + let millis = Arc::new(AtomicI64::new(1_700_000_000_000)); + let now_millis = Arc::clone(&millis); + let counter = Arc::new(AtomicU64::new(1)); + let token_counter = Arc::clone(&counter); + let store = InternalInvocationCredentialStore::with_sources( + capacity, + Duration::from_secs(30), + Arc::new(move || { + DateTime::from_timestamp_millis(now_millis.load(Ordering::SeqCst)) + .expect("test timestamp must be valid") + }), + Arc::new(move || { + let value = token_counter.fetch_add(1, Ordering::SeqCst); + let mut bytes = [0_u8; 32]; + bytes[..8].copy_from_slice(&value.to_be_bytes()); + bytes + }), + ); + (store, millis) +} + +fn context(tenant: &str, principal: &str) -> AuthenticatedRequestContext { + AuthenticatedRequestContext::new( + TenantId::new(tenant), + SecurityContext::from_resolved_identity(principal, "worker", None), + ) +} + +fn request(method: Method, target: &str) -> (Method, Uri) { + (method, target.parse().expect("test URI must parse")) +} + +#[test] +fn valid_credential_returns_exact_context_once() { + let (store, _) = test_store(8); + let mut security_context = + SecurityContext::from_resolved_identity("agent-1", "worker", Some("session-1")); + security_context.principal.role = Some("planner".to_string()); + security_context + .context_attrs + .insert("approvalLimit".to_string(), serde_json::json!(1_000)); + let token = store + .issue_for_url( + AuthenticatedRequestContext::new(TenantId::new("tenant-a"), security_context), + "POST", + "http://127.0.0.1:3000/tdata/Orders?mode=full", + ) + .expect("credential should issue"); + let (method, uri) = request(Method::POST, "/tdata/Orders?mode=full"); + + let resolved = store + .consume_for_request(&token, &TenantId::new("tenant-a"), &method, &uri) + .expect("credential should resolve"); + assert_eq!(resolved.tenant().as_str(), "tenant-a"); + assert_eq!(resolved.security_context().principal.id, "agent-1"); + assert_eq!( + resolved.security_context().principal.role.as_deref(), + Some("planner") + ); + assert_eq!( + resolved + .security_context() + .context_attrs + .get("approvalLimit"), + Some(&serde_json::json!(1_000)) + ); + assert_eq!(store.len(), Ok(0)); + assert!(matches!( + store.consume_for_request(&token, &TenantId::new("tenant-a"), &method, &uri), + Err(InternalInvocationCredentialError::InvalidCredential) + )); +} + +#[test] +fn wrong_tenant_method_and_path_each_fail_and_consume() { + let (store, _) = test_store(8); + let cases = [ + ("tenant-b", Method::GET, "/tdata/Orders"), + ("tenant-a", Method::POST, "/tdata/Orders"), + ("tenant-a", Method::GET, "/tdata/Other"), + ("tenant-a", Method::GET, "/tdata/Orders?extra=1"), + ]; + + for (tenant, method, target) in cases { + let token = store + .issue_for_url( + context("tenant-a", "agent-1"), + "GET", + "http://127.0.0.1:3000/tdata/Orders", + ) + .expect("credential should issue"); + let uri = target.parse().expect("test URI must parse"); + assert!(matches!( + store.consume_for_request(&token, &TenantId::new(tenant), &method, &uri), + Err(InternalInvocationCredentialError::BindingMismatch) + )); + assert!(matches!( + store.consume_for_request( + &token, + &TenantId::new("tenant-a"), + &Method::GET, + &"/tdata/Orders".parse().expect("test URI must parse"), + ), + Err(InternalInvocationCredentialError::InvalidCredential) + )); + } +} + +#[test] +fn expired_credential_fails_closed() { + let (store, millis) = test_store(8); + let token = store + .issue_for_url(context("tenant-a", "agent-1"), "GET", "http://local/tdata") + .expect("credential should issue"); + millis.fetch_add(30_000, Ordering::SeqCst); + assert!(matches!( + store.consume_for_request( + &token, + &TenantId::new("tenant-a"), + &Method::GET, + &"/tdata".parse().expect("test URI must parse"), + ), + Err(InternalInvocationCredentialError::InvalidCredential) + )); +} + +#[test] +fn system_context_is_refused_at_issuance_and_consumption() { + let (store, _) = test_store(8); + let system = + AuthenticatedRequestContext::new(TenantId::new("tenant-a"), SecurityContext::system()); + assert!(matches!( + store.issue_for_url(system.clone(), "GET", "http://local/tdata/Orders"), + Err(InternalInvocationCredentialError::SystemContextNotDelegable) + )); + + // Defense in depth: even a legacy/injected entry cannot reconstitute + // System authority at the bearer edge. + let token = encode_token([42_u8; 32]); + let digest = credential_digest(&token); + let now = (store.now)(); + store + .state + .lock() + .expect("test store lock must be available") + .entries + .insert( + digest, + CredentialEntry { + context: system, + tenant: TenantId::new("tenant-a"), + method: "GET".to_string(), + target: "/tdata/Orders".to_string(), + expires_at: now + chrono::Duration::seconds(30), + issue_sequence: 0, + }, + ); + assert!(matches!( + store.consume_for_request( + &token, + &TenantId::new("tenant-a"), + &Method::GET, + &"/tdata/Orders".parse().expect("test URI must parse"), + ), + Err(InternalInvocationCredentialError::SystemContextNotDelegable) + )); + assert_eq!(store.len(), Ok(0)); +} + +#[test] +fn capacity_evicts_oldest_credential_deterministically() { + let (store, _) = test_store(2); + let first = store + .issue_for_url(context("tenant-a", "one"), "GET", "http://local/one") + .expect("first credential should issue"); + let second = store + .issue_for_url(context("tenant-a", "two"), "GET", "http://local/two") + .expect("second credential should issue"); + let third = store + .issue_for_url(context("tenant-a", "three"), "GET", "http://local/three") + .expect("third credential should issue"); + assert_eq!(store.len(), Ok(2)); + + assert!(matches!( + store.consume_for_request( + &first, + &TenantId::new("tenant-a"), + &Method::GET, + &"/one".parse().expect("test URI must parse"), + ), + Err(InternalInvocationCredentialError::InvalidCredential) + )); + for (token, target, principal) in [(second, "/two", "two"), (third, "/three", "three")] { + let resolved = store + .consume_for_request( + &token, + &TenantId::new("tenant-a"), + &Method::GET, + &target.parse().expect("test URI must parse"), + ) + .expect("retained credential should resolve"); + assert_eq!(resolved.security_context().principal.id, principal); + } +} + +#[test] +fn full_store_never_evicts_another_tenants_credentials() { + let (store, _) = test_store(2); + let first = store + .issue_for_url(context("tenant-a", "one"), "GET", "http://local/one") + .expect("first credential should issue"); + let second = store + .issue_for_url(context("tenant-a", "two"), "GET", "http://local/two") + .expect("second credential should issue"); + + assert_eq!( + store.issue_for_url(context("tenant-b", "other"), "GET", "http://local/other"), + Err(InternalInvocationCredentialError::CapacityExhausted) + ); + for (token, target, principal) in [(first, "/one", "one"), (second, "/two", "two")] { + let resolved = store + .consume_for_request( + &token, + &TenantId::new("tenant-a"), + &Method::GET, + &target.parse().expect("test URI must parse"), + ) + .expect("another tenant must not evict this credential"); + assert_eq!(resolved.security_context().principal.id, principal); + } +} + +#[test] +fn canonical_target_uses_only_normalized_path_and_exact_query() { + assert_eq!( + canonical_request_target_from_url("http://LOCAL:80/a/../b?q=1%202&x=") + .expect("URL should canonicalize"), + "/b?q=1%202&x=" + ); +} diff --git a/crates/temper-server/src/lib.rs b/crates/temper-server/src/lib.rs index 1010e8e1f..e89e1561b 100644 --- a/crates/temper-server/src/lib.rs +++ b/crates/temper-server/src/lib.rs @@ -14,6 +14,8 @@ pub mod blob_store; pub mod blob_sweeper; mod blob_transport_observability; pub mod blobs; +/// Deterministic conformance checking of a recorded run against its IOA spec. +pub mod conformance; pub mod entity_actor; pub mod event_budget_metrics; pub mod events; @@ -21,6 +23,7 @@ pub mod eventual_invariants; pub mod http_endpoint; pub mod idempotency; pub mod identity; +pub mod internal_invocation; /// ADR-0153: declared composite-key index hashing (the negative-existence access path). pub mod key_index; #[cfg(feature = "observe")] diff --git a/crates/temper-server/src/observe/agents.rs b/crates/temper-server/src/observe/agents.rs index 79c624504..f970312ff 100644 --- a/crates/temper-server/src/observe/agents.rs +++ b/crates/temper-server/src/observe/agents.rs @@ -3,12 +3,15 @@ //! Provides per-agent action history and summary statistics derived from the //! configured durable metadata backend. -use axum::extract::{Path, Query, State}; -use axum::http::{HeaderMap, StatusCode}; +use axum::extract::{Extension, Path, Query, State}; +use axum::http::StatusCode; use axum::response::Json; use serde::{Deserialize, Serialize}; +use temper_authz::AuthenticatedRequestContext; -use crate::authz::{observe_tenant_scope, require_observe_auth}; +use crate::authz::{ + observe_tenant_scope, require_authenticated_context, require_observe_auth, require_tenant_match, +}; use crate::state::ServerState; /// Summary of a single agent's activity. @@ -82,28 +85,25 @@ pub struct AgentHistoryParams { /// GET /observe/agents -- list agents with action/denial counts. pub(crate) async fn handle_list_agents( State(state): State, - headers: HeaderMap, + authenticated: Option>, Query(params): Query, ) -> Result, StatusCode> { - require_observe_auth(&state, &headers, "read_agents", "AgentAudit")?; - let tenant_scope = observe_tenant_scope(&state, &headers)?; - let tenant_filter = tenant_scope - .as_ref() - .map(|t| t.as_str().to_string()) - .or(params.tenant); - let stores = if let Some(ref tf) = tenant_filter { - match state.metadata_store_for_tenant(tf).await { - Some(store) => vec![store], - None => Vec::new(), - } - } else { - state.collect_all_metadata_stores().await - }; + let authenticated = require_authenticated_context(authenticated.as_deref())?; + require_observe_auth(&state, authenticated, "read_agents", "AgentAudit")?; + if let Some(requested_tenant) = params.tenant.as_deref() { + require_tenant_match(authenticated, requested_tenant)?; + } + let tenant_filter = observe_tenant_scope(authenticated).as_str().to_string(); + let stores = state + .metadata_store_for_tenant(&tenant_filter) + .await + .into_iter() + .collect::>(); if !stores.is_empty() { let mut all_agents: Vec = Vec::new(); for store in &stores { - match store.query_agent_summaries(tenant_filter.as_deref()).await { + match store.query_agent_summaries(Some(&tenant_filter)).await { Ok(summaries) => { all_agents.extend(summaries.into_iter().map(|s| AgentSummary { agent_id: s.agent_id, @@ -173,33 +173,30 @@ pub(crate) async fn handle_list_agents( /// GET /observe/agents/{agent_id}/history -- full action timeline for one agent. pub(crate) async fn handle_get_agent_history( State(state): State, - headers: HeaderMap, + authenticated: Option>, Path(agent_id): Path, Query(params): Query, ) -> Result, StatusCode> { - require_observe_auth(&state, &headers, "read_agents", "AgentAudit")?; - let tenant_scope = observe_tenant_scope(&state, &headers)?; - let tenant_filter = tenant_scope - .as_ref() - .map(|t| t.as_str().to_string()) - .or(params.tenant); + let authenticated = require_authenticated_context(authenticated.as_deref())?; + require_observe_auth(&state, authenticated, "read_agents", "AgentAudit")?; + if let Some(requested_tenant) = params.tenant.as_deref() { + require_tenant_match(authenticated, requested_tenant)?; + } + let tenant_filter = observe_tenant_scope(authenticated).as_str().to_string(); let limit = params.limit.unwrap_or(100).min(500); - let stores = if let Some(ref tf) = tenant_filter { - match state.metadata_store_for_tenant(tf).await { - Some(store) => vec![store], - None => Vec::new(), - } - } else { - state.collect_all_metadata_stores().await - }; + let stores = state + .metadata_store_for_tenant(&tenant_filter) + .await + .into_iter() + .collect::>(); let mut all_history: Vec = Vec::new(); for store in &stores { match store .query_trajectories_by_agent( &agent_id, - tenant_filter.as_deref(), + Some(&tenant_filter), params.entity_type.as_deref(), limit as i64, ) diff --git a/crates/temper-server/src/observe/entities.rs b/crates/temper-server/src/observe/entities.rs index f0b8a4862..3c98cd4bf 100644 --- a/crates/temper-server/src/observe/entities.rs +++ b/crates/temper-server/src/observe/entities.rs @@ -3,19 +3,19 @@ use std::convert::Infallible; use std::time::Duration; -use axum::extract::{Path, Query, State}; +use axum::extract::{Extension, Path, Query, State}; use axum::http::{HeaderMap, StatusCode}; use axum::response::Json; use axum::response::sse::{Event, KeepAlive, Sse}; use serde::Deserialize; +use temper_authz::AuthenticatedRequestContext; use tokio_stream::StreamExt; use tokio_stream::wrappers::BroadcastStream; use tracing::instrument; -use crate::authz::{observe_tenant_scope, require_observe_auth}; +use crate::authz::{observe_tenant_scope, require_authenticated_context, require_observe_auth}; use crate::blobs::hydrate_blob_refs_for_tenant; use crate::entity_actor::{EntityEvent, EntityMsg, EntityResponse}; -use crate::odata::extract_tenant; use crate::state::ServerState; use super::{EntityInstanceSummary, EventStreamParams}; @@ -25,10 +25,11 @@ use super::{EntityInstanceSummary, EventStreamParams}; /// Returns deduplicated entities with their current state, sorted newest first. pub(crate) async fn handle_list_entities( State(state): State, - headers: HeaderMap, + authenticated: Option>, ) -> Result, StatusCode> { - require_observe_auth(&state, &headers, "read_entities", "Entity")?; - let tenant_scope = observe_tenant_scope(&state, &headers)?; + let authenticated = require_authenticated_context(authenticated.as_deref())?; + require_observe_auth(&state, authenticated, "read_entities", "Entity")?; + let tenant_scope = observe_tenant_scope(authenticated); let registry = state.actor_registry.read().unwrap(); // ci-ok: infallible lock let cache = state.entity_state_cache.lock().unwrap(); // ci-ok: infallible lock let mut entities: Vec = registry @@ -36,9 +37,7 @@ pub(crate) async fn handle_list_entities( .filter_map(|key| { // Actor keys are formatted as "{tenant}:{entity_type}:{entity_id}" let parts: Vec<&str> = key.splitn(3, ':').collect(); - if let Some(ref scope) = tenant_scope - && parts.first() != Some(&scope.as_str()) - { + if parts.first() != Some(&tenant_scope.as_str()) { return None; } // Use peek() to avoid updating LRU order during a bulk listing. @@ -71,11 +70,12 @@ pub(crate) async fn handle_list_entities( /// 2. Postgres event store (if configured, for inactive entities). pub(crate) async fn handle_get_entity_history( State(state): State, - headers: HeaderMap, + authenticated: Option>, Path((entity_type, entity_id)): Path<(String, String)>, ) -> Result, StatusCode> { - require_observe_auth(&state, &headers, "read", &entity_type)?; - let tenant = extract_tenant(&headers, &state).map_err(|(code, _)| code)?; + let authenticated = require_authenticated_context(authenticated.as_deref())?; + require_observe_auth(&state, authenticated, "read", &entity_type)?; + let tenant = authenticated.tenant().clone(); // Path 1: If the actor is loaded, read events from in-memory state. let actor_key = format!("{tenant}:{entity_type}:{entity_id}"); @@ -179,12 +179,13 @@ pub(crate) struct EntityEventStreamParams { )] pub(crate) async fn handle_wait_for_entity_state( State(state): State, - headers: HeaderMap, + authenticated: Option>, Path((entity_type, entity_id)): Path<(String, String)>, Query(params): Query, ) -> Result, StatusCode> { - require_observe_auth(&state, &headers, "read", &entity_type)?; - let tenant = extract_tenant(&headers, &state).map_err(|(code, _)| code)?; + let authenticated = require_authenticated_context(authenticated.as_deref())?; + require_observe_auth(&state, authenticated, "read", &entity_type)?; + let tenant = authenticated.tenant().clone(); record_wait_span_identity(&tenant, &entity_type, &entity_id); let target_statuses: std::collections::BTreeSet = params @@ -303,12 +304,14 @@ async fn wait_entity_response( /// GET /observe/entities/{entity_type}/{entity_id}/events -- replayable SSE stream for one entity. pub(crate) async fn handle_entity_event_stream( State(state): State, + authenticated: Option>, headers: HeaderMap, Path((entity_type, entity_id)): Path<(String, String)>, Query(params): Query, ) -> Result>>, StatusCode> { - require_observe_auth(&state, &headers, "read", &entity_type)?; - let tenant = extract_tenant(&headers, &state).map_err(|(code, _)| code)?; + let authenticated = require_authenticated_context(authenticated.as_deref())?; + require_observe_auth(&state, authenticated, "read", &entity_type)?; + let tenant = authenticated.tenant().clone(); let since = headers .get("last-event-id") .and_then(|value| value.to_str().ok()) @@ -393,23 +396,21 @@ fn format_history_response( /// as a JSON SSE event. Supports optional `?entity_type=X&entity_id=Y` filters. pub(crate) async fn handle_event_stream( State(state): State, - headers: HeaderMap, + authenticated: Option>, Query(params): Query, ) -> Result>>, StatusCode> { - require_observe_auth(&state, &headers, "read_events", "Entity")?; - let tenant_scope = observe_tenant_scope(&state, &headers)?; + let authenticated = require_authenticated_context(authenticated.as_deref())?; + require_observe_auth(&state, authenticated, "read_events", "Entity")?; + let filter_tenant = observe_tenant_scope(authenticated).as_str().to_string(); let rx = state.event_tx.subscribe(); let filter_type = params.entity_type; let filter_id = params.entity_id; - let filter_tenant = tenant_scope.map(|t| t.as_str().to_string()); let stream = BroadcastStream::new(rx).filter_map(move |result| { match result { Ok(change) => { // Apply tenant filter. - if let Some(ref ft) = filter_tenant - && change.tenant != *ft - { + if change.tenant != filter_tenant { return None; } // Apply entity type/id filters. diff --git a/crates/temper-server/src/observe/evolution.rs b/crates/temper-server/src/observe/evolution.rs index 5342d9cd0..a791664fd 100644 --- a/crates/temper-server/src/observe/evolution.rs +++ b/crates/temper-server/src/observe/evolution.rs @@ -6,6 +6,9 @@ mod records_detail; mod records_list; mod trajectories; +#[cfg(test)] +mod tenant_test; + pub(crate) use operations::{ handle_evolution_analyze, handle_evolution_materialize, handle_evolution_stream, handle_feature_requests, handle_intent_evidence, handle_sentinel_check, handle_unmet_intents, diff --git a/crates/temper-server/src/observe/evolution/insight_generator/mod_test.rs b/crates/temper-server/src/observe/evolution/insight_generator/mod_test.rs index f7c98eb65..2cc98c90c 100644 --- a/crates/temper-server/src/observe/evolution/insight_generator/mod_test.rs +++ b/crates/temper-server/src/observe/evolution/insight_generator/mod_test.rs @@ -23,6 +23,7 @@ fn entry(entity_type: &str, action: &str, success: bool) -> TrajectoryEntry { request_body: None, intent: None, matched_policy_ids: None, + capture_seq: None, } } diff --git a/crates/temper-server/src/observe/evolution/operations.rs b/crates/temper-server/src/observe/evolution/operations.rs index b6e06a4ed..60ecf0fc2 100644 --- a/crates/temper-server/src/observe/evolution/operations.rs +++ b/crates/temper-server/src/observe/evolution/operations.rs @@ -1,19 +1,18 @@ use std::collections::BTreeMap; use std::convert::Infallible; -use axum::extract::{Path, Query, State}; -use axum::http::{HeaderMap, StatusCode}; +use axum::extract::{Extension, Path, Query, State}; +use axum::http::StatusCode; use axum::response::Json; use axum::response::sse::{Event, KeepAlive, Sse}; +use temper_authz::AuthenticatedRequestContext; use temper_evolution::FeatureRequestDisposition; -use temper_runtime::tenant::TenantId; use tokio_stream::StreamExt; use tokio_stream::wrappers::BroadcastStream; use tracing::instrument; use super::insight_generator; -use crate::authz::require_observe_auth; -use crate::odata::extract_tenant; +use crate::authz::{require_authenticated_context, require_observe_auth}; use crate::request_context::AgentContext; use crate::sentinel; use crate::state::{ObserveRefreshHint, ServerState}; @@ -41,10 +40,12 @@ use support::{ ))] pub(crate) async fn handle_sentinel_check( State(state): State, - headers: HeaderMap, + authenticated: Option>, ) -> Result, StatusCode> { - require_observe_auth(&state, &headers, "run_sentinel", "Evolution")?; - let trajectory_entries = state.load_trajectory_entries(1_000).await; + let authenticated = require_authenticated_context(authenticated.as_deref())?; + require_observe_auth(&state, authenticated, "run_sentinel", "Evolution")?; + let tenant = authenticated.tenant().as_str(); + let trajectory_entries = state.load_trajectory_entries(tenant, 1_000).await; tracing::Span::current().record("trajectory_count", trajectory_entries.len()); tracing::info!( trajectory_count = trajectory_entries.len(), @@ -63,10 +64,9 @@ pub(crate) async fn handle_sentinel_check( "evolution.sentinel" ); } - let results = persist_alerts(&state, &alerts).await?; + let results = persist_alerts(&state, tenant, &alerts).await?; - let analysis_tenant = - extract_tenant(&headers, &state).unwrap_or_else(|_| TenantId::new("temper-system")); + let analysis_tenant = authenticated.tenant().clone(); let mut discovery_results = Vec::new(); let system_ctx = AgentContext::for_service("evolution-engine"); for alert in &alerts { @@ -105,7 +105,7 @@ pub(crate) async fn handle_sentinel_check( let insights = insight_generator::generate_insights(&trajectory_entries); tracing::Span::current().record("insights_count", insights.len()); tracing::info!(insights_count = insights.len(), "evolution.insight"); - let insight_results = persist_insights(&state, &insights).await; + let insight_results = persist_insights(&state, tenant, &insights).await?; emit_refresh_hints( &state, @@ -139,10 +139,13 @@ pub(crate) async fn handle_sentinel_check( ))] pub(crate) async fn handle_unmet_intents( State(state): State, - headers: HeaderMap, + authenticated: Option>, ) -> Result, StatusCode> { - require_observe_auth(&state, &headers, "read_evolution", "Evolution")?; - let (failure_rows, submitted_specs) = state.load_unmet_intent_rows_aggregated().await; + let authenticated = require_authenticated_context(authenticated.as_deref())?; + require_observe_auth(&state, authenticated, "read_evolution", "Evolution")?; + let (failure_rows, submitted_specs) = state + .load_unmet_intent_rows_aggregated(authenticated.tenant().as_str()) + .await; let intents = insight_generator::generate_unmet_intents_from_aggregated(&failure_rows, &submitted_specs); let open_count = intents @@ -201,10 +204,13 @@ pub(crate) async fn handle_unmet_intents( #[instrument(skip_all, fields(otel.name = "GET /observe/evolution/intent-evidence"))] pub(crate) async fn handle_intent_evidence( State(state): State, - headers: HeaderMap, + authenticated: Option>, ) -> Result, StatusCode> { - require_observe_auth(&state, &headers, "read_evolution", "Evolution")?; - let trajectory_entries = state.load_trajectory_entries(2_000).await; + let authenticated = require_authenticated_context(authenticated.as_deref())?; + require_observe_auth(&state, authenticated, "read_evolution", "Evolution")?; + let trajectory_entries = state + .load_trajectory_entries(authenticated.tenant().as_str(), 2_000) + .await; let evidence = insight_generator::generate_intent_evidence(&trajectory_entries); Ok(Json(serde_json::to_value(evidence).unwrap_or_else(|_| { serde_json::json!({ @@ -222,15 +228,17 @@ pub(crate) async fn handle_intent_evidence( #[instrument(skip_all, fields(otel.name = "GET /observe/evolution/feature-requests"))] pub(crate) async fn handle_feature_requests( State(state): State, - headers: HeaderMap, + authenticated: Option>, Query(params): Query>, ) -> Result, StatusCode> { - require_observe_auth(&state, &headers, "read_evolution", "Evolution")?; + let authenticated = require_authenticated_context(authenticated.as_deref())?; + require_observe_auth(&state, authenticated, "read_evolution", "Evolution")?; let disposition_filter = params.get("disposition").map(String::as_str); + let tenant = authenticated.tenant().as_str(); - let trajectory_entries = state.load_trajectory_entries(1_000).await; + let trajectory_entries = state.load_trajectory_entries(tenant, 1_000).await; - if let Some(store) = state.platform_metadata_store() { + if let Some(store) = state.metadata_store_for_tenant(tenant).await { let generated = insight_generator::generate_feature_requests(&trajectory_entries); for feature_request in &generated { let refs_json = serde_json::to_string(&feature_request.trajectory_refs) @@ -244,6 +252,7 @@ pub(crate) async fn handle_feature_requests( }; if let Err(error) = store .upsert_feature_request( + tenant, &feature_request.header.id, &format!("{:?}", feature_request.category), &feature_request.description, @@ -273,7 +282,10 @@ pub(crate) async fn handle_feature_requests( .await; } - return match store.list_feature_requests(disposition_filter).await { + return match store + .list_feature_requests(tenant, disposition_filter) + .await + { Ok(rows) => { let feature_requests = rows .iter() @@ -309,18 +321,18 @@ pub(crate) async fn handle_feature_requests( /// PATCH /observe/evolution/feature-requests/:id -- update disposition + notes. /// -/// Admin principals bypass Cedar; other principals require "manage_feature_requests" -/// on "FeatureRequest". +/// Requires `manage_feature_requests` on `FeatureRequest`. #[instrument(skip_all, fields(otel.name = "PATCH /observe/evolution/feature-requests/{id}"))] pub(crate) async fn handle_update_feature_request( State(state): State, - headers: HeaderMap, + authenticated: Option>, Path(id): Path, Json(body): Json, ) -> Result, StatusCode> { + let authenticated = require_authenticated_context(authenticated.as_deref())?; require_observe_auth( &state, - &headers, + authenticated, "manage_feature_requests", "FeatureRequest", )?; @@ -340,17 +352,21 @@ pub(crate) async fn handle_update_feature_request( } } - let Some(store) = state.platform_metadata_store() else { + let tenant = authenticated.tenant().as_str(); + let Some(store) = state.metadata_store_for_tenant(tenant).await else { return Err(StatusCode::SERVICE_UNAVAILABLE); }; - store - .update_feature_request(&id, disposition.unwrap_or(""), notes) + let updated = store + .update_feature_request(tenant, &id, disposition.unwrap_or(""), notes) .await .map_err(|error| { tracing::error!(error = %error, "failed to update feature request"); StatusCode::INTERNAL_SERVER_ERROR })?; + if !updated { + return Err(StatusCode::NOT_FOUND); + } emit_refresh_hints(&state, &[ObserveRefreshHint::FeatureRequests]); @@ -367,12 +383,14 @@ pub(crate) async fn handle_update_feature_request( #[instrument(skip_all, fields(otel.name = "GET /observe/evolution/stream"))] pub(crate) async fn handle_evolution_stream( State(state): State, - headers: HeaderMap, + authenticated: Option>, ) -> Result>>, StatusCode> { - require_observe_auth(&state, &headers, "read_evolution", "EvolutionStream")?; + let authenticated = require_authenticated_context(authenticated.as_deref())?; + require_observe_auth(&state, authenticated, "read_evolution", "EvolutionStream")?; + let tenant = authenticated.tenant().as_str().to_string(); let rx = state.pending_decision_tx.subscribe(); - let stream = BroadcastStream::new(rx).filter_map(|result| match result { - Ok(pending_decision) => Some(Ok(Event::default() + let stream = BroadcastStream::new(rx).filter_map(move |result| match result { + Ok(pending_decision) if pending_decision.tenant == tenant => Some(Ok(Event::default() .event("evolution_event") .json_data(serde_json::json!({ "type": "new_decision", @@ -382,7 +400,7 @@ pub(crate) async fn handle_evolution_stream( "status": "pending", })) .unwrap_or_else(|_| Event::default().data("{}")))), - Err(_) => None, + Ok(_) | Err(_) => None, }); Ok(Sse::new(stream).keep_alive(KeepAlive::default())) diff --git a/crates/temper-server/src/observe/evolution/operations/materialize.rs b/crates/temper-server/src/observe/evolution/operations/materialize.rs index 32a6e3535..572377675 100644 --- a/crates/temper-server/src/observe/evolution/operations/materialize.rs +++ b/crates/temper-server/src/observe/evolution/operations/materialize.rs @@ -1,567 +1,39 @@ -use axum::extract::{Json as ExtractJson, State}; +use axum::extract::{Extension, Json as ExtractJson, State}; use axum::http::{HeaderMap, StatusCode}; use axum::response::Json; -use serde::{Deserialize, Serialize}; -use temper_evolution::records::{ImpactAssessment, SolutionOption}; -use temper_evolution::{ - AnalysisRecord, Complexity, InsightCategory, InsightRecord, InsightSignal, ObservationClass, - ObservationRecord, ProblemRecord, RecordHeader, RecordType, Severity, SolutionRisk, Trend, -}; -use temper_runtime::scheduler::sim_now; -use temper_runtime::tenant::TenantId; +use temper_authz::AuthenticatedRequestContext; use tracing::instrument; -use crate::authz::require_observe_auth; -use crate::odata::extract_tenant; -use crate::request_context::{AgentContext, extract_agent_context}; +use crate::authz::{require_authenticated_context, require_observe_auth, require_tenant_match}; +use crate::request_context::extract_agent_context; use crate::state::{ObserveRefreshHint, ServerState}; -use super::support::{ - create_system_entity_logged, dispatch_system_action_required, emit_refresh_hints, - next_system_entity_id, persist_record, spawn_intent_discovery, -}; +use self::model::{AgentAnalysisPayload, EvolutionAnalyzeRequest, EvolutionMaterializeRequest}; +use self::records::materialize_finding; +use super::support::{emit_refresh_hints, spawn_intent_discovery}; + +mod issue; +mod model; +mod records; + +#[cfg(test)] +use self::model::{AgentFinding, finding_intent_title, finding_issue_title, finding_symptom_title}; #[cfg(test)] #[path = "materialize_test.rs"] mod tests; -#[derive(Debug, Deserialize)] -pub(crate) struct EvolutionAnalyzeRequest { - pub reason: Option, - pub source: Option, - pub trigger_context: Option, -} - -#[derive(Debug, Deserialize)] -pub(crate) struct EvolutionMaterializeRequest { - pub intent_discovery_id: String, - pub analysis_json: String, - pub signal_summary_json: String, - pub tenant: Option, - pub reason: Option, - pub source: Option, -} - -#[derive(Debug, Default, Deserialize)] -struct AgentAnalysisPayload { - #[serde(default)] - summary: String, - #[serde(default)] - findings: Vec, -} - -#[derive(Debug, Clone, Default, Deserialize, Serialize)] -struct AgentFinding { - #[serde(default)] - kind: String, - #[serde(default)] - title: String, - #[serde(default)] - symptom_title: String, - #[serde(default)] - intent_title: String, - #[serde(default)] - recommended_issue_title: String, - #[serde(default)] - intent: String, - #[serde(default)] - recommendation: String, - #[serde(default)] - priority_score: f64, - #[serde(default)] - volume: u64, - #[serde(default)] - success_rate: f64, - #[serde(default)] - trend: String, - #[serde(default)] - requires_spec_change: bool, - #[serde(default)] - problem_statement: String, - #[serde(default)] - root_cause: String, - #[serde(default)] - spec_diff: String, - #[serde(default)] - acceptance_criteria: Vec, - #[serde(default)] - dedupe_key: String, - #[serde(default)] - evidence: serde_json::Value, -} - -#[derive(Default)] -struct SpecChangeArtifacts { - record_ids: Vec, - observation_entity_id: String, - derived_from_record_id: Option, -} - -struct MaterializedFinding { - record_ids: Vec, - issue_id: String, - report: serde_json::Value, -} - -fn trend_from_str(value: &str) -> Trend { - match value.trim().to_ascii_lowercase().as_str() { - "declining" => Trend::Declining, - "stable" => Trend::Stable, - _ => Trend::Growing, - } -} - -fn severity_from_score(score: f64) -> Severity { - if score >= 0.85 { - Severity::Critical - } else if score >= 0.65 { - Severity::High - } else if score >= 0.40 { - Severity::Medium - } else { - Severity::Low - } -} - -fn solution_risk_from_score(score: f64) -> SolutionRisk { - if score >= 0.85 { - SolutionRisk::High - } else if score >= 0.65 { - SolutionRisk::Medium - } else if score >= 0.35 { - SolutionRisk::Low - } else { - SolutionRisk::None - } -} - -fn complexity_from_finding(finding: &AgentFinding) -> Complexity { - match finding.kind.trim().to_ascii_lowercase().as_str() { - "friction" | "governance_gap" => Complexity::Low, - "workaround" => Complexity::Medium, - _ => Complexity::Medium, - } -} - -fn observation_class_for_finding(finding: &AgentFinding) -> ObservationClass { - match finding.kind.trim().to_ascii_lowercase().as_str() { - "governance_gap" => ObservationClass::AuthzDenied, - _ => ObservationClass::Trajectory, - } -} - -fn insight_category_for_finding(finding: &AgentFinding) -> InsightCategory { - match finding.kind.trim().to_ascii_lowercase().as_str() { - "friction" => InsightCategory::Friction, - "workaround" => InsightCategory::Workaround, - "governance_gap" => InsightCategory::PlatformGap, - _ => InsightCategory::UnmetIntent, - } -} - -fn issue_priority_level(score: f64) -> i64 { - if score >= 0.85 { - 1 - } else if score >= 0.65 { - 2 - } else if score >= 0.40 { - 3 - } else { - 4 - } -} - -fn preferred_title(candidates: &[&str], fallback: &str) -> String { - candidates - .iter() - .find_map(|value| { - let trimmed = value.trim(); - (!trimmed.is_empty()).then(|| trimmed.to_string()) - }) - .unwrap_or_else(|| fallback.to_string()) -} - -fn finding_symptom_title(finding: &AgentFinding) -> String { - preferred_title( - &[ - &finding.symptom_title, - &finding.title, - &finding.problem_statement, - ], - "Observed workflow symptom", - ) -} - -fn finding_intent_title(finding: &AgentFinding) -> String { - preferred_title( - &[&finding.intent_title, &finding.intent, &finding.title], - "Enable unmet intent", - ) -} - -fn finding_issue_title(finding: &AgentFinding) -> String { - preferred_title( - &[ - &finding.recommended_issue_title, - &finding.intent_title, - &finding.title, - &finding.intent, - &finding.symptom_title, - ], - "Investigate unmet intent", - ) -} - -fn default_acceptance_criteria(finding: &AgentFinding) -> Vec { - if !finding.acceptance_criteria.is_empty() { - return finding.acceptance_criteria.clone(); - } - let issue_title = finding_issue_title(finding); - vec![ - format!( - "Agents can complete '{}' without the current failure mode.", - issue_title - ), - "Observe metrics show improved completion for the affected workflow.".to_string(), - ] -} - -fn build_issue_description(summary: &str, finding: &AgentFinding, record_ids: &[String]) -> String { - let acceptance_criteria = default_acceptance_criteria(finding) - .into_iter() - .map(|item| format!("- {item}")) - .collect::>() - .join("\n"); - format!( - "Summary:\n{summary}\n\nIntent Title:\n{}\n\nObserved Symptom:\n{}\n\nIntent:\n{}\n\nRecommendation:\n{}\n\nProblem Statement:\n{}\n\nRoot Cause:\n{}\n\nSpec Diff:\n{}\n\nAcceptance Criteria:\n{}\n\nEvidence:\n{}\n\nEvolution Records:\n{}", - finding_intent_title(finding), - finding_symptom_title(finding), - if finding.intent.is_empty() { - "No explicit intent supplied." - } else { - finding.intent.as_str() - }, - finding.recommendation, - if finding.problem_statement.is_empty() { - "No formal problem statement supplied." - } else { - finding.problem_statement.as_str() - }, - if finding.root_cause.is_empty() { - "No root cause supplied." - } else { - finding.root_cause.as_str() - }, - if finding.spec_diff.is_empty() { - "No spec diff supplied." - } else { - finding.spec_diff.as_str() - }, - acceptance_criteria, - serde_json::to_string_pretty(&finding.evidence).unwrap_or_else(|_| "{}".to_string()), - record_ids.join(", ") - ) -} - -async fn create_issue_for_finding( - state: &ServerState, - tenant: &TenantId, - summary: &str, - finding: &AgentFinding, - record_ids: &[String], -) -> Result { - let issue_id = temper_runtime::scheduler::sim_uuid().to_string(); - let now = sim_now().to_rfc3339(); - let issue_title = finding_issue_title(finding); - let description = build_issue_description(summary, finding, record_ids); - let acceptance_criteria = default_acceptance_criteria(finding).join("\n"); - - state - .get_or_create_tenant_entity( - tenant, - "Issue", - &issue_id, - serde_json::json!({ - "Id": issue_id.clone(), - "Title": issue_title, - "Description": description, - "AcceptanceCriteria": acceptance_criteria, - "Priority": issue_priority_level(finding.priority_score), - "CreatedAt": now, - "UpdatedAt": now, - }), - ) - .await?; - - // Walk the issue into Todo. The issue itself is already persisted, so a - // failed transition leaves it in an earlier state rather than failing - // materialization — but it must be visible, not silently swallowed. - let system_ctx = AgentContext::for_service("evolution-engine"); - let setup_actions = [ - ( - "SetPriority", - serde_json::json!({ "level": issue_priority_level(finding.priority_score) }), - ), - ("MoveToTriage", serde_json::json!({})), - ("MoveToTodo", serde_json::json!({})), - ]; - for (action, params) in setup_actions { - if let Err(e) = state - .dispatch_tenant_action(tenant, "Issue", &issue_id, action, params, &system_ctx) - .await - { - tracing::warn!( - issue_id = %issue_id, - action, - error = %e, - "issue setup transition failed; issue left in earlier state" - ); - } - } - - Ok(issue_id) -} - -async fn materialize_spec_change_records( - state: &ServerState, - tenant: &TenantId, - payload: &EvolutionMaterializeRequest, - signal_summary: &serde_json::Value, - finding: &AgentFinding, -) -> Result { - if !finding.requires_spec_change { - return Ok(SpecChangeArtifacts::default()); - } - - let observation = ObservationRecord { - header: RecordHeader::new(RecordType::Observation, "intent-discovery"), - source: format!( - "intent-discovery:{}", - if finding.kind.is_empty() { - "analysis" - } else { - finding.kind.as_str() - } - ), - classification: observation_class_for_finding(finding), - evidence_query: format!( - "intent discovery {} -> symptom={} intent={}", - payload.intent_discovery_id, - finding_symptom_title(finding), - finding_intent_title(finding) - ), - threshold_field: None, - threshold_value: None, - observed_value: Some(finding.volume as f64), - context: serde_json::json!({ - "tenant": tenant.as_str(), - "reason": payload.reason, - "source": payload.source, - "signal_summary": signal_summary.clone(), - "finding": finding, - }), - }; - persist_record(state, "Observation", &observation.header, &observation).await?; - - let observation_entity_id = next_system_entity_id("OBS"); - create_system_entity_logged( - state, - "Observation", - &observation_entity_id, - "CreateObservation", - serde_json::json!({ - "source": observation.source, - "classification": format!("{:?}", observation.classification), - "evidence_query": observation.evidence_query, - "context": serde_json::to_string(&observation.context).unwrap_or_default(), - "tenant": tenant.as_str(), - "legacy_record_id": observation.header.id, - }), - ) - .await; - - let problem = ProblemRecord { - header: RecordHeader::new(RecordType::Problem, "intent-discovery") - .derived_from(&observation.header.id), - problem_statement: if finding.problem_statement.is_empty() { - format!( - "{} blocks intended workflow completion.", - finding_intent_title(finding) - ) - } else { - finding.problem_statement.clone() - }, - invariants: default_acceptance_criteria(finding), - constraints: if finding.dedupe_key.is_empty() { - Vec::new() - } else { - vec![format!("dedupe_key={}", finding.dedupe_key)] - }, - impact: ImpactAssessment { - affected_users: Some(finding.volume), - severity: severity_from_score(finding.priority_score), - trend: trend_from_str(&finding.trend), - }, - }; - persist_record(state, "Problem", &problem.header, &problem).await?; - - let problem_entity_id = next_system_entity_id("PRB"); - dispatch_system_action_required( - state, - "Problem", - &problem_entity_id, - "CreateProblem", - serde_json::json!({ - "observation_id": observation_entity_id, - "problem_statement": problem.problem_statement, - "severity": problem.impact.severity.to_string(), - "invariants": serde_json::to_string(&problem.invariants).unwrap_or_default(), - }), - ) - .await?; - dispatch_system_action_required( - state, - "Problem", - &problem_entity_id, - "MarkReviewed", - serde_json::json!({}), - ) - .await?; - - let analysis = AnalysisRecord { - header: RecordHeader::new(RecordType::Analysis, "intent-discovery") - .derived_from(&problem.header.id), - root_cause: if finding.root_cause.is_empty() { - "IntentDiscovery inferred a missing platform capability.".to_string() - } else { - finding.root_cause.clone() - }, - options: vec![SolutionOption { - description: finding.recommendation.clone(), - spec_diff: if finding.spec_diff.is_empty() { - "No explicit spec diff supplied.".to_string() - } else { - finding.spec_diff.clone() - }, - tla_impact: "NONE".to_string(), - risk: solution_risk_from_score(finding.priority_score), - complexity: complexity_from_finding(finding), - }], - recommendation: Some(0), - }; - persist_record(state, "Analysis", &analysis.header, &analysis).await?; - - let analysis_entity_id = next_system_entity_id("ANL"); - dispatch_system_action_required( - state, - "Analysis", - &analysis_entity_id, - "CreateAnalysis", - serde_json::json!({ - "problem_id": problem_entity_id, - "root_cause": analysis.root_cause, - "options": serde_json::to_string(&analysis.options).unwrap_or_default(), - "recommendation": analysis.recommendation.unwrap_or_default().to_string(), - }), - ) - .await?; - - Ok(SpecChangeArtifacts { - record_ids: vec![ - observation.header.id.clone(), - problem.header.id.clone(), - analysis.header.id.clone(), - ], - observation_entity_id, - derived_from_record_id: Some(analysis.header.id.clone()), - }) -} - -async fn materialize_finding( - state: &ServerState, - tenant: &TenantId, - summary: &str, - payload: &EvolutionMaterializeRequest, - signal_summary: &serde_json::Value, - finding: &AgentFinding, -) -> Result { - let mut artifacts = - materialize_spec_change_records(state, tenant, payload, signal_summary, finding).await?; - - let mut insight_header = RecordHeader::new(RecordType::Insight, "intent-discovery"); - if let Some(parent) = artifacts.derived_from_record_id.as_ref() { - insight_header = insight_header.derived_from(parent.clone()); - } - let insight = InsightRecord { - header: insight_header, - category: insight_category_for_finding(finding), - signal: InsightSignal { - intent: if finding.intent.is_empty() { - finding_intent_title(finding) - } else { - finding.intent.clone() - }, - volume: finding.volume, - success_rate: finding.success_rate, - trend: trend_from_str(&finding.trend), - growth_rate: None, - }, - recommendation: finding.recommendation.clone(), - priority_score: finding.priority_score, - }; - persist_record(state, "Insight", &insight.header, &insight).await?; - artifacts.record_ids.push(insight.header.id.clone()); - - create_system_entity_logged( - state, - "Insight", - &next_system_entity_id("INS"), - "CreateInsight", - serde_json::json!({ - "observation_id": artifacts.observation_entity_id, - "category": format!("{:?}", insight.category), - "signal": insight.signal.intent, - "recommendation": insight.recommendation, - "priority_score": format!("{:.4}", insight.priority_score), - "legacy_record_id": insight.header.id, - }), - ) - .await; - - let issue_id = create_issue_for_finding(state, tenant, summary, finding, &artifacts.record_ids) - .await - .map_err(|error| { - tracing::warn!( - error = %error, - issue_title = %finding_issue_title(finding), - "evolution.issue.create" - ); - StatusCode::INTERNAL_SERVER_ERROR - })?; - - Ok(MaterializedFinding { - report: serde_json::json!({ - "title": finding_issue_title(finding), - "intent_title": finding_intent_title(finding), - "symptom_title": finding_symptom_title(finding), - "kind": finding.kind.clone(), - "record_ids": artifacts.record_ids, - "issue_id": issue_id, - }), - record_ids: artifacts.record_ids, - issue_id, - }) -} - /// POST /api/evolution/analyze -- create and run one IntentDiscovery cycle. #[instrument(skip_all, fields(otel.name = "POST /api/evolution/analyze"))] pub(crate) async fn handle_evolution_analyze( State(state): State, headers: HeaderMap, + authenticated: Option>, body: axum::body::Bytes, ) -> Result, StatusCode> { - require_observe_auth(&state, &headers, "run_sentinel", "Evolution")?; - let tenant = extract_tenant(&headers, &state).map_err(|_| StatusCode::BAD_REQUEST)?; + let authenticated = require_authenticated_context(authenticated.as_deref())?; + require_observe_auth(&state, authenticated, "run_sentinel", "Evolution")?; + let tenant = authenticated.tenant().clone(); let payload = if body.is_empty() { EvolutionAnalyzeRequest { reason: None, @@ -572,7 +44,14 @@ pub(crate) async fn handle_evolution_analyze( serde_json::from_slice::(&body) .map_err(|_| StatusCode::BAD_REQUEST)? }; - let agent_ctx = extract_agent_context(&headers); + let mut agent_ctx = extract_agent_context(&headers); + agent_ctx.security_ctx = Some(authenticated.security_context().clone()); + agent_ctx.agent_id = Some(authenticated.security_context().principal.id.clone()); + agent_ctx.agent_type = authenticated + .security_context() + .principal + .agent_type + .clone(); let reason = payload.reason.unwrap_or_else(|| "manual".to_string()); let source = payload.source.unwrap_or_else(|| "developer".to_string()); let trigger_context = payload @@ -608,11 +87,17 @@ pub(crate) async fn handle_evolution_analyze( #[instrument(skip_all, fields(otel.name = "POST /api/evolution/materialize"))] pub(crate) async fn handle_evolution_materialize( State(state): State, - headers: HeaderMap, - ExtractJson(payload): ExtractJson, + authenticated: Option>, + ExtractJson(payload_json): ExtractJson, ) -> Result, StatusCode> { - require_observe_auth(&state, &headers, "run_sentinel", "Evolution")?; - let tenant = extract_tenant(&headers, &state).map_err(|_| StatusCode::BAD_REQUEST)?; + let authenticated = require_authenticated_context(authenticated.as_deref())?; + require_observe_auth(&state, authenticated, "run_sentinel", "Evolution")?; + let payload = serde_json::from_value::(payload_json) + .map_err(|_| StatusCode::BAD_REQUEST)?; + if let Some(requested_tenant) = payload.tenant.as_deref() { + require_tenant_match(authenticated, requested_tenant)?; + } + let tenant = authenticated.tenant().clone(); let analysis = serde_json::from_str::(&payload.analysis_json) .map_err(|_| StatusCode::BAD_REQUEST)?; let signal_summary = serde_json::from_str::(&payload.signal_summary_json) @@ -653,7 +138,7 @@ pub(crate) async fn handle_evolution_materialize( Ok(Json(serde_json::json!({ "intent_discovery_id": payload.intent_discovery_id, - "tenant": payload.tenant.unwrap_or_else(|| tenant.as_str().to_string()), + "tenant": tenant.as_str(), "records_created_count": record_ids.len(), "issues_created_count": issue_ids.len(), "record_ids": record_ids, diff --git a/crates/temper-server/src/observe/evolution/operations/materialize/issue.rs b/crates/temper-server/src/observe/evolution/operations/materialize/issue.rs new file mode 100644 index 000000000..c9f3b6de5 --- /dev/null +++ b/crates/temper-server/src/observe/evolution/operations/materialize/issue.rs @@ -0,0 +1,69 @@ +use temper_runtime::scheduler::sim_now; +use temper_runtime::tenant::TenantId; + +use crate::request_context::AgentContext; +use crate::state::ServerState; + +use super::model::{ + AgentFinding, build_issue_description, default_acceptance_criteria, finding_issue_title, + issue_priority_level, +}; + +pub(super) async fn create_issue_for_finding( + state: &ServerState, + tenant: &TenantId, + summary: &str, + finding: &AgentFinding, + record_ids: &[String], +) -> Result { + let issue_id = temper_runtime::scheduler::sim_uuid().to_string(); + let now = sim_now().to_rfc3339(); + let issue_title = finding_issue_title(finding); + let description = build_issue_description(summary, finding, record_ids); + let acceptance_criteria = default_acceptance_criteria(finding).join("\n"); + + state + .get_or_create_tenant_entity( + tenant, + "Issue", + &issue_id, + serde_json::json!({ + "Id": issue_id.clone(), + "Title": issue_title, + "Description": description, + "AcceptanceCriteria": acceptance_criteria, + "Priority": issue_priority_level(finding.priority_score), + "CreatedAt": now, + "UpdatedAt": now, + }), + ) + .await?; + + // Walk the issue into Todo. The issue itself is already persisted, so a + // failed transition leaves it in an earlier state rather than failing + // materialization — but it must be visible, not silently swallowed. + let system_ctx = AgentContext::for_service("evolution-engine"); + let setup_actions = [ + ( + "SetPriority", + serde_json::json!({ "level": issue_priority_level(finding.priority_score) }), + ), + ("MoveToTriage", serde_json::json!({})), + ("MoveToTodo", serde_json::json!({})), + ]; + for (action, params) in setup_actions { + if let Err(error) = state + .dispatch_tenant_action(tenant, "Issue", &issue_id, action, params, &system_ctx) + .await + { + tracing::warn!( + issue_id = %issue_id, + action, + error = %error, + "issue setup transition failed; issue left in earlier state" + ); + } + } + + Ok(issue_id) +} diff --git a/crates/temper-server/src/observe/evolution/operations/materialize/model.rs b/crates/temper-server/src/observe/evolution/operations/materialize/model.rs new file mode 100644 index 000000000..75b8d49aa --- /dev/null +++ b/crates/temper-server/src/observe/evolution/operations/materialize/model.rs @@ -0,0 +1,233 @@ +use serde::{Deserialize, Serialize}; +use temper_evolution::{ + Complexity, InsightCategory, ObservationClass, Severity, SolutionRisk, Trend, +}; + +#[derive(Debug, Deserialize)] +pub(super) struct EvolutionAnalyzeRequest { + pub(super) reason: Option, + pub(super) source: Option, + pub(super) trigger_context: Option, +} + +#[derive(Debug, Deserialize)] +pub(super) struct EvolutionMaterializeRequest { + pub(super) intent_discovery_id: String, + pub(super) analysis_json: String, + pub(super) signal_summary_json: String, + pub(super) tenant: Option, + pub(super) reason: Option, + pub(super) source: Option, +} + +#[derive(Debug, Default, Deserialize)] +pub(super) struct AgentAnalysisPayload { + #[serde(default)] + pub(super) summary: String, + #[serde(default)] + pub(super) findings: Vec, +} + +#[derive(Debug, Clone, Default, Deserialize, Serialize)] +pub(super) struct AgentFinding { + #[serde(default)] + pub(super) kind: String, + #[serde(default)] + pub(super) title: String, + #[serde(default)] + pub(super) symptom_title: String, + #[serde(default)] + pub(super) intent_title: String, + #[serde(default)] + pub(super) recommended_issue_title: String, + #[serde(default)] + pub(super) intent: String, + #[serde(default)] + pub(super) recommendation: String, + #[serde(default)] + pub(super) priority_score: f64, + #[serde(default)] + pub(super) volume: u64, + #[serde(default)] + pub(super) success_rate: f64, + #[serde(default)] + pub(super) trend: String, + #[serde(default)] + pub(super) requires_spec_change: bool, + #[serde(default)] + pub(super) problem_statement: String, + #[serde(default)] + pub(super) root_cause: String, + #[serde(default)] + pub(super) spec_diff: String, + #[serde(default)] + pub(super) acceptance_criteria: Vec, + #[serde(default)] + pub(super) dedupe_key: String, + #[serde(default)] + pub(super) evidence: serde_json::Value, +} + +pub(super) fn trend_from_str(value: &str) -> Trend { + match value.trim().to_ascii_lowercase().as_str() { + "declining" => Trend::Declining, + "stable" => Trend::Stable, + _ => Trend::Growing, + } +} + +pub(super) fn severity_from_score(score: f64) -> Severity { + if score >= 0.85 { + Severity::Critical + } else if score >= 0.65 { + Severity::High + } else if score >= 0.40 { + Severity::Medium + } else { + Severity::Low + } +} + +pub(super) fn solution_risk_from_score(score: f64) -> SolutionRisk { + if score >= 0.85 { + SolutionRisk::High + } else if score >= 0.65 { + SolutionRisk::Medium + } else if score >= 0.35 { + SolutionRisk::Low + } else { + SolutionRisk::None + } +} + +pub(super) fn complexity_from_finding(finding: &AgentFinding) -> Complexity { + match finding.kind.trim().to_ascii_lowercase().as_str() { + "friction" | "governance_gap" => Complexity::Low, + "workaround" => Complexity::Medium, + _ => Complexity::Medium, + } +} + +pub(super) fn observation_class_for_finding(finding: &AgentFinding) -> ObservationClass { + match finding.kind.trim().to_ascii_lowercase().as_str() { + "governance_gap" => ObservationClass::AuthzDenied, + _ => ObservationClass::Trajectory, + } +} + +pub(super) fn insight_category_for_finding(finding: &AgentFinding) -> InsightCategory { + match finding.kind.trim().to_ascii_lowercase().as_str() { + "friction" => InsightCategory::Friction, + "workaround" => InsightCategory::Workaround, + "governance_gap" => InsightCategory::PlatformGap, + _ => InsightCategory::UnmetIntent, + } +} + +pub(super) fn issue_priority_level(score: f64) -> i64 { + if score >= 0.85 { + 1 + } else if score >= 0.65 { + 2 + } else if score >= 0.40 { + 3 + } else { + 4 + } +} + +fn preferred_title(candidates: &[&str], fallback: &str) -> String { + candidates + .iter() + .find_map(|value| { + let trimmed = value.trim(); + (!trimmed.is_empty()).then(|| trimmed.to_string()) + }) + .unwrap_or_else(|| fallback.to_string()) +} + +pub(super) fn finding_symptom_title(finding: &AgentFinding) -> String { + preferred_title( + &[ + &finding.symptom_title, + &finding.title, + &finding.problem_statement, + ], + "Observed workflow symptom", + ) +} + +pub(super) fn finding_intent_title(finding: &AgentFinding) -> String { + preferred_title( + &[&finding.intent_title, &finding.intent, &finding.title], + "Enable unmet intent", + ) +} + +pub(super) fn finding_issue_title(finding: &AgentFinding) -> String { + preferred_title( + &[ + &finding.recommended_issue_title, + &finding.intent_title, + &finding.title, + &finding.intent, + &finding.symptom_title, + ], + "Investigate unmet intent", + ) +} + +pub(super) fn default_acceptance_criteria(finding: &AgentFinding) -> Vec { + if !finding.acceptance_criteria.is_empty() { + return finding.acceptance_criteria.clone(); + } + let issue_title = finding_issue_title(finding); + vec![ + format!( + "Agents can complete '{}' without the current failure mode.", + issue_title + ), + "Observe metrics show improved completion for the affected workflow.".to_string(), + ] +} + +pub(super) fn build_issue_description( + summary: &str, + finding: &AgentFinding, + record_ids: &[String], +) -> String { + let acceptance_criteria = default_acceptance_criteria(finding) + .into_iter() + .map(|item| format!("- {item}")) + .collect::>() + .join("\n"); + format!( + "Summary:\n{summary}\n\nIntent Title:\n{}\n\nObserved Symptom:\n{}\n\nIntent:\n{}\n\nRecommendation:\n{}\n\nProblem Statement:\n{}\n\nRoot Cause:\n{}\n\nSpec Diff:\n{}\n\nAcceptance Criteria:\n{}\n\nEvidence:\n{}\n\nEvolution Records:\n{}", + finding_intent_title(finding), + finding_symptom_title(finding), + if finding.intent.is_empty() { + "No explicit intent supplied." + } else { + finding.intent.as_str() + }, + finding.recommendation, + if finding.problem_statement.is_empty() { + "No formal problem statement supplied." + } else { + finding.problem_statement.as_str() + }, + if finding.root_cause.is_empty() { + "No root cause supplied." + } else { + finding.root_cause.as_str() + }, + if finding.spec_diff.is_empty() { + "No spec diff supplied." + } else { + finding.spec_diff.as_str() + }, + acceptance_criteria, + serde_json::to_string_pretty(&finding.evidence).unwrap_or_else(|_| "{}".to_string()), + record_ids.join(", ") + ) +} diff --git a/crates/temper-server/src/observe/evolution/operations/materialize/records.rs b/crates/temper-server/src/observe/evolution/operations/materialize/records.rs new file mode 100644 index 000000000..f3c86acae --- /dev/null +++ b/crates/temper-server/src/observe/evolution/operations/materialize/records.rs @@ -0,0 +1,279 @@ +use axum::http::StatusCode; +use temper_evolution::records::{ImpactAssessment, SolutionOption}; +use temper_evolution::{ + AnalysisRecord, InsightRecord, InsightSignal, ObservationRecord, ProblemRecord, RecordHeader, + RecordType, +}; +use temper_runtime::tenant::TenantId; + +use crate::state::ServerState; + +use super::super::support::{ + create_system_entity_logged, dispatch_system_action_required, next_system_entity_id, + persist_record, +}; +use super::issue::create_issue_for_finding; +use super::model::{ + AgentFinding, EvolutionMaterializeRequest, complexity_from_finding, + default_acceptance_criteria, finding_intent_title, finding_issue_title, finding_symptom_title, + insight_category_for_finding, observation_class_for_finding, severity_from_score, + solution_risk_from_score, trend_from_str, +}; + +#[derive(Default)] +struct SpecChangeArtifacts { + record_ids: Vec, + observation_entity_id: String, + derived_from_record_id: Option, +} + +pub(super) struct MaterializedFinding { + pub(super) record_ids: Vec, + pub(super) issue_id: String, + pub(super) report: serde_json::Value, +} + +async fn materialize_spec_change_records( + state: &ServerState, + tenant: &TenantId, + payload: &EvolutionMaterializeRequest, + signal_summary: &serde_json::Value, + finding: &AgentFinding, +) -> Result { + if !finding.requires_spec_change { + return Ok(SpecChangeArtifacts::default()); + } + + let observation = ObservationRecord { + header: RecordHeader::new(RecordType::Observation, "intent-discovery"), + source: format!( + "intent-discovery:{}", + if finding.kind.is_empty() { + "analysis" + } else { + finding.kind.as_str() + } + ), + classification: observation_class_for_finding(finding), + evidence_query: format!( + "intent discovery {} -> symptom={} intent={}", + payload.intent_discovery_id, + finding_symptom_title(finding), + finding_intent_title(finding) + ), + threshold_field: None, + threshold_value: None, + observed_value: Some(finding.volume as f64), + context: serde_json::json!({ + "tenant": tenant.as_str(), + "reason": payload.reason, + "source": payload.source, + "signal_summary": signal_summary.clone(), + "finding": finding, + }), + }; + persist_record( + state, + tenant.as_str(), + "Observation", + &observation.header, + &observation, + ) + .await?; + + let observation_entity_id = next_system_entity_id("OBS"); + create_system_entity_logged( + state, + "Observation", + &observation_entity_id, + "CreateObservation", + serde_json::json!({ + "source": observation.source, + "classification": format!("{:?}", observation.classification), + "evidence_query": observation.evidence_query, + "context": serde_json::to_string(&observation.context).unwrap_or_default(), + "tenant": tenant.as_str(), + "legacy_record_id": observation.header.id, + }), + ) + .await; + + let problem = ProblemRecord { + header: RecordHeader::new(RecordType::Problem, "intent-discovery") + .derived_from(&observation.header.id), + problem_statement: if finding.problem_statement.is_empty() { + format!( + "{} blocks intended workflow completion.", + finding_intent_title(finding) + ) + } else { + finding.problem_statement.clone() + }, + invariants: default_acceptance_criteria(finding), + constraints: if finding.dedupe_key.is_empty() { + Vec::new() + } else { + vec![format!("dedupe_key={}", finding.dedupe_key)] + }, + impact: ImpactAssessment { + affected_users: Some(finding.volume), + severity: severity_from_score(finding.priority_score), + trend: trend_from_str(&finding.trend), + }, + }; + persist_record(state, tenant.as_str(), "Problem", &problem.header, &problem).await?; + + let problem_entity_id = next_system_entity_id("PRB"); + dispatch_system_action_required( + state, + "Problem", + &problem_entity_id, + "CreateProblem", + serde_json::json!({ + "observation_id": observation_entity_id, + "problem_statement": problem.problem_statement, + "severity": problem.impact.severity.to_string(), + "invariants": serde_json::to_string(&problem.invariants).unwrap_or_default(), + }), + ) + .await?; + dispatch_system_action_required( + state, + "Problem", + &problem_entity_id, + "MarkReviewed", + serde_json::json!({}), + ) + .await?; + + let analysis = AnalysisRecord { + header: RecordHeader::new(RecordType::Analysis, "intent-discovery") + .derived_from(&problem.header.id), + root_cause: if finding.root_cause.is_empty() { + "IntentDiscovery inferred a missing platform capability.".to_string() + } else { + finding.root_cause.clone() + }, + options: vec![SolutionOption { + description: finding.recommendation.clone(), + spec_diff: if finding.spec_diff.is_empty() { + "No explicit spec diff supplied.".to_string() + } else { + finding.spec_diff.clone() + }, + tla_impact: "NONE".to_string(), + risk: solution_risk_from_score(finding.priority_score), + complexity: complexity_from_finding(finding), + }], + recommendation: Some(0), + }; + persist_record( + state, + tenant.as_str(), + "Analysis", + &analysis.header, + &analysis, + ) + .await?; + + let analysis_entity_id = next_system_entity_id("ANL"); + dispatch_system_action_required( + state, + "Analysis", + &analysis_entity_id, + "CreateAnalysis", + serde_json::json!({ + "problem_id": problem_entity_id, + "root_cause": analysis.root_cause, + "options": serde_json::to_string(&analysis.options).unwrap_or_default(), + "recommendation": analysis.recommendation.unwrap_or_default().to_string(), + }), + ) + .await?; + + Ok(SpecChangeArtifacts { + record_ids: vec![ + observation.header.id.clone(), + problem.header.id.clone(), + analysis.header.id.clone(), + ], + observation_entity_id, + derived_from_record_id: Some(analysis.header.id.clone()), + }) +} + +pub(super) async fn materialize_finding( + state: &ServerState, + tenant: &TenantId, + summary: &str, + payload: &EvolutionMaterializeRequest, + signal_summary: &serde_json::Value, + finding: &AgentFinding, +) -> Result { + let mut artifacts = + materialize_spec_change_records(state, tenant, payload, signal_summary, finding).await?; + + let mut insight_header = RecordHeader::new(RecordType::Insight, "intent-discovery"); + if let Some(parent) = artifacts.derived_from_record_id.as_ref() { + insight_header = insight_header.derived_from(parent.clone()); + } + let insight = InsightRecord { + header: insight_header, + category: insight_category_for_finding(finding), + signal: InsightSignal { + intent: if finding.intent.is_empty() { + finding_intent_title(finding) + } else { + finding.intent.clone() + }, + volume: finding.volume, + success_rate: finding.success_rate, + trend: trend_from_str(&finding.trend), + growth_rate: None, + }, + recommendation: finding.recommendation.clone(), + priority_score: finding.priority_score, + }; + persist_record(state, tenant.as_str(), "Insight", &insight.header, &insight).await?; + artifacts.record_ids.push(insight.header.id.clone()); + + create_system_entity_logged( + state, + "Insight", + &next_system_entity_id("INS"), + "CreateInsight", + serde_json::json!({ + "observation_id": artifacts.observation_entity_id, + "category": format!("{:?}", insight.category), + "signal": insight.signal.intent, + "recommendation": insight.recommendation, + "priority_score": format!("{:.4}", insight.priority_score), + "legacy_record_id": insight.header.id, + }), + ) + .await; + + let issue_id = create_issue_for_finding(state, tenant, summary, finding, &artifacts.record_ids) + .await + .map_err(|error| { + tracing::warn!( + error = %error, + issue_title = %finding_issue_title(finding), + "evolution.issue.create" + ); + StatusCode::INTERNAL_SERVER_ERROR + })?; + + Ok(MaterializedFinding { + report: serde_json::json!({ + "title": finding_issue_title(finding), + "intent_title": finding_intent_title(finding), + "symptom_title": finding_symptom_title(finding), + "kind": finding.kind.clone(), + "record_ids": artifacts.record_ids, + "issue_id": issue_id, + }), + record_ids: artifacts.record_ids, + issue_id, + }) +} diff --git a/crates/temper-server/src/observe/evolution/operations/support.rs b/crates/temper-server/src/observe/evolution/operations/support.rs index 81c84e52b..7f2638ddb 100644 --- a/crates/temper-server/src/observe/evolution/operations/support.rs +++ b/crates/temper-server/src/observe/evolution/operations/support.rs @@ -7,17 +7,22 @@ use temper_runtime::tenant::TenantId; use crate::request_context::AgentContext; use crate::sentinel; use crate::state::{DispatchExtOptions, ObserveRefreshHint, ServerState}; +use crate::storage::EvolutionRecordWrite; pub(super) async fn persist_evolution_record( state: &ServerState, - record_id: &str, - record_type: &str, - status: &str, - created_by: &str, - derived_from: Option<&str>, - data_json: &str, + record: EvolutionRecordWrite<'_>, ) -> Result<(), String> { - let Some(store) = state.platform_metadata_store() else { + let EvolutionRecordWrite { + tenant, + id: record_id, + record_type, + status, + created_by, + derived_from, + data_json, + } = record; + let Some(store) = state.metadata_store_for_tenant(tenant).await else { tracing::debug!( record_id, record_type, @@ -29,14 +34,15 @@ pub(super) async fn persist_evolution_record( }; store - .insert_evolution_record( - record_id, + .insert_evolution_record(EvolutionRecordWrite { + tenant, + id: record_id, record_type, status, created_by, derived_from, data_json, - ) + }) .await .map_err(|error| { tracing::warn!( @@ -63,6 +69,7 @@ pub(super) async fn persist_evolution_record( pub(super) async fn persist_record( state: &ServerState, + tenant: &str, record_type: &str, header: &RecordHeader, record: &T, @@ -78,12 +85,15 @@ pub(super) async fn persist_record( })?; persist_evolution_record( state, - &header.id, - record_type, - &format!("{:?}", header.status), - &header.created_by, - header.derived_from.as_deref(), - &data_json, + EvolutionRecordWrite { + tenant, + id: &header.id, + record_type, + status: &format!("{:?}", header.status), + created_by: &header.created_by, + derived_from: header.derived_from.as_deref(), + data_json: &data_json, + }, ) .await .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR) @@ -151,6 +161,7 @@ pub(super) async fn create_system_entity_logged( pub(super) async fn persist_alerts( state: &ServerState, + tenant: &str, alerts: &[sentinel::SentinelAlert], ) -> Result, StatusCode> { let mut results = Vec::new(); @@ -165,7 +176,14 @@ pub(super) async fn persist_alerts( "evolution.sentinel" ); - persist_record(state, "Observation", &alert.record.header, &alert.record).await?; + persist_record( + state, + tenant, + "Observation", + &alert.record.header, + &alert.record, + ) + .await?; let observation_id = next_system_entity_id("OBS"); create_system_entity_logged( @@ -199,8 +217,9 @@ pub(super) async fn persist_alerts( pub(super) async fn persist_insights( state: &ServerState, + tenant: &str, insights: &[InsightRecord], -) -> Vec { +) -> Result, StatusCode> { let mut results = Vec::new(); for insight in insights { tracing::info!( @@ -212,13 +231,7 @@ pub(super) async fn persist_insights( priority_score = insight.priority_score, "evolution.insight" ); - if let Err(e) = persist_record(state, "Insight", &insight.header, insight).await { - tracing::warn!( - record_id = %insight.header.id, - error = %e, - "failed to persist insight record" - ); - } + persist_record(state, tenant, "Insight", &insight.header, insight).await?; let insight_id = next_system_entity_id("INS"); create_system_entity_logged( @@ -246,7 +259,7 @@ pub(super) async fn persist_insights( "recommendation": insight.recommendation, })); } - results + Ok(results) } pub(super) async fn spawn_intent_discovery( diff --git a/crates/temper-server/src/observe/evolution/records_detail.rs b/crates/temper-server/src/observe/evolution/records_detail.rs index 6cc13b15c..0ea3132b3 100644 --- a/crates/temper-server/src/observe/evolution/records_detail.rs +++ b/crates/temper-server/src/observe/evolution/records_detail.rs @@ -1,14 +1,14 @@ -use axum::extract::{Path, State}; -use axum::http::{HeaderMap, StatusCode}; +use axum::extract::{Extension, Path, State}; +use axum::http::StatusCode; use axum::response::Json; use serde::Deserialize; -use temper_authz::PrincipalKind; +use temper_authz::AuthenticatedRequestContext; use temper_evolution::{Decision, DecisionRecord, RecordHeader, RecordStatus, RecordType}; use temper_runtime::scheduler::{sim_now, sim_uuid}; use temper_runtime::tenant::TenantId; use tracing::instrument; -use crate::authz::{require_observe_auth, security_context_from_headers}; +use crate::authz::{require_authenticated_context, require_observe_auth}; use crate::request_context::AgentContext; use crate::state::ServerState; @@ -16,12 +16,14 @@ use crate::state::ServerState; #[instrument(skip_all, fields(otel.name = "GET /observe/evolution/records/{id}"))] pub(crate) async fn handle_get_evolution_record( State(state): State, - headers: HeaderMap, + authenticated: Option>, Path(id): Path, ) -> Result, StatusCode> { - require_observe_auth(&state, &headers, "read_evolution", "Evolution")?; + let authenticated = require_authenticated_context(authenticated.as_deref())?; + require_observe_auth(&state, authenticated, "read_evolution", "Evolution")?; - match state.get_evolution_record(&id).await { + let tenant = authenticated.tenant().as_str(); + match state.get_evolution_record(tenant, &id).await { Ok(Some(row)) => { let mut record: serde_json::Value = serde_json::from_str(&row.data).unwrap_or_else(|_| serde_json::json!({})); @@ -38,7 +40,7 @@ pub(crate) async fn handle_get_evolution_record( obj.insert("derived_from".to_string(), serde_json::json!(df)); } } - let chain = validate_chain(&state, &id).await; + let chain = validate_chain(&state, tenant, &id).await; Ok(Json(serde_json::json!({ "record": record, "chain": { @@ -64,8 +66,6 @@ pub(crate) async fn handle_get_evolution_record( pub(crate) struct DecideRequest { /// The decision: "approved", "rejected", or "deferred". pub decision: String, - /// Who is making the decision (email or identifier). - pub decided_by: String, /// Human rationale for the decision. pub rationale: String, } @@ -73,36 +73,34 @@ pub(crate) struct DecideRequest { /// POST /api/evolution/records/{id}/decide -- create a D-Record for a record. /// /// The target record (by ID) must exist. Creates a DecisionRecord derived from it. -/// Admin principals bypass Cedar; other principals require "manage_decisions" on "EvolutionRecord". +/// Requires `manage_decisions` on the exact tenant's `EvolutionRecord` surface. #[instrument(skip_all, fields(otel.name = "POST /api/evolution/records/{id}/decide"))] pub(crate) async fn handle_decide( State(state): State, - headers: HeaderMap, + authenticated: Option>, Path(id): Path, Json(body): Json, ) -> Result, StatusCode> { - // Cedar authorization: admin bypass, others need manage_decisions. - let security_ctx = security_context_from_headers(&headers, None, None, None); - let tenant_hint = headers - .get("x-tenant-id") - .and_then(|v| v.to_str().ok()) - .unwrap_or("system"); - if !matches!(security_ctx.principal.kind, PrincipalKind::Admin) - && let Err(denial) = state.authorize_with_context( - &security_ctx, - "manage_decisions", - "EvolutionRecord", - &std::collections::BTreeMap::new(), - tenant_hint, - ) - { + let authenticated = require_authenticated_context(authenticated.as_deref())?; + let security_ctx = authenticated.security_context(); + let tenant = authenticated.tenant(); + if let Err(denial) = state.authorize_with_context( + security_ctx, + "manage_decisions", + "EvolutionRecord", + &std::collections::BTreeMap::from([( + "id".to_string(), + serde_json::Value::String(id.clone()), + )]), + tenant.as_str(), + ) { tracing::warn!(reason = %denial, "unauthorized decide attempt"); return Err(StatusCode::FORBIDDEN); } // Verify the target record exists. let exists = state - .get_evolution_record(&id) + .get_evolution_record(tenant.as_str(), &id) .await .map_err(|e| { tracing::error!(record_id = %id, error = %e, "failed to lookup record"); @@ -130,18 +128,19 @@ pub(crate) async fn handle_decide( let id_suffix = &temper_runtime::scheduler::sim_uuid().to_string()[..8]; let year = now.format("%Y"); let record_id = format!("D-{year}-{id_suffix}"); + let decided_by = security_ctx.principal.id.clone(); let d_record = DecisionRecord { header: RecordHeader { id: record_id.clone(), record_type: RecordType::Decision, timestamp: now, - created_by: body.decided_by.clone(), + created_by: decided_by.clone(), derived_from: Some(id.clone()), status: RecordStatus::Open, }, decision, - decided_by: body.decided_by, + decided_by, rationale: body.rationale, verification_results: None, implementation: None, @@ -150,14 +149,15 @@ pub(crate) async fn handle_decide( // Persist to the available evolution store. let data_json = serde_json::to_string(&d_record).unwrap_or_default(); state - .insert_evolution_record( - &record_id, - "Decision", - &format!("{:?}", d_record.header.status), - &d_record.header.created_by, - d_record.header.derived_from.as_deref(), - &data_json, - ) + .insert_evolution_record(crate::storage::EvolutionRecordWrite { + tenant: tenant.as_str(), + id: &record_id, + record_type: "Decision", + status: &format!("{:?}", d_record.header.status), + created_by: &d_record.header.created_by, + derived_from: d_record.header.derived_from.as_deref(), + data_json: &data_json, + }) .await .map_err(|e| { tracing::error!(record_id = %record_id, error = %e, "failed to persist decision record"); @@ -227,7 +227,11 @@ fn record_type_from_id_prefix(id: &str) -> Option { } } -async fn validate_chain(state: &ServerState, leaf_id: &str) -> ChainValidationSummary { +async fn validate_chain( + state: &ServerState, + tenant: &str, + leaf_id: &str, +) -> ChainValidationSummary { let mut errors = Vec::new(); let mut chain_length = 0usize; let mut current_id = leaf_id.to_string(); @@ -256,7 +260,7 @@ async fn validate_chain(state: &ServerState, leaf_id: &str) -> ChainValidationSu RecordType::FeatureRequest => vec![RecordType::Insight], }; - let derived_from = match state.get_evolution_record(¤t_id).await { + let derived_from = match state.get_evolution_record(tenant, ¤t_id).await { Ok(Some(row)) => row.derived_from, Ok(None) => { errors.push(format!("record \'{current_id}\' not found")); diff --git a/crates/temper-server/src/observe/evolution/records_list.rs b/crates/temper-server/src/observe/evolution/records_list.rs index 35e8f7b6d..b1d88544b 100644 --- a/crates/temper-server/src/observe/evolution/records_list.rs +++ b/crates/temper-server/src/observe/evolution/records_list.rs @@ -1,10 +1,11 @@ -use axum::extract::{Query, State}; -use axum::http::{HeaderMap, StatusCode}; +use axum::extract::{Extension, Query, State}; +use axum::http::StatusCode; use axum::response::Json; use serde::Deserialize; +use temper_authz::AuthenticatedRequestContext; use tracing::instrument; -use crate::authz::require_observe_auth; +use crate::authz::{require_authenticated_context, require_observe_auth}; use crate::state::ServerState; /// Query parameters for listing evolution records. @@ -20,13 +21,18 @@ pub(crate) struct EvolutionRecordParams { #[instrument(skip_all, fields(otel.name = "GET /observe/evolution/records"))] pub(crate) async fn handle_list_evolution_records( State(state): State, - headers: HeaderMap, + authenticated: Option>, Query(params): Query, ) -> Result, StatusCode> { - require_observe_auth(&state, &headers, "read_evolution", "Evolution")?; + let authenticated = require_authenticated_context(authenticated.as_deref())?; + require_observe_auth(&state, authenticated, "read_evolution", "Evolution")?; match state - .list_evolution_records(params.record_type.as_deref(), params.status.as_deref()) + .list_evolution_records( + authenticated.tenant().as_str(), + params.record_type.as_deref(), + params.status.as_deref(), + ) .await { Ok(rows) => { @@ -77,11 +83,15 @@ pub(crate) async fn handle_list_evolution_records( #[instrument(skip_all, fields(otel.name = "GET /observe/evolution/insights"))] pub(crate) async fn handle_list_evolution_insights( State(state): State, - headers: HeaderMap, + authenticated: Option>, ) -> Result, StatusCode> { - require_observe_auth(&state, &headers, "read_evolution", "Evolution")?; + let authenticated = require_authenticated_context(authenticated.as_deref())?; + require_observe_auth(&state, authenticated, "read_evolution", "Evolution")?; - match state.list_ranked_insights().await { + match state + .list_ranked_insights(authenticated.tenant().as_str()) + .await + { Ok(rows) => { let items: Vec = rows .iter() diff --git a/crates/temper-server/src/observe/evolution/tenant_test.rs b/crates/temper-server/src/observe/evolution/tenant_test.rs new file mode 100644 index 000000000..df4c6df0d --- /dev/null +++ b/crates/temper-server/src/observe/evolution/tenant_test.rs @@ -0,0 +1,198 @@ +use axum::Router; +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use temper_authz::{AuthenticatedRequestContext, Principal, PrincipalKind, SecurityContext}; +use temper_runtime::ActorSystem; +use temper_runtime::tenant::TenantId; +use temper_store_turso::TursoEventStore; +use tokio_stream::StreamExt; +use tower::ServiceExt; + +use crate::observe::build_observe_router; +use crate::registry::SpecRegistry; +use crate::state::{DecisionStatus, PendingDecision, ServerState}; +use crate::storage::StorageStack; + +async fn state_and_store() -> (ServerState, TursoEventStore) { + let url = format!( + "file:{}/temper-evolution-tenant-{}.db", + std::env::temp_dir().display(), + uuid::Uuid::new_v4() + ); + let store = TursoEventStore::new(&url, None).await.unwrap(); + let mut state = ServerState::from_registry( + ActorSystem::new("evolution-tenant-test"), + SpecRegistry::new(), + ); + state.set_storage_stack(StorageStack::from_turso(store.clone())); + state + .authz + .reload_tenant_policies( + "tenant-a", + r#"permit(principal == Admin::"admin", action, resource);"#, + ) + .expect("install explicit tenant-a test authority"); + (state, store) +} + +fn admin_request(tenant: &str, request: Request) -> Request { + let context = SecurityContext { + principal: Principal { + id: "admin".to_string(), + kind: PrincipalKind::Admin, + role: None, + acting_for: None, + agent_type: None, + attributes: Default::default(), + }, + context_attrs: Default::default(), + correlation_id: "evolution-tenant-test".to_string(), + }; + let mut request = request; + request + .extensions_mut() + .insert(AuthenticatedRequestContext::new( + TenantId::new(tenant), + context, + )); + request +} + +fn app(state: ServerState) -> Router { + Router::new() + .nest("/observe", build_observe_router()) + .with_state(state) +} + +async fn response_json(response: axum::response::Response) -> serde_json::Value { + let bytes = axum::body::to_bytes(response.into_body(), 1024 * 1024) + .await + .unwrap(); + serde_json::from_slice(&bytes).unwrap() +} + +#[tokio::test] +async fn record_and_feature_handlers_cannot_cross_tenants() { + let (state, store) = state_and_store().await; + for (tenant, id) in [("tenant-a", "O-a"), ("tenant-b", "O-b")] { + store + .insert_evolution_record(temper_store_turso::TursoEvolutionRecordInsert { + tenant, + id, + record_type: "Observation", + status: "Open", + created_by: "test", + derived_from: None, + data_json: "{}", + }) + .await + .unwrap(); + } + for (tenant, id) in [("tenant-a", "feature-a"), ("tenant-b", "feature-b")] { + store + .upsert_feature_request(tenant, id, "Workflow", id, 1, "[]", "Open", None) + .await + .unwrap(); + } + let router = app(state); + + let response = router + .clone() + .oneshot(admin_request( + "tenant-a", + Request::get("/observe/evolution/records") + .body(Body::empty()) + .unwrap(), + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let json = response_json(response).await; + assert_eq!(json["records"].as_array().unwrap().len(), 1); + assert_eq!(json["records"][0]["id"], "O-a"); + + let response = router + .clone() + .oneshot(admin_request( + "tenant-a", + Request::get("/observe/evolution/records/O-b") + .body(Body::empty()) + .unwrap(), + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::NOT_FOUND); + + let response = router + .oneshot(admin_request( + "tenant-a", + Request::patch("/observe/evolution/feature-requests/feature-b") + .header("content-type", "application/json") + .body(Body::from(r#"{"disposition":"Resolved"}"#)) + .unwrap(), + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::NOT_FOUND); + let tenant_b = store.list_feature_requests("tenant-b", None).await.unwrap(); + assert_eq!(tenant_b[0].disposition, "Open"); +} + +fn pending_decision(tenant: &str, id: &str) -> PendingDecision { + PendingDecision { + id: id.to_string(), + tenant: tenant.to_string(), + agent_id: "agent".to_string(), + action: "read".to_string(), + resource_type: "Order".to_string(), + resource_id: "order-1".to_string(), + resource_attrs: serde_json::json!({}), + denial_reason: "test".to_string(), + module_name: None, + created_at: "2026-01-01T00:00:00Z".to_string(), + status: DecisionStatus::Pending, + decided_by: None, + decided_at: None, + generated_policy: None, + approved_scope: None, + evolution_record_id: None, + agent_type: None, + principal_kind: None, + session_id: None, + governance_decision_id: None, + } +} + +#[tokio::test] +async fn evolution_stream_emits_only_authenticated_tenant_events() { + let (state, _) = state_and_store().await; + let sender = state.pending_decision_tx.clone(); + let response = app(state) + .oneshot(admin_request( + "tenant-a", + Request::get("/observe/evolution/stream") + .body(Body::empty()) + .unwrap(), + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + sender + .send(pending_decision("tenant-b", "decision-b")) + .unwrap(); + sender + .send(pending_decision("tenant-a", "decision-a")) + .unwrap(); + + let mut stream = response.into_body().into_data_stream(); + let mut text = String::new(); + while !text.contains("decision-a") { + let chunk = tokio::time::timeout(std::time::Duration::from_secs(1), stream.next()) + .await + .expect("SSE frame timeout") + .expect("SSE stream ended") + .expect("SSE body error"); + text.push_str(&String::from_utf8_lossy(&chunk)); + } + assert!(!text.contains("decision-b")); +} diff --git a/crates/temper-server/src/observe/evolution/trajectories.rs b/crates/temper-server/src/observe/evolution/trajectories.rs index cf4e6396e..a68238e10 100644 --- a/crates/temper-server/src/observe/evolution/trajectories.rs +++ b/crates/temper-server/src/observe/evolution/trajectories.rs @@ -1,11 +1,14 @@ -use axum::extract::{Query, State}; -use axum::http::{HeaderMap, StatusCode}; +use axum::extract::{Extension, Query, State}; +use axum::http::StatusCode; use axum::response::Json; use serde::Deserialize; +use temper_authz::AuthenticatedRequestContext; use temper_runtime::scheduler::{sim_now, sim_uuid}; use tracing::instrument; -use crate::authz::{observe_tenant_scope, require_observe_auth}; +use crate::authz::{ + observe_tenant_scope, require_authenticated_context, require_observe_auth, require_tenant_match, +}; use crate::ots_trajectory_outbox::{OtsTrajectoryEnqueueError, OtsTrajectoryWrite}; use crate::state::{ServerState, TrajectoryEntry, TrajectorySource}; @@ -32,22 +35,17 @@ pub(crate) struct TrajectoryQueryParams { #[instrument(skip_all, fields(otel.name = "GET /observe/trajectories"))] pub(crate) async fn handle_trajectories( State(state): State, - headers: HeaderMap, + authenticated: Option>, Query(params): Query, ) -> Result, StatusCode> { - require_observe_auth(&state, &headers, "read_trajectories", "Trajectory")?; - let tenant_scope = observe_tenant_scope(&state, &headers)?; + let authenticated = require_authenticated_context(authenticated.as_deref())?; + require_observe_auth(&state, authenticated, "read_trajectories", "Trajectory")?; + let tenant_scope = observe_tenant_scope(authenticated); let failed_limit = params.failed_limit.unwrap_or(50).min(500); let success_filter: Option = params.success.as_deref().map(|s| s == "true"); - let stores = if let Some(ref scope) = tenant_scope { - match state.metadata_store_for_tenant(scope.as_str()).await { - Some(store) => vec![store], - None => Vec::new(), - } - } else { - state.collect_all_metadata_stores().await - }; + let store = state.metadata_store_for_tenant(tenant_scope.as_str()).await; + let stores = store.into_iter().collect::>(); if !stores.is_empty() { // Aggregate stats across all queried stores. @@ -61,6 +59,7 @@ pub(crate) async fn handle_trajectories( for store in &stores { match store .query_trajectory_stats( + tenant_scope.as_str(), params.entity_type.as_deref(), params.action.as_deref(), success_filter, @@ -127,31 +126,58 @@ pub(crate) async fn handle_trajectories( /// /// Called by the production chat proxy when a user asks for something /// that doesn't map to any available action. This feeds the Evolution Engine. +/// +/// The row's tenant, agent id, and agent type come from the credential; the +/// descriptive fields — entity type, action name, session — come from the request +/// body, so the row is a caller's account of something that never reached a +/// governed dispatch. It is written `spec_governed = false` for +/// that reason: the conformance checker judges governed dispatches, and a row +/// any caller can post under any session and entity type would otherwise let +/// one caller inject violations into another run's report +/// (`crate::conformance::walk::row_disposition`). #[instrument(skip_all, fields(otel.name = "POST /api/evolution/trajectories/unmet"))] pub(crate) async fn handle_unmet_intent( State(state): State, - headers: HeaderMap, + authenticated: Option>, Json(body): Json, ) -> Result { - require_observe_auth(&state, &headers, "write_trajectories", "Trajectory") + let authenticated = require_authenticated_context(authenticated.as_deref()) + .map_err(|status| (status, "unauthorized".to_string()))?; + require_observe_auth(&state, authenticated, "write_trajectories", "Trajectory") .map_err(|sc| (sc, "unauthorized".to_string()))?; + if !state.enqueue_trajectory_entry(unmet_intent_entry(authenticated, &body)?) { + tracing::warn!("failed to enqueue unmet-intent trajectory"); + } + + Ok(StatusCode::CREATED) +} + +/// Build the row an unmet-intent report becomes. +/// +/// The row's tenant comes from the credential, never from the body: authorizing +/// against one tenant and writing into another is the split-brain ADR-0157 closes. +fn unmet_intent_entry( + authenticated: &AuthenticatedRequestContext, + body: &serde_json::Value, +) -> Result { let intent = body .get("action") .or_else(|| body.get("intent")) .and_then(|v| v.as_str()) .unwrap_or("unknown"); - let tenant = body - .get("tenant") - .and_then(|v| v.as_str()) - .unwrap_or("default"); + if let Some(requested_tenant) = body.get("tenant").and_then(|value| value.as_str()) { + require_tenant_match(authenticated, requested_tenant) + .map_err(|status| (status, "tenant mismatch".to_string()))?; + } + let tenant = authenticated.tenant().as_str(); let entity_type = body .get("entity_type") .and_then(|v| v.as_str()) .unwrap_or(""); let error_msg = body.get("error").and_then(|v| v.as_str()).unwrap_or(""); - let entry = TrajectoryEntry { + Ok(TrajectoryEntry { timestamp: sim_now().to_rfc3339(), tenant: tenant.to_string(), entity_type: entity_type.to_string(), @@ -160,10 +186,7 @@ pub(crate) async fn handle_unmet_intent( success: false, from_status: None, to_status: None, - agent_id: body - .get("agent_id") - .and_then(|v| v.as_str()) - .map(str::to_string), + agent_id: Some(authenticated.security_context().principal.id.clone()), session_id: body .get("session_id") .and_then(|v| v.as_str()) @@ -185,8 +208,17 @@ pub(crate) async fn handle_unmet_intent( } else { error_msg.to_string() }), - spec_governed: None, - agent_type: None, + // An unmet intent is by definition an action the kernel never + // dispatched, and the row body is caller-supplied. Both make it a + // report about the run rather than a record of it, so it stays out of + // conformance verdicts. + spec_governed: Some(false), + // Provenance comes from the authenticated credential, not the body. + agent_type: authenticated + .security_context() + .principal + .agent_type + .clone(), request_body: body.get("request_body").cloned(), intent: body .get("intent") @@ -194,12 +226,8 @@ pub(crate) async fn handle_unmet_intent( .map(str::to_string) .or_else(|| Some(intent.to_string())), matched_policy_ids: None, - }; - if !state.enqueue_trajectory_entry(entry) { - tracing::warn!("failed to enqueue unmet-intent trajectory"); - } - - Ok(StatusCode::CREATED) + capture_seq: None, + }) } // --------------------------------------------------------------------------- @@ -215,55 +243,66 @@ pub(crate) struct OtsTrajectoryQueryParams { } /// POST /api/ots/trajectories — receive a full OTS trajectory from an MCP session. +/// +/// The body is parsed as an [`OTSTrajectory`], not as free JSON. Two things +/// depend on that: +/// +/// - **Identity.** The run's id is the document's top-level `trajectory_id`. +/// It is what the uploader holds and what +/// `GET /api/ots/trajectories/{id}/atif` and a conformance check address the +/// row by, so storing anything else makes a successfully uploaded run +/// unreachable. +/// - **The token-signal contract.** `OTSTurn` refuses to deserialize a turn +/// whose completion-side signals disagree, so a misaligned training sample +/// is rejected at the door instead of persisted and later exported as valid +/// RL data. #[instrument(skip_all, fields(otel.name = "POST /api/ots/trajectories"))] pub(crate) async fn handle_post_ots_trajectory( State(state): State, - headers: HeaderMap, + authenticated: Option>, body: String, ) -> Result { - // Parse the OTS trajectory JSON to extract indexed fields. - let trajectory: serde_json::Value = serde_json::from_str(&body) - .map_err(|e| (StatusCode::BAD_REQUEST, format!("invalid JSON: {e}")))?; + let authenticated = require_authenticated_context(authenticated.as_deref()) + .map_err(|status| (status, "unauthorized".to_string()))?; + require_observe_auth(&state, authenticated, "write_trajectories", "OtsTrajectory") + .map_err(|status| (status, "unauthorized".to_string()))?; + let trajectory: temper_ots::models::OTSTrajectory = + serde_json::from_str(&body).map_err(|e| { + ( + StatusCode::BAD_REQUEST, + format!("not a valid OTS trajectory: {e}"), + ) + })?; - let trajectory_id = trajectory - .get("metadata") - .and_then(|m| m.get("trajectory_id")) - .and_then(|v| v.as_str()) - .map(|s| s.to_string()) - .unwrap_or_else(|| sim_uuid().to_string()); - - let agent_id = headers - .get("X-Agent-Id") - .and_then(|v| v.to_str().ok()) - .or_else(|| { - trajectory - .get("metadata") - .and_then(|m| m.get("agent_id")) - .and_then(|v| v.as_str()) - }) - .unwrap_or("unknown"); + let trajectory_id = if trajectory.trajectory_id.is_empty() { + // Uploads have carried an id since the format existed; generating one + // keeps a legacy producer working, at the cost of an id it cannot + // address the row by. + let generated = sim_uuid().to_string(); + tracing::warn!( + generated_trajectory_id = %generated, + "OTS upload carried no trajectory_id; storing under a generated id" + ); + generated + } else { + trajectory.trajectory_id.clone() + }; - let session_id = headers - .get("X-Session-Id") - .and_then(|v| v.to_str().ok()) - .unwrap_or(""); + let agent_id = authenticated.security_context().principal.id.as_str(); - let outcome = trajectory - .get("metadata") - .and_then(|m| m.get("outcome")) - .and_then(|v| v.as_str()) - .unwrap_or("unknown"); + // The uploader's declared session, carried on the request context by the + // bearer edge (never a raw header read, and never a Cedar input). + let session_id = authenticated.session_id().unwrap_or(""); + + let outcome = match trajectory.metadata.outcome { + temper_ots::models::OutcomeType::Success => "success", + temper_ots::models::OutcomeType::PartialSuccess => "partial_success", + temper_ots::models::OutcomeType::Failure => "failure", + }; - let turn_count = trajectory - .get("turns") - .and_then(|t| t.as_array()) - .map(|a| a.len() as i64) - .unwrap_or(0); + let turn_count = trajectory.turns.len() as i64; - let tenant = headers - .get("X-Tenant-Id") - .and_then(|v| v.to_str().ok()) - .unwrap_or("default"); + let tenant = authenticated.tenant().as_str(); let Some(store) = state.metadata_store_for_tenant(tenant).await else { tracing::warn!( @@ -325,14 +364,21 @@ pub(crate) async fn handle_post_ots_trajectory( #[instrument(skip_all, fields(otel.name = "GET /api/ots/trajectories"))] pub(crate) async fn handle_get_ots_trajectories( State(state): State, - headers: HeaderMap, + authenticated: Option>, Query(params): Query, -) -> Result, StatusCode> { - let tenant = headers - .get("X-Tenant-Id") - .and_then(|v| v.to_str().ok()) - .unwrap_or("default"); - let limit = params.limit.unwrap_or(50).min(500); +) -> Result, (StatusCode, String)> { + let authenticated = require_authenticated_context(authenticated.as_deref()) + .map_err(|status| (status, "unauthorized".to_string()))?; + require_observe_auth(&state, authenticated, "read_trajectories", "OtsTrajectory") + .map_err(|status| (status, "unauthorized".to_string()))?; + let tenant = authenticated.tenant().as_str(); + let limit = params.limit.unwrap_or(50); + if !(1..=500).contains(&limit) { + return Err(( + StatusCode::BAD_REQUEST, + format!("limit must be between 1 and 500, got {limit}"), + )); + } let Some(store) = state.metadata_store_for_tenant(tenant).await else { return Ok(Json(serde_json::json!({ @@ -366,3 +412,99 @@ pub(crate) async fn handle_get_ots_trajectories( } } } + +#[cfg(test)] +#[path = "trajectories_route_test.rs"] +mod route_tests; + +#[cfg(test)] +mod tests { + use super::*; + use crate::conformance::{ConformanceInput, SpecResolution, check_conformance}; + use temper_spec::automaton::parse_automaton; + + const ORDER_IOA: &str = include_str!("../../../../../test-fixtures/specs/order.ioa.toml"); + + #[test] + fn an_unmet_intent_row_is_never_a_governed_dispatch() { + // Even when the caller declares an entity source, a governed session, + // and a real actor's entity type. + let authenticated = AuthenticatedRequestContext::new( + temper_runtime::tenant::TenantId::default(), + temper_authz::SecurityContext::system(), + ); + let entry = unmet_intent_entry( + &authenticated, + &serde_json::json!({ + "tenant": "default", + "entity_type": "Order", + "action": "ShipOrder", + "session_id": "session-1", + "source": "entity", + }), + ) + .expect("the system context is bound to the default tenant"); + + assert_eq!( + entry.spec_governed, + Some(false), + "the kernel never dispatched this action; the row is a report about the run" + ); + } + + #[tokio::test] + async fn an_unmet_intent_row_cannot_inject_a_violation_into_a_session() { + // The whole row is caller-chosen, so without the exclusion any caller + // could post an illegal transition into another run's report. Written + // and read back through a real store, because what the checker sees is + // the stored row, not the entry. + let dir = tempfile::tempdir().expect("create temp dir"); + let db_url = format!("file:{}", dir.path().join("unmet.db").display()); + let store = temper_store_turso::TursoEventStore::new(&db_url, None) + .await + .expect("create local turso store"); + + let authenticated = AuthenticatedRequestContext::new( + temper_runtime::tenant::TenantId::default(), + temper_authz::SecurityContext::system(), + ); + let entry = unmet_intent_entry( + &authenticated, + &serde_json::json!({ + "tenant": "default", + "entity_type": "Order", + "action": "ShipOrder", + "session_id": "session-1", + "source": "entity", + }), + ) + .expect("the system context is bound to the default tenant"); + crate::storage::TrajectorySink::persist_trajectory_entry(&store, &entry) + .await + .expect("persist unmet intent"); + + let rows = store + .query_trajectories_by_session("session-1", Some("default"), None, 10) + .await + .expect("read the session back"); + assert_eq!(rows.len(), 1, "the row is stored and readable"); + + let automaton = parse_automaton(ORDER_IOA).expect("order fixture parses"); + let report = check_conformance(ConformanceInput { + automaton: &automaton, + kernel_rows: &rows, + ots_trajectory: None, + rows_truncated: false, + spec_resolution: SpecResolution::Pinned, + capture_degraded: false, + }); + + assert!( + report.violations.is_empty(), + "a caller-supplied unmet intent is not this actor executing its spec: {:?}", + report.violations + ); + assert_eq!(report.stats.non_governed_rows_skipped, 1); + assert_eq!(report.stats.actor_rows, 0); + } +} diff --git a/crates/temper-server/src/observe/evolution/trajectories_route_test.rs b/crates/temper-server/src/observe/evolution/trajectories_route_test.rs new file mode 100644 index 000000000..7c1a368b7 --- /dev/null +++ b/crates/temper-server/src/observe/evolution/trajectories_route_test.rs @@ -0,0 +1,330 @@ +//! Security regression tests for the OTS trajectory routes (ARN-187). +//! +//! `GET`/`POST /api/ots/trajectories` previously read the raw `X-Tenant-Id` +//! header and selected that tenant's store without authorization, so any +//! caller could read another tenant's full agent-execution traces or forge +//! writes by setting a header. These tests lock in the credential gate +//! (ADR-0157): identity and tenant come only from the typed request context, +//! authorization from `require_observe_auth`. +//! +//! Ported from the ARN-187 branch (`claude/arn-187-ots-auth-gate`, PR #347) +//! and adapted: the app-level `{tenant}::{id}` key prefix it tested is +//! superseded by the store-level `(tenant, trajectory_id)` primary key. +//! +//! NOTE: behind the `observe` feature; a bare `cargo test -p temper-server` +//! filters these out. + +use axum::Router; +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use temper_authz::{AuthenticatedRequestContext, Principal, PrincipalKind, SecurityContext}; +use temper_runtime::ActorSystem; +use temper_runtime::tenant::TenantId; +use tower::ServiceExt; + +use crate::ServerState; +use crate::registry::SpecRegistry; + +/// Enforcing baseline: only `Admin` principals are permitted anything, so the +/// deny path is real rather than the permissive test default. +const ADMIN_ONLY_POLICY: &str = r#"permit(principal is Admin, action, resource);"#; + +/// A minimal well-formed OTS trajectory payload. +fn sample_trajectory() -> String { + serde_json::json!({ + "trajectory_id": "traj-route-1", + "version": "0.1.0", + "metadata": { + "task_description": "t", + "timestamp_start": "2026-01-01T00:00:00Z", + "agent_id": "a1", + "outcome": "success", + "human_reviewed": false, + }, + "context": {}, + }) + .to_string() +} + +fn enforcing_state() -> ServerState { + let state = ServerState::from_registry(ActorSystem::new("test-ots-auth"), SpecRegistry::new()); + state + .authz + .reload_policies(ADMIN_ONLY_POLICY) + .expect("baseline policy should parse"); + state + .authz + .reload_tenant_policies("tenant-a", ADMIN_ONLY_POLICY) + .expect("tenant-a baseline policy should parse"); + state +} + +fn app(state: ServerState) -> Router { + Router::new() + .nest("/api", crate::api::build_api_router()) + .with_state(state) +} + +fn with_auth( + mut request: Request, + tenant: &str, + security_context: SecurityContext, +) -> Request { + request + .extensions_mut() + .insert(AuthenticatedRequestContext::new( + TenantId::new(tenant), + security_context, + )); + request +} + +fn admin_auth() -> SecurityContext { + SecurityContext { + principal: Principal { + id: "admin-1".to_string(), + kind: PrincipalKind::Admin, + role: None, + acting_for: None, + agent_type: None, + attributes: Default::default(), + }, + context_attrs: Default::default(), + correlation_id: "ots-admin-test".to_string(), + } +} + +fn agent_auth(id: &str) -> SecurityContext { + SecurityContext::from_resolved_identity(id, "worker", Some("session-1")) +} + +/// Exploit (a): an anonymous caller spoofs a victim tenant via `X-Tenant-Id` +/// and reads its OTS traces. Must be refused at the edge. +#[tokio::test] +async fn unauthenticated_get_ots_trajectories_is_denied() { + let app = app(enforcing_state()); + let resp = app + .oneshot( + Request::get("/api/ots/trajectories") + .header("X-Tenant-Id", "victim-tenant") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); +} + +/// Exploit (b): an anonymous caller forges an OTS trajectory into a victim +/// tenant. Must be refused at the edge. +#[tokio::test] +async fn unauthenticated_post_ots_trajectory_is_denied() { + let app = app(enforcing_state()); + let resp = app + .oneshot( + Request::post("/api/ots/trajectories") + .header("content-type", "application/json") + .header("X-Tenant-Id", "victim-tenant") + .body(Body::from(sample_trajectory())) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); +} + +#[tokio::test] +async fn raw_admin_headers_do_not_authorize_ots_trajectories() { + let app = app(enforcing_state()); + let resp = app + .oneshot( + Request::get("/api/ots/trajectories") + .header("X-Temper-Principal-Kind", "admin") + .header("X-Temper-Principal-Id", "admin-1") + .header("X-Tenant-Id", "victim-tenant") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::UNAUTHORIZED, + "raw identity headers must not create an authenticated OTS context" + ); +} + +/// A properly-authorized same-tenant GET still succeeds — the gate must not +/// over-block legitimate callers. +#[tokio::test] +async fn authorized_admin_get_ots_trajectories_succeeds() { + let app = app(enforcing_state()); + let resp = app + .oneshot(with_auth( + Request::get("/api/ots/trajectories") + .body(Body::empty()) + .unwrap(), + "tenant-a", + admin_auth(), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); +} + +/// Out-of-range limits are rejected before any store work (ARN-187 delta: +/// strict rejection rather than a silent clamp). +#[tokio::test] +async fn out_of_range_ots_list_limits_are_rejected_before_store_query() { + let app = app(enforcing_state()); + for query in ["limit=-1", "limit=0", "limit=501"] { + let resp = app + .clone() + .oneshot(with_auth( + Request::get(format!("/api/ots/trajectories?{query}").as_str()) + .body(Body::empty()) + .unwrap(), + "tenant-a", + admin_auth(), + )) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::BAD_REQUEST, + "{query} must not reach the storage adapter" + ); + } +} + +/// A properly-authorized same-tenant POST still succeeds. +#[tokio::test] +async fn authorized_admin_post_ots_trajectory_succeeds() { + let app = app(enforcing_state()); + let resp = app + .oneshot(with_auth( + Request::post("/api/ots/trajectories") + .header("content-type", "application/json") + .body(Body::from(sample_trajectory())) + .unwrap(), + "tenant-a", + admin_auth(), + )) + .await + .unwrap(); + // With no durable metadata store configured the handler short-circuits to + // `201 Created`; the point is that an authorized write is not refused. + assert_eq!(resp.status(), StatusCode::CREATED); +} + +/// Cross-tenant isolation on the read path: the tenant is the credential's, +/// and a spoofed `X-Tenant-Id` naming another tenant changes nothing. +#[tokio::test] +async fn tenant_scoped_credential_cannot_read_other_tenant() { + let state = ServerState::from_registry( + ActorSystem::new("test-ots-cross-tenant"), + SpecRegistry::new(), + ); + state + .authz + .reload_policies(ADMIN_ONLY_POLICY) + .expect("baseline policy should parse"); + // tenant-a grants its agents read access; tenant-b has no loaded tenant + // policy set, so a non-System principal there fails closed + // (NoMatchingPermit) — the global baseline is never consulted. + state + .authz + .reload_tenant_policies( + "tenant-a", + r#"permit(principal is Agent, action == Action::"read_trajectories", resource is OtsTrajectory);"#, + ) + .expect("tenant-a policy should parse"); + let app = app(state); + + let same_tenant = app + .clone() + .oneshot(with_auth( + Request::get("/api/ots/trajectories") + .body(Body::empty()) + .unwrap(), + "tenant-a", + agent_auth("agent-a"), + )) + .await + .unwrap(); + assert_eq!(same_tenant.status(), StatusCode::OK); + + // The spoofed header must neither redirect the read into tenant-b nor + // change the Cedar tenant: the request still runs as tenant-a. + let cross_tenant = app + .oneshot(with_auth( + Request::get("/api/ots/trajectories") + .header("X-Tenant-Id", "tenant-b") + .body(Body::empty()) + .unwrap(), + "tenant-a", + agent_auth("agent-a"), + )) + .await + .unwrap(); + assert_eq!( + cross_tenant.status(), + StatusCode::OK, + "spoofed X-Tenant-Id must not override the credential-bound tenant" + ); +} + +/// Cross-tenant isolation on the write path: same property for ingestion. +#[tokio::test] +async fn tenant_scoped_credential_cannot_write_other_tenant() { + let state = ServerState::from_registry( + ActorSystem::new("test-ots-cross-tenant-write"), + SpecRegistry::new(), + ); + state + .authz + .reload_policies(ADMIN_ONLY_POLICY) + .expect("baseline policy should parse"); + // tenant-a grants its agents write access; tenant-b has no loaded tenant + // policy set, so a non-System principal there fails closed. + state + .authz + .reload_tenant_policies( + "tenant-a", + r#"permit(principal is Agent, action == Action::"write_trajectories", resource is OtsTrajectory);"#, + ) + .expect("tenant-a policy should parse"); + let app = app(state); + + let same_tenant = app + .clone() + .oneshot(with_auth( + Request::post("/api/ots/trajectories") + .header("content-type", "application/json") + .body(Body::from(sample_trajectory())) + .unwrap(), + "tenant-a", + agent_auth("agent-a"), + )) + .await + .unwrap(); + assert_eq!(same_tenant.status(), StatusCode::CREATED); + + let cross_tenant = app + .oneshot(with_auth( + Request::post("/api/ots/trajectories") + .header("content-type", "application/json") + .header("X-Tenant-Id", "tenant-b") + .body(Body::from(sample_trajectory())) + .unwrap(), + "tenant-a", + agent_auth("agent-a"), + )) + .await + .unwrap(); + assert_eq!( + cross_tenant.status(), + StatusCode::CREATED, + "spoofed X-Tenant-Id must not override the credential-bound tenant" + ); +} diff --git a/crates/temper-server/src/observe/mod.rs b/crates/temper-server/src/observe/mod.rs index fbd3e3cfd..250d943e1 100644 --- a/crates/temper-server/src/observe/mod.rs +++ b/crates/temper-server/src/observe/mod.rs @@ -47,6 +47,18 @@ pub struct SpecSummary { pub struct SpecDetail { /// Entity type name. pub entity_type: String, + /// Content hash of the IOA source this spec was parsed from. + /// + /// The authoritative value: it is what the spec store keeps, and what a + /// conformance check compares a run's `metadata.spec_version` against. + /// Reported here so a harness can record the digest the kernel holds + /// rather than computing one from a spec file — a deploy path that + /// rewrites line endings or re-emits the TOML hashes to something else + /// entirely, and every conformance check for that run is then refused. + /// + /// Empty when the spec is not one this response could hash. + #[serde(default)] + pub spec_version: String, /// Valid status states. pub states: Vec, /// Initial state. @@ -74,6 +86,25 @@ pub struct ActionDetail { pub guards: Vec, /// Effects (Debug representation). pub effects: Vec, + /// Parameters this action accepts, in spec order. + #[serde(default)] + pub params: Vec, + /// Agent-facing hint from the spec (empty when the spec has none). + #[serde(default)] + pub hint: String, +} + +/// A single action parameter: name plus its declared type ("string" default). +#[derive(Serialize, Deserialize)] +pub struct ActionParamDetail { + /// Parameter name exactly as declared in the spec's `params` list. + pub name: String, + /// Declared parameter type, serialized under the JSON key `"type"`. + /// Bare-named params carry `"string"`; typed params carry the spec's + /// type name (e.g. `"uint64"`). This is an OPEN set — generated clients + /// must not assume a closed enum. + #[serde(rename = "type")] + pub param_type: String, } /// Detail of a single invariant. diff --git a/crates/temper-server/src/observe/mod_test.rs b/crates/temper-server/src/observe/mod_test.rs index b5bf6abd0..2898adec8 100644 --- a/crates/temper-server/src/observe/mod_test.rs +++ b/crates/temper-server/src/observe/mod_test.rs @@ -75,15 +75,98 @@ fn build_test_app() -> Router { .with_state(state) } -/// Build a GET request with admin auth headers for observe endpoints. -/// -/// Uses "admin" principal kind — "system" is no longer accepted from headers -/// (only via `SecurityContext::system()` on internal paths). +fn admin_security_context() -> temper_authz::SecurityContext { + temper_authz::SecurityContext { + principal: temper_authz::Principal { + id: "admin-1".to_string(), + kind: temper_authz::PrincipalKind::Admin, + role: None, + acting_for: None, + agent_type: None, + attributes: Default::default(), + }, + context_attrs: Default::default(), + correlation_id: "observe-test-admin".to_string(), + } +} + +fn customer_security_context(id: &str) -> temper_authz::SecurityContext { + temper_authz::SecurityContext { + principal: temper_authz::Principal { + id: id.to_string(), + kind: temper_authz::PrincipalKind::Customer, + role: None, + acting_for: None, + agent_type: None, + attributes: Default::default(), + }, + context_attrs: Default::default(), + correlation_id: "observe-test-customer".to_string(), + } +} + +fn with_security_context( + mut request: Request, + security_context: temper_authz::SecurityContext, +) -> Request { + request + .extensions_mut() + .insert(temper_authz::AuthenticatedRequestContext::new( + TenantId::default(), + security_context, + )); + request +} + +fn with_tenant_security_context( + mut request: Request, + tenant: &str, + security_context: temper_authz::SecurityContext, +) -> Request { + request + .extensions_mut() + .insert(temper_authz::AuthenticatedRequestContext::new( + TenantId::new(tenant), + security_context, + )); + request +} + +fn admin_request(request: Request) -> Request { + with_security_context(request, admin_security_context()) +} + +fn system_request(request: Request) -> Request { + with_security_context(request, temper_authz::SecurityContext::system()) +} + +fn agent_request(request: Request) -> Request { + with_security_context( + request, + temper_authz::SecurityContext::from_resolved_identity("agent-1", "swe", Some("session-1")), + ) +} + +fn customer_request(request: Request, id: &str) -> Request { + with_security_context(request, customer_security_context(id)) +} + +#[tokio::test] +async fn typed_admin_kind_does_not_bypass_observe_cedar() { + let response = build_app_with_state(test_state_with_registry()) + .oneshot(admin_request( + Request::get("/observe/specs") + .body(Body::empty()) + .expect("request should build"), + )) + .await + .expect("request should run"); + assert_eq!(response.status(), StatusCode::FORBIDDEN); +} + +/// Build a GET request with kernel System authority for functional tests. fn system_get(uri: &str) -> Request { - Request::get(uri) - .header("X-Temper-Principal-Kind", "admin") - .body(Body::empty()) - .unwrap() + system_request(Request::get(uri).body(Body::empty()).unwrap()) } async fn observe_json(app: Router, uri: &str) -> serde_json::Value { @@ -201,45 +284,32 @@ async fn projection_replay_parity_endpoint_reports_projection_drift() { assert_eq!(json["report"]["drift_examples"][0]["drift_kind"], "fields"); } -/// Build a POST request with admin auth headers for observe endpoints. +/// Build a POST request with kernel System authority for functional tests. fn system_post(uri: &str, body: &str) -> Request { - Request::post(uri) - .header("Content-Type", "application/json") - .header("X-Temper-Principal-Kind", "admin") - .body(Body::from(body.to_string())) - .unwrap() -} - -/// Build a POST request with admin auth headers. -#[allow(dead_code)] -fn admin_post(uri: &str, body: &str) -> Request { - Request::post(uri) - .header("Content-Type", "application/json") - .header("X-Temper-Principal-Kind", "admin") - .body(Body::from(body.to_string())) - .unwrap() + system_request( + Request::post(uri) + .header("Content-Type", "application/json") + .body(Body::from(body.to_string())) + .unwrap(), + ) } fn agent_post(uri: &str, body: impl Into) -> Request { - Request::post(uri) - .header("X-Tenant-Id", "default") - .header("X-Temper-Principal-Id", "agent-1") - .header("X-Temper-Principal-Kind", "agent") - .header("X-Temper-Agent-Type", "swe") - .header("X-Temper-Ctx-SessionId", "session-1") - .body(body.into()) - .unwrap() + agent_request( + Request::post(uri) + .header("X-Tenant-Id", "default") + .body(body.into()) + .unwrap(), + ) } fn agent_get(uri: &str) -> Request { - Request::get(uri) - .header("X-Tenant-Id", "default") - .header("X-Temper-Principal-Id", "agent-1") - .header("X-Temper-Principal-Kind", "agent") - .header("X-Temper-Agent-Type", "swe") - .header("X-Temper-Ctx-SessionId", "session-1") - .body(Body::empty()) - .unwrap() + agent_request( + Request::get(uri) + .header("X-Tenant-Id", "default") + .body(Body::empty()) + .unwrap(), + ) } const ADMIN_MANAGE_POLICIES_POLICY: &str = r#" @@ -261,17 +331,38 @@ permit( fn install_admin_policy(state: &ServerState) { state .authz - .reload_policies(ADMIN_MANAGE_POLICIES_POLICY) + .reload_tenant_policies("default", ADMIN_MANAGE_POLICIES_POLICY) .expect("admin policy should parse"); } -fn install_admin_submit_specs_policy(state: &ServerState) { +fn install_admin_submit_specs_policy(state: &ServerState, tenant: &str) { state .authz - .reload_policies(ADMIN_SUBMIT_SPECS_POLICY) + .reload_tenant_policies(tenant, ADMIN_SUBMIT_SPECS_POLICY) .expect("submit_specs policy should parse"); } +fn install_admin_file_read_policy(state: &ServerState) { + state + .authz + .reload_tenant_policies( + "default", + r#" +permit( + principal == Admin::"admin-1", + action == Action::"read", + resource is File +); +permit( + principal == Admin::"admin-1", + action == Action::"read", + resource is FileVersion +); +"#, + ) + .expect("admin file read policy should parse"); +} + fn build_app_with_state(state: ServerState) -> Router { Router::new() .nest("/observe", build_observe_router()) @@ -389,7 +480,7 @@ async fn wasm_upload_denial_creates_pending_decision_for_module_resource() { let turso = state.platform_turso_store().expect("turso configured"); let data_str = turso - .get_pending_decision(decision_id) + .get_pending_decision("default", decision_id) .await .expect("query decision") .expect("decision should be persisted"); @@ -414,16 +505,12 @@ async fn wasm_delete_denial_creates_pending_decision_for_module_resource() { let app = build_app_with_state(state.clone()); let response = app - .oneshot( + .oneshot(agent_request( Request::delete("/api/wasm/modules/git_receive_pack") .header("X-Tenant-Id", "default") - .header("X-Temper-Principal-Id", "agent-1") - .header("X-Temper-Principal-Kind", "agent") - .header("X-Temper-Agent-Type", "swe") - .header("X-Temper-Ctx-SessionId", "session-1") .body(Body::empty()) .unwrap(), - ) + )) .await .unwrap(); @@ -436,7 +523,7 @@ async fn wasm_delete_denial_creates_pending_decision_for_module_resource() { let turso = state.platform_turso_store().expect("turso configured"); let data_str = turso - .get_pending_decision(decision_id) + .get_pending_decision("default", decision_id) .await .expect("query decision") .expect("decision should be persisted"); @@ -459,7 +546,7 @@ async fn wasm_upload_accepts_json_base64_body() { }); let response = app - .oneshot(admin_post( + .oneshot(system_post( "/api/wasm/modules/base64_module", &payload.to_string(), )) @@ -501,16 +588,14 @@ async fn approved_wasm_upload_decision_allows_agent_retry() { let approved = app .clone() - .oneshot( + .oneshot(admin_request( Request::post(format!( "/api/tenants/default/decisions/{decision_id}/approve" )) .header("content-type", "application/json") - .header("x-temper-principal-id", "admin-1") - .header("x-temper-principal-kind", "admin") .body(Body::from(r#"{"scope":{"principal":"this_agent","action":"this_action","resource":"this_resource","duration":"always"},"decided_by":"admin-1"}"#)) .unwrap(), - ) + )) .await .unwrap(); assert_eq!(approved.status(), StatusCode::OK); @@ -553,12 +638,11 @@ async fn tenant_decision_lookup_returns_known_decision_by_id() { let app = build_app_with_state(state); let response = app - .oneshot( + .oneshot(admin_request( Request::get(format!("/api/tenants/default/decisions/{decision_id}")) - .header("X-Temper-Principal-Kind", "admin") .body(Body::empty()) .unwrap(), - ) + )) .await .unwrap(); @@ -630,6 +714,7 @@ async fn tenant_decision_list_allows_agent_to_read_owned_pending_decisions() { #[tokio::test] async fn batch_file_text_read_returns_projected_file_contents_in_request_order() { let state = test_state_with_turso().await; + install_admin_file_read_policy(&state); let tenant = "default"; let turso = state .turso_store_for_tenant(tenant) @@ -706,6 +791,7 @@ async fn batch_file_text_read_returns_projected_file_contents_in_request_order() #[tokio::test] async fn batch_file_version_text_read_returns_immutable_version_contents_in_request_order() { let state = test_state_with_turso().await; + install_admin_file_read_policy(&state); let tenant = "default"; let turso = state .turso_store_for_tenant(tenant) @@ -780,6 +866,7 @@ async fn batch_file_version_text_read_returns_immutable_version_contents_in_requ #[tokio::test] async fn batch_file_version_text_read_uses_local_store_for_internal_blob_endpoint() { let mut state = test_state_with_turso().await; + install_admin_file_read_policy(&state); let tenant = "default"; let turso = state .turso_store_for_tenant(tenant) @@ -874,6 +961,106 @@ async fn test_get_spec_detail_found() { assert_eq!(detail.entity_type, "Order"); assert!(!detail.states.is_empty()); assert!(!detail.actions.is_empty()); + + // The params/hint contract (consumed by generated typed clients): + // AddItem declares two bare-named params (default "string" type, spec + // order preserved) and a hint; RemoveItem declares no hint (empty, not + // null). + let add_item = detail + .actions + .iter() + .find(|a| a.name == "AddItem") + .expect("Order fixture declares AddItem"); + let param_pairs: Vec<(&str, &str)> = add_item + .params + .iter() + .map(|p| (p.name.as_str(), p.param_type.as_str())) + .collect(); + assert_eq!( + param_pairs, + vec![("ProductId", "string"), ("Quantity", "string")] + ); + assert!(add_item.hint.starts_with("Add a product")); + let remove_item = detail + .actions + .iter() + .find(|a| a.name == "RemoveItem") + .expect("Order fixture declares RemoveItem"); + assert_eq!(remove_item.params.len(), 1); + assert_eq!(remove_item.params[0].name, "ItemId"); + assert_eq!(remove_item.hint, ""); + + // Wire-level contract: the type field serializes under the key "type", + // and params are objects, not bare strings. + let raw: serde_json::Value = serde_json::from_slice(&body).unwrap(); + let raw_add_item = raw["actions"] + .as_array() + .unwrap() + .iter() + .find(|a| a["name"] == "AddItem") + .unwrap(); + assert_eq!(raw_add_item["params"][0]["name"], "ProductId"); + assert_eq!(raw_add_item["params"][0]["type"], "string"); + assert_eq!(raw_add_item["hint"], add_item.hint); +} + +#[test] +fn action_param_detail_serializes_typed_params_under_type_key() { + // Typed params (e.g. `{ name = "sleep_seconds", type = "uint64" }` in a + // spec) reach ActionParamDetail through ActionParam::name()/param_type(); + // this locks the wire shape those values serialize into. + let detail = ActionParamDetail { + name: "sleep_seconds".to_string(), + param_type: "uint64".to_string(), + }; + let json = serde_json::to_value(&detail).unwrap(); + assert_eq!( + json, + serde_json::json!({"name": "sleep_seconds", "type": "uint64"}) + ); + let back: ActionParamDetail = serde_json::from_value(json).unwrap(); + assert_eq!(back.name, "sleep_seconds"); + assert_eq!(back.param_type, "uint64"); +} + +#[tokio::test] +async fn spec_detail_reports_the_registered_content_hash() { + // The reason the field exists: a producer that recomputes the digest from + // a spec file gets a different one the moment any deploy path rewrites + // that file, and every conformance check for the run is then refused. This + // is the digest the kernel actually holds. + let app = build_test_app(); + let response = app + .oneshot(system_get("/observe/specs/Order")) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + let body = axum::body::to_bytes(response.into_body(), 1024 * 1024) + .await + .unwrap(); + let raw: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert_eq!( + raw["spec_version"], + serde_json::json!(temper_store_turso::spec_content_hash(ORDER_IOA)), + "the reported digest must be the one a conformance check compares against" + ); + + let detail: SpecDetail = serde_json::from_slice(&body).unwrap(); + assert_eq!( + detail.spec_version.len(), + 64, + "a sha256 digest is 64 hex characters: {}", + detail.spec_version + ); + assert!( + detail + .spec_version + .chars() + .all(|c| c.is_ascii_hexdigit() && !c.is_ascii_uppercase()), + "the digest is reported bare and lowercase, not algorithm-qualified: {}", + detail.spec_version + ); } #[tokio::test] @@ -890,20 +1077,26 @@ async fn test_get_spec_detail_not_found() { #[tokio::test] async fn test_load_inline_supports_nested_paths() { let state = test_state_with_registry(); - install_admin_submit_specs_policy(&state); + install_admin_submit_specs_policy(&state, "nested-inline"); let app = build_app_with_state(state.clone()); let response = app - .oneshot(system_post( - "/api/specs/load-inline", - &serde_json::json!({ - "tenant": "nested-inline", - "specs": { - "InlineProbe/model.csdl.xml": CSDL_XML, - "InlineProbe/order.ioa.toml": ORDER_IOA - } - }) - .to_string(), + .oneshot(with_tenant_security_context( + Request::post("/api/specs/load-inline") + .header("Content-Type", "application/json") + .body(Body::from( + serde_json::json!({ + "tenant": "nested-inline", + "specs": { + "InlineProbe/model.csdl.xml": CSDL_XML, + "InlineProbe/order.ioa.toml": ORDER_IOA + } + }) + .to_string(), + )) + .unwrap(), + "nested-inline", + admin_security_context(), )) .await .unwrap(); @@ -919,12 +1112,56 @@ async fn test_load_inline_supports_nested_paths() { } #[tokio::test] -async fn test_tenant_decisions_accessible_without_auth() { +async fn test_load_inline_cannot_bundle_policy_authority() { let state = test_state_with_registry(); + let app = build_app_with_state(state.clone()); + let response = app + .oneshot(system_post( + "/api/specs/load-inline", + &serde_json::json!({ + "tenant": "default", + "specs": { + "model.csdl.xml": CSDL_XML, + "order.ioa.toml": ORDER_IOA + }, + "cedar_policies": "permit(principal, action, resource);" + }) + .to_string(), + )) + .await + .expect("request should run"); + + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + assert!( + state + .authorize_with_context( + &admin_security_context(), + "delete", + "Secret", + &BTreeMap::new(), + "default", + ) + .is_err(), + "bundled policy text must never become active" + ); +} + +#[tokio::test] +async fn tenant_decisions_require_auth_without_leaking_rows() { + let state = test_state_with_turso().await; + state + .platform_turso_store() + .expect("turso configured") + .upsert_pending_decision( + "PD-private", + "default", + "pending", + r#"{"id":"PD-private","tenant":"default","status":"pending","marker":"tenant-secret"}"#, + ) + .await + .expect("seed private decision"); let app = build_app_with_state(state); - // Decision list is accessible without auth headers (consistent with - // other observe endpoints). let response = app .oneshot( Request::get("/api/tenants/default/decisions") @@ -933,15 +1170,18 @@ async fn test_tenant_decisions_accessible_without_auth() { ) .await .unwrap(); - assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + let body = axum::body::to_bytes(response.into_body(), 1024 * 1024) + .await + .unwrap(); + assert!(!String::from_utf8_lossy(&body).contains("tenant-secret")); } #[tokio::test] -async fn test_tenant_decision_stream_accessible_without_auth() { +async fn tenant_decision_stream_requires_auth() { let state = test_state_with_registry(); let app = build_app_with_state(state); - // Decision stream is accessible without auth headers. let response = app .oneshot( Request::get("/api/tenants/default/decisions/stream") @@ -950,16 +1190,13 @@ async fn test_tenant_decision_stream_accessible_without_auth() { ) .await .unwrap(); - assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); let ct = response .headers() .get("content-type") .and_then(|v| v.to_str().ok()) .unwrap_or(""); - assert!( - ct.contains("text/event-stream"), - "expected SSE content-type, got: {ct}" - ); + assert!(!ct.contains("text/event-stream")); } #[tokio::test] @@ -970,32 +1207,123 @@ async fn test_tenant_decision_mutations_require_manage_policies() { let deny_approve = app .clone() - .oneshot( + .oneshot(customer_request( Request::post("/api/tenants/default/decisions/PD-does-not-exist/approve") .header("content-type", "application/json") - .header("x-temper-principal-id", "cust-1") - .header("x-temper-principal-kind", "customer") .body(Body::from(r#"{"scope":{"principal":"this_agent","action":"this_action","resource":"this_resource","duration":"always"}}"#)) .unwrap(), - ) + "cust-1", + )) .await .unwrap(); assert_eq!(deny_approve.status(), StatusCode::FORBIDDEN); let deny_deny = app - .oneshot( + .oneshot(customer_request( Request::post("/api/tenants/default/decisions/PD-does-not-exist/deny") .header("content-type", "application/json") - .header("x-temper-principal-id", "cust-1") - .header("x-temper-principal-kind", "customer") .body(Body::from(r#"{"decided_by":"cust-1"}"#)) .unwrap(), - ) + "cust-1", + )) .await .unwrap(); assert_eq!(deny_deny.status(), StatusCode::FORBIDDEN); } +#[tokio::test] +async fn denied_principal_cannot_approve_or_deny_even_with_manage_policies() { + let state = test_state_with_turso().await; + state + .authz + .reload_tenant_policies( + "default", + r#" +permit( + principal is Agent, + action == Action::"manage_policies", + resource is PolicySet +); +permit( + principal is Admin, + action == Action::"manage_policies", + resource is PolicySet +); +"#, + ) + .expect("manage_policies policy should parse"); + + let pending = crate::state::PendingDecision::from_denial( + "default", + "agent-1", + "Assign", + "Issue", + "issue-1", + serde_json::json!({"id":"issue-1"}), + "test denial", + None, + ); + let decision_id = pending.id.clone(); + state + .persist_pending_decision(&pending) + .await + .expect("persist pending decision"); + + let app = build_app_with_state(state.clone()); + let approve_body = r#"{"scope":{"principal":"this_agent","action":"this_action","resource":"this_resource","duration":"always"}}"#; + + let self_approve = app + .clone() + .oneshot(agent_request( + Request::post(format!( + "/api/tenants/default/decisions/{decision_id}/approve" + )) + .header("content-type", "application/json") + .body(Body::from(approve_body)) + .unwrap(), + )) + .await + .unwrap(); + assert_eq!( + self_approve.status(), + StatusCode::FORBIDDEN, + "denied agent must not approve their own decision" + ); + + let self_deny = app + .clone() + .oneshot(agent_request( + Request::post(format!("/api/tenants/default/decisions/{decision_id}/deny")) + .header("content-type", "application/json") + .body(Body::from("{}")) + .unwrap(), + )) + .await + .unwrap(); + assert_eq!( + self_deny.status(), + StatusCode::FORBIDDEN, + "denied agent must not deny their own decision" + ); + + let operator_approve = app + .oneshot(admin_request( + Request::post(format!( + "/api/tenants/default/decisions/{decision_id}/approve" + )) + .header("content-type", "application/json") + .body(Body::from(approve_body)) + .unwrap(), + )) + .await + .unwrap(); + assert_eq!( + operator_approve.status(), + StatusCode::OK, + "a different principal with manage_policies must still approve" + ); +} + #[tokio::test] async fn test_approve_decision_reload_failure_keeps_pending_and_policies_unchanged() { let state = test_state_with_turso().await; @@ -1025,16 +1353,14 @@ async fn test_approve_decision_reload_failure_keeps_pending_and_policies_unchang let app = build_app_with_state(state.clone()); let response = app - .oneshot( + .oneshot(admin_request( Request::post(format!( "/api/tenants/default/decisions/{decision_id}/approve" )) .header("content-type", "application/json") - .header("x-temper-principal-id", "admin-1") - .header("x-temper-principal-kind", "admin") .body(Body::from(r#"{"scope":{"principal":"this_agent","action":"this_action","resource":"this_resource","duration":"always"},"decided_by":"admin-1"}"#)) .unwrap(), - ) + )) .await .unwrap(); assert_eq!(response.status(), StatusCode::BAD_REQUEST); @@ -1042,7 +1368,7 @@ async fn test_approve_decision_reload_failure_keeps_pending_and_policies_unchang // Verify decision status unchanged in Turso. let turso = state.platform_turso_store().expect("turso configured"); let data_str = turso - .get_pending_decision(&decision_id) + .get_pending_decision("default", &decision_id) .await .expect("query turso") .expect("decision should still exist"); @@ -1588,6 +1914,7 @@ async fn test_intent_evidence_returns_richer_intent_candidates() { request_body: Some(serde_json::json!({"customer_id":"c-1"})), intent: Some(intent.to_string()), matched_policy_ids: None, + capture_seq: None, }) .await .unwrap(); @@ -1613,6 +1940,7 @@ async fn test_intent_evidence_returns_richer_intent_candidates() { request_body: Some(serde_json::json!({"customer_id":"c-1"})), intent: Some(intent.to_string()), matched_policy_ids: None, + capture_seq: None, }) .await .unwrap(); @@ -1817,14 +2145,15 @@ async fn test_evolution_decide_creates_d_record() { state .platform_turso_store() .expect("turso configured") - .insert_evolution_record( - &obs.header.id, - "Observation", - &format!("{:?}", obs.header.status), - &obs.header.created_by, - obs.header.derived_from.as_deref(), - &data_json, - ) + .insert_evolution_record(temper_store_turso::TursoEvolutionRecordInsert { + tenant: "default", + id: &obs.header.id, + record_type: "Observation", + status: &format!("{:?}", obs.header.status), + created_by: &obs.header.created_by, + derived_from: obs.header.derived_from.as_deref(), + data_json: &data_json, + }) .await .expect("insert O-Record to Turso"); @@ -1834,16 +2163,14 @@ async fn test_evolution_decide_creates_d_record() { .with_state(state); // Create a D-Record decision. - let response = app.clone() - .oneshot( - Request::post("/api/evolution/records/O-test-decide/decide") - .header("Content-Type", "application/json") - .header("X-Temper-Principal-Kind", "admin") - .body(Body::from(r#"{"decision":"approved","decided_by":"alice@example.com","rationale":"Looks good"}"#)) - .unwrap(), - ) - .await - .unwrap(); + let response = app + .clone() + .oneshot(system_post( + "/api/evolution/records/O-test-decide/decide", + r#"{"decision":"approved","decided_by":"alice@example.com","rationale":"Looks good"}"#, + )) + .await + .unwrap(); assert_eq!(response.status(), StatusCode::OK); let body = axum::body::to_bytes(response.into_body(), 1024 * 1024) @@ -1861,15 +2188,10 @@ async fn test_evolution_decide_not_found() { let app = build_app_with_state(state); let response = app - .oneshot( - Request::post("/api/evolution/records/O-nonexistent/decide") - .header("Content-Type", "application/json") - .header("X-Temper-Principal-Kind", "admin") - .body(Body::from( - r#"{"decision":"rejected","decided_by":"bob","rationale":"nope"}"#, - )) - .unwrap(), - ) + .oneshot(system_post( + "/api/evolution/records/O-nonexistent/decide", + r#"{"decision":"rejected","decided_by":"bob","rationale":"nope"}"#, + )) .await .unwrap(); @@ -1881,14 +2203,7 @@ async fn test_evolution_decide_not_found() { #[tokio::test] async fn test_workflows_returns_tenant_data() { let app = build_test_app(); - let response = app - .oneshot( - Request::get("/observe/workflows") - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); + let response = app.oneshot(system_get("/observe/workflows")).await.unwrap(); assert_eq!(response.status(), StatusCode::OK); let body = axum::body::to_bytes(response.into_body(), 1024 * 1024) @@ -1914,250 +2229,6 @@ async fn test_workflows_returns_tenant_data() { assert_eq!(steps[6]["step"], "deployed"); } -// -- Load-dir endpoint tests -- - -#[tokio::test] -async fn test_load_dir_registers_specs() { - let system = ActorSystem::new("test-load-dir"); - let registry = SpecRegistry::new(); - let state = ServerState::from_registry(system, registry); - - let app = Router::new() - .nest("/observe", build_observe_router()) - .nest("/api", crate::api::build_api_router()) - .with_state(state.clone()); - - // Use the test-fixtures/specs directory which has valid specs - let specs_dir = - std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../test-fixtures/specs"); - - let body = serde_json::json!({ - "tenant": "test-tenant", - "specs_dir": specs_dir.to_str().unwrap(), - }); - - let response = app - .oneshot( - Request::post("/api/specs/load-dir") - .header("Content-Type", "application/json") - .body(Body::from(serde_json::to_string(&body).unwrap())) - .unwrap(), - ) - .await - .unwrap(); - - assert_eq!(response.status(), StatusCode::OK); - // Response is NDJSON — parse each line - let body = axum::body::to_bytes(response.into_body(), 10 * 1024 * 1024) - .await - .unwrap(); - let body_str = std::str::from_utf8(&body).unwrap(); - let lines: Vec = body_str - .lines() - .filter(|l| !l.is_empty()) - .map(|l| serde_json::from_str(l).unwrap()) - .collect(); - - // First line: specs_loaded - assert_eq!(lines[0]["type"], "specs_loaded"); - assert_eq!(lines[0]["tenant"], "test-tenant"); - let entities = lines[0]["entities"].as_array().unwrap(); - assert!( - !entities.is_empty(), - "should have loaded at least one entity" - ); - - // Last line: summary - let summary = lines.last().unwrap(); - assert_eq!(summary["type"], "summary"); - assert_eq!(summary["tenant"], "test-tenant"); - - // Verify specs are in the registry - let registry = state.registry.read().unwrap(); - let tenant_id: temper_runtime::tenant::TenantId = "test-tenant".into(); - let entity_types = registry.entity_types(&tenant_id); - assert!( - !entity_types.is_empty(), - "registry should have entity types for test-tenant" - ); -} - -#[tokio::test] -async fn test_load_dir_missing_dir_returns_error() { - let system = ActorSystem::new("test-load-dir-missing"); - let registry = SpecRegistry::new(); - let state = ServerState::from_registry(system, registry); - - let app = Router::new() - .nest("/observe", build_observe_router()) - .nest("/api", crate::api::build_api_router()) - .with_state(state); - - let body = serde_json::json!({ - "tenant": "test-tenant", - "specs_dir": "/nonexistent/path/to/specs", - }); - - let response = app - .oneshot( - Request::post("/api/specs/load-dir") - .header("Content-Type", "application/json") - .body(Body::from(serde_json::to_string(&body).unwrap())) - .unwrap(), - ) - .await - .unwrap(); - - assert_eq!(response.status(), StatusCode::BAD_REQUEST); -} - -#[tokio::test] -async fn test_load_dir_lint_error_aborts_registration() { - let system = ActorSystem::new("test-load-dir-lint-error"); - let registry = SpecRegistry::new(); - let state = ServerState::from_registry(system, registry); - - let app = Router::new() - .nest("/observe", build_observe_router()) - .nest("/api", crate::api::build_api_router()) - .with_state(state.clone()); - - let temp_specs = - std::env::temp_dir().join(format!("temper-load-dir-lint-{}", uuid::Uuid::new_v4())); // determinism-ok: test-only temp dir - std::fs::create_dir_all(&temp_specs).expect("create temp specs dir"); // determinism-ok: test-only - std::fs::write( - // determinism-ok: test-only - temp_specs.join("model.csdl.xml"), - include_str!("../../../../test-fixtures/specs/model.csdl.xml"), - ) - .expect("write csdl"); - std::fs::write( - // determinism-ok: test-only - temp_specs.join("order.ioa.toml"), - r#" -[automaton] -name = "Order" -states = ["Draft", "Done"] -initial = "Draft" - -[[action]] -name = "Complete" -from = ["Draft"] -to = "Done" -effect = "set phantom true" -"#, - ) - .expect("write ioa"); - - let body = serde_json::json!({ - "tenant": "lint-tenant", - "specs_dir": temp_specs.to_str().unwrap(), - }); - - let response = app - .oneshot( - Request::post("/api/specs/load-dir") - .header("Content-Type", "application/json") - .body(Body::from(serde_json::to_string(&body).unwrap())) - .unwrap(), - ) - .await - .unwrap(); - - let _ = std::fs::remove_dir_all(&temp_specs); // determinism-ok: test-only - - assert_eq!(response.status(), StatusCode::BAD_REQUEST); - let body = axum::body::to_bytes(response.into_body(), 1024 * 1024) - .await - .unwrap(); - let body_str = std::str::from_utf8(&body).unwrap(); - let lines: Vec = body_str - .lines() - .filter(|l| !l.is_empty()) - .map(|l| serde_json::from_str(l).unwrap()) - .collect(); - - assert_eq!(lines[0]["type"], "specs_loaded"); - assert!(lines.iter().any(|l| l["type"] == "lint_error")); - assert!(!lines.iter().any(|l| l["type"] == "verification_started")); - - let registry = state.registry.read().unwrap(); - let tenant_id: temper_runtime::tenant::TenantId = "lint-tenant".into(); - assert!( - registry.get_tenant(&tenant_id).is_none(), - "tenant should not be registered when lint errors exist" - ); -} - -#[tokio::test] -async fn test_load_dir_emits_design_time_events() { - let db_url = format!( - "file:/tmp/temper-design-time-test-{}.db", - std::process::id(), - ); - let turso = TursoEventStore::new(&db_url, None) - .await - .expect("create local turso db"); - let system = ActorSystem::new("test-load-dir-events"); - let registry = SpecRegistry::new(); - let mut state = ServerState::from_registry(system, registry); - state.set_storage_stack(StorageStack::from_turso(turso)); - - let app = Router::new() - .nest("/observe", build_observe_router()) - .nest("/api", crate::api::build_api_router()) - .with_state(state.clone()); - - let specs_dir = - std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../test-fixtures/specs"); - - let body = serde_json::json!({ - "tenant": "event-tenant", - "specs_dir": specs_dir.to_str().unwrap(), - }); - - let response = app - .oneshot( - Request::post("/api/specs/load-dir") - .header("Content-Type", "application/json") - .body(Body::from(serde_json::to_string(&body).unwrap())) - .unwrap(), - ) - .await - .unwrap(); - - assert_eq!(response.status(), StatusCode::OK); - - // Consume entire body to wait for verification to complete - let _ = axum::body::to_bytes(response.into_body(), 10 * 1024 * 1024) - .await - .unwrap(); - - // Check that design-time events were persisted to Turso. - let turso = state.platform_turso_store().expect("turso configured"); - let events = turso - .list_design_time_events(None, 1000) - .await - .expect("query design-time events from Turso"); - assert!(!events.is_empty(), "design-time events should be in Turso"); - - // Should have spec_loaded, verify_started, and verify_done events - let loaded_events: Vec<_> = events.iter().filter(|e| e.kind == "spec_loaded").collect(); - assert!(!loaded_events.is_empty(), "should have spec_loaded events"); - - let started_events: Vec<_> = events - .iter() - .filter(|e| e.kind == "verify_started") - .collect(); - assert!( - !started_events.is_empty(), - "should have verify_started events" - ); - - let done_events: Vec<_> = events.iter().filter(|e| e.kind == "verify_done").collect(); - assert!(!done_events.is_empty(), "should have verify_done events"); -} - #[tokio::test] async fn test_evolution_insights_empty() { let app = build_test_app(); @@ -2176,3 +2247,6 @@ async fn test_evolution_insights_empty() { let insights = json["insights"].as_array().unwrap(); assert!(insights.is_empty()); } + +#[path = "mod_test/load_dir_tests.rs"] +mod load_dir_tests; diff --git a/crates/temper-server/src/observe/mod_test/load_dir_tests.rs b/crates/temper-server/src/observe/mod_test/load_dir_tests.rs new file mode 100644 index 000000000..688def799 --- /dev/null +++ b/crates/temper-server/src/observe/mod_test/load_dir_tests.rs @@ -0,0 +1,428 @@ +use super::*; + +// -- Load-dir endpoint tests -- + +#[tokio::test] +async fn test_load_dir_registers_specs() { + let system = ActorSystem::new("test-load-dir"); + let registry = SpecRegistry::new(); + let state = ServerState::from_registry(system, registry); + + let app = Router::new() + .nest("/observe", build_observe_router()) + .nest("/api", crate::api::build_api_router()) + .with_state(state.clone()); + + // Use the test-fixtures/specs directory which has valid specs + let specs_dir = + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../test-fixtures/specs"); + + let body = serde_json::json!({ + "tenant": "test-tenant", + "specs_dir": specs_dir.to_str().unwrap(), + }); + + let response = app + .oneshot(with_tenant_security_context( + Request::post("/api/specs/load-dir") + .header("Content-Type", "application/json") + .body(Body::from(serde_json::to_string(&body).unwrap())) + .unwrap(), + "test-tenant", + temper_authz::SecurityContext::system(), + )) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + // Response is NDJSON — parse each line + let body = axum::body::to_bytes(response.into_body(), 10 * 1024 * 1024) + .await + .unwrap(); + let body_str = std::str::from_utf8(&body).unwrap(); + let lines: Vec = body_str + .lines() + .filter(|l| !l.is_empty()) + .map(|l| serde_json::from_str(l).unwrap()) + .collect(); + + // First line: specs_loaded + assert_eq!(lines[0]["type"], "specs_loaded"); + assert_eq!(lines[0]["tenant"], "test-tenant"); + let entities = lines[0]["entities"].as_array().unwrap(); + assert!( + !entities.is_empty(), + "should have loaded at least one entity" + ); + + // Last line: summary + let summary = lines.last().unwrap(); + assert_eq!(summary["type"], "summary"); + assert_eq!(summary["tenant"], "test-tenant"); + + // Verify specs are in the registry + let registry = state.registry.read().unwrap(); + let tenant_id: temper_runtime::tenant::TenantId = "test-tenant".into(); + let entity_types = registry.entity_types(&tenant_id); + assert!( + !entity_types.is_empty(), + "registry should have entity types for test-tenant" + ); +} + +#[tokio::test] +async fn test_load_dir_missing_dir_returns_error() { + let system = ActorSystem::new("test-load-dir-missing"); + let registry = SpecRegistry::new(); + let state = ServerState::from_registry(system, registry); + + let app = Router::new() + .nest("/observe", build_observe_router()) + .nest("/api", crate::api::build_api_router()) + .with_state(state); + + let body = serde_json::json!({ + "tenant": "test-tenant", + "specs_dir": "/nonexistent/path/to/specs", + }); + + let response = app + .oneshot(with_tenant_security_context( + Request::post("/api/specs/load-dir") + .header("Content-Type", "application/json") + .body(Body::from(serde_json::to_string(&body).unwrap())) + .unwrap(), + "test-tenant", + temper_authz::SecurityContext::system(), + )) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::BAD_REQUEST); +} + +#[tokio::test] +async fn test_load_dir_rejects_missing_and_cross_tenant_authority_before_fs_probe() { + let state = ServerState::from_registry( + ActorSystem::new("test-load-dir-auth-boundary"), + SpecRegistry::new(), + ); + let app = Router::new() + .nest("/api", crate::api::build_api_router()) + .with_state(state); + let body = serde_json::json!({ + "tenant": "victim", + "specs_dir": "/a/path/whose/existence/must/not/be-probed", + }); + let request = || { + Request::post("/api/specs/load-dir") + .header("Content-Type", "application/json") + .body(Body::from(serde_json::to_string(&body).unwrap())) + .unwrap() + }; + + let missing_context = app + .clone() + .oneshot(request()) + .await + .expect("request should run"); + assert_eq!(missing_context.status(), StatusCode::UNAUTHORIZED); + + let same_tenant_denied = app + .clone() + .oneshot(with_tenant_security_context( + request(), + "victim", + temper_authz::SecurityContext::from_resolved_identity( + "agent-1", + "swe", + Some("session-1"), + ), + )) + .await + .expect("request should run"); + assert_eq!(same_tenant_denied.status(), StatusCode::FORBIDDEN); + + let wrong_tenant = app + .oneshot(with_tenant_security_context( + request(), + "attacker", + temper_authz::SecurityContext::from_resolved_identity( + "agent-1", + "swe", + Some("session-1"), + ), + )) + .await + .expect("request should run"); + assert_eq!(wrong_tenant.status(), StatusCode::UNAUTHORIZED); +} + +#[tokio::test] +async fn test_load_dir_agent_needs_exact_directory_cedar_authority() { + let state = ServerState::from_registry( + ActorSystem::new("test-load-dir-exact-authz"), + SpecRegistry::new(), + ); + let app = Router::new() + .nest("/api", crate::api::build_api_router()) + .with_state(state.clone()); + let specs_dir = + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../test-fixtures/specs"); + let canonical = std::fs::canonicalize(&specs_dir).expect("fixture path should canonicalize"); + let body = serde_json::json!({ + "tenant": "test-tenant", + "specs_dir": canonical.to_str().unwrap(), + }); + let request = || { + with_tenant_security_context( + Request::post("/api/specs/load-dir") + .header("Content-Type", "application/json") + .body(Body::from(serde_json::to_string(&body).unwrap())) + .unwrap(), + "test-tenant", + temper_authz::SecurityContext::from_resolved_identity( + "agent-1", + "swe", + Some("session-1"), + ), + ) + }; + + let default_denied = app + .clone() + .oneshot(request()) + .await + .expect("request should run"); + assert_eq!(default_denied.status(), StatusCode::FORBIDDEN); + + let resource_id = serde_json::to_string(canonical.to_str().unwrap()).unwrap(); + state + .authz + .reload_tenant_policies( + "test-tenant", + &format!( + r#" +permit( + principal == Agent::"agent-1", + action == Action::"load_specs_from_directory", + resource == SpecDirectory::{resource_id} +); +"#, + ), + ) + .expect("exact directory policy should parse"); + let allowed = app.oneshot(request()).await.expect("request should run"); + assert_eq!(allowed.status(), StatusCode::OK); +} + +#[cfg(unix)] +#[tokio::test] +async fn test_load_dir_rejects_symlinked_directory_and_spec_file() { + use std::os::unix::fs::symlink; + + let state = ServerState::from_registry( + ActorSystem::new("test-load-dir-symlink"), + SpecRegistry::new(), + ); + let app = Router::new() + .nest("/api", crate::api::build_api_router()) + .with_state(state); + let fixture_dir = + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../test-fixtures/specs"); + let staging = tempfile::tempdir().expect("temp directory should build"); + let directory_link = staging.path().join("linked-specs"); + symlink(&fixture_dir, &directory_link).expect("directory symlink should build"); + let request_for = |specs_dir: &std::path::Path| { + with_tenant_security_context( + Request::post("/api/specs/load-dir") + .header("Content-Type", "application/json") + .body(Body::from( + serde_json::json!({ + "tenant": "test-tenant", + "specs_dir": specs_dir.to_str().unwrap(), + }) + .to_string(), + )) + .unwrap(), + "test-tenant", + temper_authz::SecurityContext::system(), + ) + }; + + let linked_directory = app + .clone() + .oneshot(request_for(&directory_link)) + .await + .expect("request should run"); + assert_eq!(linked_directory.status(), StatusCode::BAD_REQUEST); + + let file_link_root = staging.path().join("file-link-root"); + std::fs::create_dir(&file_link_root).expect("spec root should build"); + symlink( + fixture_dir.join("model.csdl.xml"), + file_link_root.join("model.csdl.xml"), + ) + .expect("model symlink should build"); + std::fs::copy( + fixture_dir.join("order.ioa.toml"), + file_link_root.join("order.ioa.toml"), + ) + .expect("IOA fixture should copy"); + let linked_file = app + .oneshot(request_for(&file_link_root)) + .await + .expect("request should run"); + assert_eq!(linked_file.status(), StatusCode::BAD_REQUEST); +} + +#[tokio::test] +async fn test_load_dir_lint_error_aborts_registration() { + let system = ActorSystem::new("test-load-dir-lint-error"); + let registry = SpecRegistry::new(); + let state = ServerState::from_registry(system, registry); + + let app = Router::new() + .nest("/observe", build_observe_router()) + .nest("/api", crate::api::build_api_router()) + .with_state(state.clone()); + + let temp_specs = + std::env::temp_dir().join(format!("temper-load-dir-lint-{}", uuid::Uuid::new_v4())); // determinism-ok: test-only temp dir + std::fs::create_dir_all(&temp_specs).expect("create temp specs dir"); // determinism-ok: test-only + std::fs::write( + // determinism-ok: test-only + temp_specs.join("model.csdl.xml"), + include_str!("../../../../../test-fixtures/specs/model.csdl.xml"), + ) + .expect("write csdl"); + std::fs::write( + // determinism-ok: test-only + temp_specs.join("order.ioa.toml"), + r#" +[automaton] +name = "Order" +states = ["Draft", "Done"] +initial = "Draft" + +[[action]] +name = "Complete" +from = ["Draft"] +to = "Done" +effect = "set phantom true" +"#, + ) + .expect("write ioa"); + + let body = serde_json::json!({ + "tenant": "lint-tenant", + "specs_dir": temp_specs.to_str().unwrap(), + }); + + let response = app + .oneshot(with_tenant_security_context( + Request::post("/api/specs/load-dir") + .header("Content-Type", "application/json") + .body(Body::from(serde_json::to_string(&body).unwrap())) + .unwrap(), + "lint-tenant", + temper_authz::SecurityContext::system(), + )) + .await + .unwrap(); + + let _ = std::fs::remove_dir_all(&temp_specs); // determinism-ok: test-only + + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + let body = axum::body::to_bytes(response.into_body(), 1024 * 1024) + .await + .unwrap(); + let body_str = std::str::from_utf8(&body).unwrap(); + let lines: Vec = body_str + .lines() + .filter(|l| !l.is_empty()) + .map(|l| serde_json::from_str(l).unwrap()) + .collect(); + + assert_eq!(lines[0]["type"], "specs_loaded"); + assert!(lines.iter().any(|l| l["type"] == "lint_error")); + assert!(!lines.iter().any(|l| l["type"] == "verification_started")); + + let registry = state.registry.read().unwrap(); + let tenant_id: temper_runtime::tenant::TenantId = "lint-tenant".into(); + assert!( + registry.get_tenant(&tenant_id).is_none(), + "tenant should not be registered when lint errors exist" + ); +} + +#[tokio::test] +async fn test_load_dir_emits_design_time_events() { + let db_url = format!( + "file:/tmp/temper-design-time-test-{}.db", + std::process::id(), + ); + let turso = TursoEventStore::new(&db_url, None) + .await + .expect("create local turso db"); + let system = ActorSystem::new("test-load-dir-events"); + let registry = SpecRegistry::new(); + let mut state = ServerState::from_registry(system, registry); + state.set_storage_stack(StorageStack::from_turso(turso)); + + let app = Router::new() + .nest("/observe", build_observe_router()) + .nest("/api", crate::api::build_api_router()) + .with_state(state.clone()); + + let specs_dir = + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../test-fixtures/specs"); + + let body = serde_json::json!({ + "tenant": "event-tenant", + "specs_dir": specs_dir.to_str().unwrap(), + }); + + let response = app + .oneshot(with_tenant_security_context( + Request::post("/api/specs/load-dir") + .header("Content-Type", "application/json") + .body(Body::from(serde_json::to_string(&body).unwrap())) + .unwrap(), + "event-tenant", + temper_authz::SecurityContext::system(), + )) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + + // Consume entire body to wait for verification to complete + let _ = axum::body::to_bytes(response.into_body(), 10 * 1024 * 1024) + .await + .unwrap(); + + // Check that design-time events were persisted to Turso. + let turso = state.platform_turso_store().expect("turso configured"); + let events = turso + .list_design_time_events(None, 1000) + .await + .expect("query design-time events from Turso"); + assert!(!events.is_empty(), "design-time events should be in Turso"); + + // Should have spec_loaded, verify_started, and verify_done events + let loaded_events: Vec<_> = events.iter().filter(|e| e.kind == "spec_loaded").collect(); + assert!(!loaded_events.is_empty(), "should have spec_loaded events"); + + let started_events: Vec<_> = events + .iter() + .filter(|e| e.kind == "verify_started") + .collect(); + assert!( + !started_events.is_empty(), + "should have verify_started events" + ); + + let done_events: Vec<_> = events.iter().filter(|e| e.kind == "verify_done").collect(); + assert!(!done_events.is_empty(), "should have verify_done events"); +} diff --git a/crates/temper-server/src/observe/projections.rs b/crates/temper-server/src/observe/projections.rs index b41a5ae20..7cf1bcea6 100644 --- a/crates/temper-server/src/observe/projections.rs +++ b/crates/temper-server/src/observe/projections.rs @@ -1,13 +1,13 @@ //! Projection correctness observe endpoints. -use axum::extract::{Query, State}; -use axum::http::{HeaderMap, StatusCode}; +use axum::extract::{Extension, Query, State}; +use axum::http::StatusCode; use axum::response::Json; use serde::Deserialize; +use temper_authz::AuthenticatedRequestContext; use tracing::instrument; -use crate::authz::require_observe_auth; -use crate::odata::extract_tenant; +use crate::authz::{require_authenticated_context, require_observe_auth}; use crate::state::ServerState; const DEFAULT_REPLAY_PARITY_LIMIT: usize = 100; @@ -36,11 +36,12 @@ pub(crate) struct ReplayParityParams { )] pub(crate) async fn handle_replay_parity( State(state): State, - headers: HeaderMap, + authenticated: Option>, Query(params): Query, ) -> Result, StatusCode> { - require_observe_auth(&state, &headers, "read_entities", "Projection")?; - let tenant = extract_tenant(&headers, &state).map_err(|(code, _)| code)?; + let authenticated = require_authenticated_context(authenticated.as_deref())?; + require_observe_auth(&state, authenticated, "read_entities", "Projection")?; + let tenant = authenticated.tenant(); let entity_type = params .entity_type .as_deref() @@ -57,7 +58,7 @@ pub(crate) async fn handle_replay_parity( let report = state .verify_query_projection_replay_parity_bounded( - &tenant, + tenant, entity_type, Some(limit), "observe_probe", diff --git a/crates/temper-server/src/observe/specs.rs b/crates/temper-server/src/observe/specs.rs index dab48e848..8b6584fc9 100644 --- a/crates/temper-server/src/observe/specs.rs +++ b/crates/temper-server/src/observe/specs.rs @@ -1,14 +1,17 @@ //! Spec management endpoints: list, load, and inspect IOA specifications. -use axum::extract::{Path, State}; -use axum::http::{HeaderMap, StatusCode}; +use axum::extract::{Extension, Path, State}; +use axum::http::StatusCode; use axum::response::Json; +use temper_authz::AuthenticatedRequestContext; -use crate::authz::{observe_tenant_scope, require_observe_auth}; +use crate::authz::{observe_tenant_scope, require_authenticated_context, require_observe_auth}; use crate::registry::VerificationStatus; use crate::state::ServerState; -use super::{ActionDetail, InvariantDetail, SpecDetail, SpecSummary, StateVarDetail}; +use super::{ + ActionDetail, ActionParamDetail, InvariantDetail, SpecDetail, SpecSummary, StateVarDetail, +}; mod load_dir; mod load_inline; @@ -20,20 +23,19 @@ pub(crate) use load_dir::handle_load_dir; pub(crate) use load_inline::handle_load_inline; pub(crate) use validate_ioa::handle_validate_ioa; -/// GET /observe/specs -- list all loaded specs across all tenants. +/// GET /observe/specs -- list loaded specs in the credential-bound tenant. pub(crate) async fn handle_list_specs( State(state): State, - headers: HeaderMap, + authenticated: Option>, ) -> Result, StatusCode> { - require_observe_auth(&state, &headers, "read_specs", "Spec")?; - let tenant_scope = observe_tenant_scope(&state, &headers)?; + let authenticated = require_authenticated_context(authenticated.as_deref())?; + require_observe_auth(&state, authenticated, "read_specs", "Spec")?; + let tenant_scope = observe_tenant_scope(authenticated); let registry = state.registry.read().unwrap(); // ci-ok: infallible lock let mut specs = Vec::new(); for tenant_id in registry.tenant_ids() { - if let Some(ref scope) = tenant_scope - && tenant_id != scope - { + if tenant_id != tenant_scope { continue; } for entity_type in registry.entity_types(tenant_id) { @@ -83,26 +85,32 @@ pub(crate) async fn handle_list_specs( /// GET /observe/specs/{entity} -- full spec detail for a named entity type. /// -/// Searches across all tenants and returns the first match. +/// Searches only the credential-bound tenant. +/// +/// Carries `spec_version`, the content hash of the returned spec's IOA source. +/// That is the digest a conformance check compares a run's recorded +/// `metadata.spec_version` against, so a harness that reads it here records a +/// version the kernel will recognise, rather than one it computed from a file +/// some deploy path may have rewritten. pub(crate) async fn handle_get_spec_detail( State(state): State, - headers: HeaderMap, + authenticated: Option>, Path(entity): Path, ) -> Result, StatusCode> { - require_observe_auth(&state, &headers, "read_specs", "Spec")?; - let tenant_scope = observe_tenant_scope(&state, &headers)?; + let authenticated = require_authenticated_context(authenticated.as_deref())?; + require_observe_auth(&state, authenticated, "read_specs", "Spec")?; + let tenant_scope = observe_tenant_scope(authenticated); let registry = state.registry.read().unwrap(); // ci-ok: infallible lock for tenant_id in registry.tenant_ids() { - if let Some(ref scope) = tenant_scope - && tenant_id != scope - { + if tenant_id != tenant_scope { continue; } if let Some(entity_spec) = registry.get_spec(tenant_id, &entity) { let automaton = &entity_spec.automaton; let detail = SpecDetail { entity_type: entity.clone(), + spec_version: temper_store_turso::spec_content_hash(&entity_spec.ioa_source), states: automaton.automaton.states.clone(), initial_state: automaton.automaton.initial.clone(), actions: automaton @@ -115,6 +123,15 @@ pub(crate) async fn handle_get_spec_detail( to: a.to.clone(), guards: a.guard.iter().map(|g| format!("{g:?}")).collect(), effects: a.effect.iter().map(|e| format!("{e:?}")).collect(), + params: a + .params + .iter() + .map(|p| ActionParamDetail { + name: p.name().to_string(), + param_type: p.param_type().to_string(), + }) + .collect(), + hint: a.hint.clone().unwrap_or_default(), }) .collect(), invariants: automaton diff --git a/crates/temper-server/src/observe/specs/load_dir.rs b/crates/temper-server/src/observe/specs/load_dir.rs index ca3500398..6169f6aba 100644 --- a/crates/temper-server/src/observe/specs/load_dir.rs +++ b/crates/temper-server/src/observe/specs/load_dir.rs @@ -1,6 +1,12 @@ -use axum::extract::State; +use std::collections::BTreeMap; +use std::fs; // determinism-ok: authenticated server-local spec management boundary +use std::io::Read as _; +use std::path::{Path, PathBuf}; + +use axum::extract::{Extension, State}; use axum::http::StatusCode; use axum::response::Json; +use temper_authz::AuthenticatedRequestContext; use temper_spec::automaton::LintSeverity; use temper_spec::cross_invariant::{ CrossInvariantLintSeverity, lint_cross_invariants, parse_cross_invariants, @@ -13,42 +19,216 @@ use super::super::specs_helpers::{ }; use super::types::LoadDirRequest; use super::verification_stream::build_verification_stream_response; +use crate::authz::{ + GovernedMutationAuth, require_authenticated_context, require_governed_mutation_auth, + require_tenant_match, +}; use crate::state::ServerState; -/// POST /api/specs/load-dir -- hot-load specs from a directory into the running server./// +const SPEC_DIRECTORY_PATH_BUDGET: usize = 4 * 1024; +const SPEC_DIRECTORY_ENTRY_BUDGET: usize = 512; +const SPEC_FILE_COUNT_BUDGET: usize = 128; +const SPEC_FILE_BYTE_BUDGET: usize = 1024 * 1024; +const SPEC_DIRECTORY_BYTE_BUDGET: usize = 16 * 1024 * 1024; + +pub(super) struct ValidatedSpecDirectory { + path: PathBuf, + resource_id: String, +} + +fn validate_spec_directory_request(requested: &str) -> Result<(), (StatusCode, String)> { + if requested.trim().is_empty() || requested.len() > SPEC_DIRECTORY_PATH_BUDGET { + return Err(( + StatusCode::BAD_REQUEST, + "Specs directory path is empty or exceeds its budget".to_string(), + )); + } + Ok(()) +} + +pub(super) fn validate_spec_directory( + requested: &str, +) -> Result { + validate_spec_directory_request(requested)?; + let requested_path = Path::new(requested); + let metadata = fs::symlink_metadata(requested_path).map_err(|error| { + ( + StatusCode::BAD_REQUEST, + format!("Specs directory is unavailable: {error}"), + ) + })?; + if metadata.file_type().is_symlink() || !metadata.is_dir() { + return Err(( + StatusCode::BAD_REQUEST, + "Specs directory must be a real directory, not a file or symbolic link".to_string(), + )); + } + let path = fs::canonicalize(requested_path).map_err(|error| { + ( + StatusCode::BAD_REQUEST, + format!("Specs directory cannot be canonicalized: {error}"), + ) + })?; + let resource_id = path.to_string_lossy().to_string(); + if resource_id.len() > SPEC_DIRECTORY_PATH_BUDGET { + return Err(( + StatusCode::BAD_REQUEST, + "Canonical specs directory path exceeds its budget".to_string(), + )); + } + Ok(ValidatedSpecDirectory { path, resource_id }) +} + +/// POST /api/specs/load-dir -- hot-load specs from a server-local directory. /// Reads CSDL and IOA files from `specs_dir`, registers them under `tenant`, /// emits design-time SSE events for each entity, and spawns background -/// verification tasks that stream progress via SSE. +/// verification tasks that stream progress via SSE. The dedicated +/// `load_specs_from_directory` Cedar action is separate from inline spec +/// submission because it grants access to a host filesystem path. #[instrument(skip_all, fields(otel.name = "POST /api/specs/load-dir"))] pub(crate) async fn handle_load_dir( State(state): State, + authenticated: Option>, Json(body): Json, ) -> Result { - let specs_path = std::path::Path::new(&body.specs_dir); + let authenticated = require_authenticated_context(authenticated.as_deref()) + .map_err(|status| (status, "authentication required".to_string()))?; + require_tenant_match(authenticated, &body.tenant) + .map_err(|status| (status, "credential tenant mismatch".to_string()))?; + validate_spec_directory_request(&body.specs_dir)?; + + // Authorize the caller-controlled path before asking the host filesystem + // whether it exists. A second check below binds authority to the canonical + // target so aliases and symbolic links cannot redirect a permitted path. + let requested_resource_attrs = BTreeMap::from([ + ( + "id".to_string(), + serde_json::Value::String(body.specs_dir.clone()), + ), + ( + "targetTenant".to_string(), + serde_json::Value::String(body.tenant.clone()), + ), + ( + "requestedPath".to_string(), + serde_json::Value::String(body.specs_dir.clone()), + ), + ]); + if let Some(denial) = require_governed_mutation_auth( + &state, + authenticated, + GovernedMutationAuth { + tenant: &body.tenant, + action: "load_specs_from_directory", + resource_type: "SpecDirectory", + resource_id: &body.specs_dir, + resource_attrs: requested_resource_attrs, + module_name: None, + from_status: None, + }, + ) + .await + { + return Err(denial); + } + + let directory = validate_spec_directory(&body.specs_dir)?; + let resource_attrs = BTreeMap::from([ + ( + "id".to_string(), + serde_json::Value::String(directory.resource_id.clone()), + ), + ( + "targetTenant".to_string(), + serde_json::Value::String(body.tenant.clone()), + ), + ( + "canonicalPath".to_string(), + serde_json::Value::String(directory.resource_id.clone()), + ), + ]); + if let Some(denial) = require_governed_mutation_auth( + &state, + authenticated, + GovernedMutationAuth { + tenant: &body.tenant, + action: "load_specs_from_directory", + resource_type: "SpecDirectory", + resource_id: &directory.resource_id, + resource_attrs, + module_name: None, + from_status: None, + }, + ) + .await + { + return Err(denial); + } + load_specs_from_directory(state, body, directory).await +} - if !specs_path.is_dir() { +fn read_bounded_spec_text( + path: &Path, + label: &str, + remaining_bytes: &mut usize, +) -> Result { + let metadata = fs::symlink_metadata(path).map_err(|error| { + ( + StatusCode::BAD_REQUEST, + format!("{label} is unavailable: {error}"), + ) + })?; + if metadata.file_type().is_symlink() || !metadata.is_file() { return Err(( StatusCode::BAD_REQUEST, - format!("Specs directory not found: {}", specs_path.display()), + format!("{label} must be a regular file, not a symbolic link"), )); } - - // Read CSDL model - let csdl_path = specs_path.join("model.csdl.xml"); - if !csdl_path.exists() { + let byte_length = usize::try_from(metadata.len()).unwrap_or(usize::MAX); + if byte_length > SPEC_FILE_BYTE_BUDGET || byte_length > *remaining_bytes { return Err(( - StatusCode::BAD_REQUEST, - format!("CSDL model not found at {}", csdl_path.display()), + StatusCode::PAYLOAD_TOO_LARGE, + format!("{label} exceeds the spec-loading byte budget"), )); } - - let csdl_xml = std::fs::read_to_string(&csdl_path).map_err(|e| { - // determinism-ok: HTTP handler reads spec files + let read_budget = SPEC_FILE_BYTE_BUDGET.min(*remaining_bytes); + let file = fs::File::open(path).map_err(|error| { ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Failed to read CSDL: {e}"), + StatusCode::BAD_REQUEST, + format!("Failed to open {label}: {error}"), ) })?; + let mut content = String::new(); + file.take((read_budget as u64).saturating_add(1)) + .read_to_string(&mut content) + .map_err(|error| { + ( + StatusCode::BAD_REQUEST, + format!("Failed to read UTF-8 {label}: {error}"), + ) + })?; + if content.len() > byte_length || content.len() > read_budget { + return Err(( + StatusCode::PAYLOAD_TOO_LARGE, + format!("{label} changed or exceeded the spec-loading byte budget while reading"), + )); + } + *remaining_bytes -= content.len(); + Ok(content) +} + +pub(super) async fn load_specs_from_directory( + state: ServerState, + mut body: LoadDirRequest, + directory: ValidatedSpecDirectory, +) -> Result { + let specs_path = &directory.path; + body.specs_dir = directory.resource_id; + let mut remaining_bytes = SPEC_DIRECTORY_BYTE_BUDGET; + + // Read CSDL model + let csdl_path = specs_path.join("model.csdl.xml"); + let csdl_xml = read_bounded_spec_text(&csdl_path, "model.csdl.xml", &mut remaining_bytes)?; let csdl = temper_spec::csdl::parse_csdl(&csdl_xml).map_err(|e| { ( StatusCode::BAD_REQUEST, @@ -59,13 +239,14 @@ pub(crate) async fn handle_load_dir( // Read all *.ioa.toml files let mut ioa_sources: std::collections::BTreeMap = std::collections::BTreeMap::new(); - let entries = std::fs::read_dir(specs_path).map_err(|e| { + let entries = fs::read_dir(specs_path).map_err(|e| { // determinism-ok: HTTP handler reads spec directory ( StatusCode::INTERNAL_SERVER_ERROR, format!("Failed to read specs directory: {e}"), ) })?; + let mut paths = Vec::new(); for entry in entries { let entry = entry.map_err(|e| { ( @@ -73,22 +254,42 @@ pub(crate) async fn handle_load_dir( format!("Failed to read directory entry: {e}"), ) })?; - let path = entry.path(); + if paths.len() >= SPEC_DIRECTORY_ENTRY_BUDGET { + return Err(( + StatusCode::PAYLOAD_TOO_LARGE, + "Specs directory exceeds its entry budget".to_string(), + )); + } + paths.push(entry.path()); + } + paths.sort(); + let mut spec_file_count = 0usize; + for path in paths { let file_name = path .file_name() .and_then(|n| n.to_str()) .unwrap_or_default(); if file_name.ends_with(".ioa.toml") { + spec_file_count += 1; + if spec_file_count > SPEC_FILE_COUNT_BUDGET { + return Err(( + StatusCode::PAYLOAD_TOO_LARGE, + "Specs directory exceeds its IOA file-count budget".to_string(), + )); + } let entity_name = file_name.strip_suffix(".ioa.toml").unwrap_or_default(); let entity_name = to_pascal_case(entity_name); - let source = std::fs::read_to_string(&path).map_err(|e| { - // determinism-ok: HTTP handler reads spec files - ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Failed to read {}: {e}", path.display()), - ) - })?; - ioa_sources.insert(entity_name, source); + let source = read_bounded_spec_text( + &path, + &format!("IOA spec {}", path.display()), + &mut remaining_bytes, + )?; + if ioa_sources.insert(entity_name.clone(), source).is_some() { + return Err(( + StatusCode::BAD_REQUEST, + format!("Multiple IOA filenames normalize to entity {entity_name}"), + )); + } } } @@ -100,7 +301,7 @@ pub(crate) async fn handle_load_dir( } let legacy_reactions_path = specs_path.join("reactions.toml"); - if legacy_reactions_path.exists() { + if fs::symlink_metadata(&legacy_reactions_path).is_ok() { return Err(( StatusCode::BAD_REQUEST, format!( @@ -113,16 +314,19 @@ pub(crate) async fn handle_load_dir( // Optional cross-invariants.toml. let cross_invariants_toml = { let path = specs_path.join("cross-invariants.toml"); - if path.exists() { - Some(std::fs::read_to_string(&path).map_err(|e| { - // determinism-ok: HTTP handler reads cross-invariants - ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Failed to read {}: {e}", path.display()), - ) - })?) - } else { - None + match fs::symlink_metadata(&path) { + Ok(_) => Some(read_bounded_spec_text( + &path, + "cross-invariants.toml", + &mut remaining_bytes, + )?), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, + Err(error) => { + return Err(( + StatusCode::BAD_REQUEST, + format!("Unable to inspect cross-invariants.toml: {error}"), + )); + } } }; @@ -228,7 +432,7 @@ pub(crate) async fn handle_load_dir( let registry_path = state.data_dir.join("specs-registry.json"); let mut specs_registry = std::collections::BTreeMap::::new(); - if let Ok(content) = std::fs::read_to_string(®istry_path) { + if let Ok(content) = fs::read_to_string(®istry_path) { // determinism-ok: HTTP handler reads specs registry if let Ok(value) = serde_json::from_str::(&content) && let Some(obj) = value.as_object() @@ -244,8 +448,8 @@ pub(crate) async fn handle_load_dir( specs_registry.insert(body.tenant.clone(), body.specs_dir.clone()); if let Ok(encoded) = serde_json::to_string_pretty(&specs_registry) { - let _ = std::fs::create_dir_all(&state.data_dir); // determinism-ok: HTTP handler creates data dir - let _ = std::fs::write(registry_path, encoded); // determinism-ok: HTTP handler writes specs registry + let _ = fs::create_dir_all(&state.data_dir); + let _ = fs::write(registry_path, encoded); } } diff --git a/crates/temper-server/src/observe/specs/load_inline.rs b/crates/temper-server/src/observe/specs/load_inline.rs index 0eb24abf3..715f1c759 100644 --- a/crates/temper-server/src/observe/specs/load_inline.rs +++ b/crates/temper-server/src/observe/specs/load_inline.rs @@ -1,24 +1,25 @@ -use std::path::{Path, PathBuf}; +use std::fs; // determinism-ok: authenticated server-local spec staging boundary -use axum::extract::State; -use axum::http::HeaderMap; +use axum::extract::{Extension, State}; use axum::http::StatusCode; -use axum::response::Json; -use serde_json::json; +use temper_authz::AuthenticatedRequestContext; use temper_evolution::records::{ AnalysisRecord, ObservationClass, ObservationRecord, RecordHeader, RecordType, SolutionOption, }; use temper_runtime::scheduler::sim_now; -use temper_runtime::tenant::TenantId; -use temper_spec::csdl::parse_csdl; use tracing::instrument; -use crate::authz::{DenialInput, record_authz_denial, security_context_from_headers}; +use crate::authz::{ + DenialInput, record_authz_denial, require_authenticated_context, require_tenant_match, +}; use crate::state::{ServerState, TrajectoryEntry, TrajectorySource}; -use super::load_dir::handle_load_dir; +use super::load_dir::{load_specs_from_directory, validate_spec_directory}; use super::types::{LoadDirRequest, LoadInlineRequest}; +mod support; +use support::{build_adr_warning_context, resolve_inline_specs_root}; + /// POST /api/specs/load-inline -- load specs from inline content. /// /// Accepts a JSON body with `tenant` and `specs` (map of filename -> content). @@ -27,9 +28,11 @@ use super::types::{LoadDirRequest, LoadInlineRequest}; #[instrument(skip_all, fields(otel.name = "POST /api/specs/load-inline"))] pub(crate) async fn handle_load_inline( State(state): State, - headers: HeaderMap, + authenticated: Option>, raw_body: String, ) -> Result { + let authenticated = require_authenticated_context(authenticated.as_deref()) + .map_err(|status| (status, "authentication required".to_string()))?; let body: LoadInlineRequest = serde_json::from_str(&raw_body).map_err(|e| { ( StatusCode::BAD_REQUEST, @@ -39,9 +42,21 @@ pub(crate) async fn handle_load_inline( ) })?; let tenant = body.tenant.clone(); + require_tenant_match(authenticated, &tenant) + .map_err(|status| (status, "credential tenant mismatch".to_string()))?; + if body + .cedar_policies + .as_deref() + .is_some_and(|policy| !policy.trim().is_empty()) + { + return Err(( + StatusCode::BAD_REQUEST, + "Inline spec submission cannot activate Cedar policy text; use the tenant policy management API and its separate manage_policies authorization" + .to_string(), + )); + } - // Cedar authorization gate. - let security_ctx = security_context_from_headers(&headers, None, None, None); + let security_ctx = authenticated.security_context(); let entity_names: Vec = body .specs .keys() @@ -56,29 +71,25 @@ pub(crate) async fn handle_load_inline( && let Ok(automaton) = temper_spec::automaton::parse_automaton(spec_content) { let metadata = automaton.extract_metadata(); - for (k, v) in metadata.to_flat_map() { - spec_resource_attrs.insert(k, v); + for (key, value) in metadata.to_flat_map() { + spec_resource_attrs.insert(key, value); } } } - let authz_result = state.authorize_with_context( - &security_ctx, + if let Err(denial) = state.authorize_with_context( + security_ctx, "submit_specs", "SpecRegistry", &spec_resource_attrs, &tenant, - ); - if let Err(denial) = authz_result { + ) { let reason = denial.to_string(); - // Record denials and use the first decision ID as the primary chain anchor. - // Record a single denial per authz check. The resource_id must match - // what Cedar evaluated: SpecRegistry::"". - let pd = record_authz_denial( + let pending_decision = record_authz_denial( &state, DenialInput { tenant: &tenant, - security_ctx: &security_ctx, + security_ctx, agent_id_override: None, action: "submit_specs", resource_type: "SpecRegistry", @@ -87,14 +98,17 @@ pub(crate) async fn handle_load_inline( reason: &reason, module_name: None, from_status: None, + intent: authenticated.intent().map(str::to_string), + session_id: authenticated.session_id().map(str::to_string), + // Management-plane denial, not a spec-governed dispatch. + spec_governed: Some(false), }, ) .await; - let primary_decision_id = pd.id.clone(); - let decision_ids = vec![pd.id]; + let primary_decision_id = pending_decision.id.clone(); + let decision_ids = vec![pending_decision.id]; - // Create O-Record for the denied spec submission. - let o_record = ObservationRecord { + let observation = ObservationRecord { header: RecordHeader::new(RecordType::Observation, "cedar:spec_submission"), source: "cedar:spec_submission".to_string(), classification: ObservationClass::AuthzDenied, @@ -113,32 +127,31 @@ pub(crate) async fn handle_load_inline( "spec_metadata": spec_resource_attrs, }), }; - let o_id = o_record.header.id.clone(); - let data_json = serde_json::to_string(&o_record).unwrap_or_default(); + let observation_id = observation.header.id.clone(); + let data_json = serde_json::to_string(&observation).unwrap_or_default(); let _ = state - .insert_evolution_record( - &o_record.header.id, - "Observation", - &format!("{:?}", o_record.header.status), - &o_record.header.created_by, - o_record.header.derived_from.as_deref(), - &data_json, - ) + .insert_evolution_record(crate::storage::EvolutionRecordWrite { + tenant: &tenant, + id: &observation.header.id, + record_type: "Observation", + status: &format!("{:?}", observation.header.status), + created_by: &observation.header.created_by, + derived_from: observation.header.derived_from.as_deref(), + data_json: &data_json, + }) .await; - // Create A-Record with the proposed spec as spec_diff. - let spec_summary: String = body + let spec_summary = body .specs .keys() - .map(|k| k.as_str()) + .map(String::as_str) .collect::>() .join(", "); - let a_record = AnalysisRecord { + let analysis = AnalysisRecord { header: RecordHeader::new(RecordType::Analysis, "cedar:spec_submission") - .derived_from(o_id), + .derived_from(observation_id), root_cause: format!( - "Agent proposed new entity types ({}) but lacks Cedar permission.", - spec_summary, + "Agent proposed new entity types ({spec_summary}) but lacks Cedar permission." ), options: vec![SolutionOption { description: "Approve spec submission via Observe UI".to_string(), @@ -149,30 +162,28 @@ pub(crate) async fn handle_load_inline( }], recommendation: Some(0), }; - let a_record_id = a_record.header.id.clone(); - let data_json = serde_json::to_string(&a_record).unwrap_or_default(); + let analysis_id = analysis.header.id.clone(); + let data_json = serde_json::to_string(&analysis).unwrap_or_default(); let _ = state - .insert_evolution_record( - &a_record.header.id, - "Analysis", - &format!("{:?}", a_record.header.status), - &a_record.header.created_by, - a_record.header.derived_from.as_deref(), - &data_json, - ) + .insert_evolution_record(crate::storage::EvolutionRecordWrite { + tenant: &tenant, + id: &analysis.header.id, + record_type: "Analysis", + status: &format!("{:?}", analysis.header.status), + created_by: &analysis.header.created_by, + derived_from: analysis.header.derived_from.as_deref(), + data_json: &data_json, + }) .await; - // Link the PendingDecision to the A-Record for O-A-D chain tracing. - // Decisions are persisted to the tenant store; the evolution_record_id - // link will be available when the decision is read back from the durable backend. if let Some(store) = state.metadata_store_for_tenant(&tenant).await { for decision_id in &decision_ids { - if let Ok(Some(data_str)) = store.get_pending_decision(decision_id).await - && let Ok(mut pd) = + if let Ok(Some(data_str)) = store.get_pending_decision(&tenant, decision_id).await + && let Ok(mut decision) = serde_json::from_str::(&data_str) { - pd.evolution_record_id = Some(a_record_id.clone()); - let _ = state.persist_pending_decision(&pd).await; + decision.evolution_record_id = Some(analysis_id.clone()); + let _ = state.persist_pending_decision(&decision).await; } } } @@ -182,97 +193,66 @@ pub(crate) async fn handle_load_inline( serde_json::json!({ "error": { "code": "AuthorizationDenied", - "message": format!("{reason} Decision {}", primary_decision_id), + "message": format!("{reason} Decision {primary_decision_id}"), } }) .to_string(), )); } - // Write specs to a temp directory - let tmp_dir = std::env::temp_dir().join(format!("temper-inline-{}", tenant)); // determinism-ok: HTTP handler writes user specs to temp dir for loading - let _ = std::fs::remove_dir_all(&tmp_dir); // determinism-ok: HTTP handler cleans previous temp dir - std::fs::create_dir_all(&tmp_dir).map_err(|e| { - // determinism-ok: HTTP handler creates temp dir - ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Failed to create temp dir: {e}"), - ) - })?; - - let specs_root = resolve_inline_specs_root(&tmp_dir, &body.specs)?; + let temp_dir = tempfile::Builder::new() + .prefix("temper-inline-") + .tempdir() + .map_err(|error| { + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to create isolated spec staging directory: {error}"), + ) + })?; + let specs_root = resolve_inline_specs_root(temp_dir.path(), &body.specs)?; for (filename, content) in &body.specs { - let path = tmp_dir.join(filename); + let path = temp_dir.path().join(filename); if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent).map_err(|e| { - // determinism-ok: HTTP handler creates temp subdirectories for nested inline specs + fs::create_dir_all(parent).map_err(|error| { ( StatusCode::INTERNAL_SERVER_ERROR, - format!("Failed to create parent directory for {filename}: {e}"), + format!("Failed to create parent directory for {filename}: {error}"), ) })?; } - std::fs::write(&path, content).map_err(|e| { - // determinism-ok: HTTP handler writes specs + fs::write(&path, content).map_err(|error| { ( StatusCode::INTERNAL_SERVER_ERROR, - format!("Failed to write {filename}: {e}"), + format!("Failed to write {filename}: {error}"), ) })?; } if let Some(source) = body.cross_invariants_toml.as_deref() { - std::fs::write(specs_root.join("cross-invariants.toml"), source).map_err(|e| { - // determinism-ok: HTTP handler writes cross-invariants + fs::write(specs_root.join("cross-invariants.toml"), source).map_err(|error| { ( StatusCode::INTERNAL_SERVER_ERROR, - format!("Failed to write cross-invariants.toml: {e}"), + format!("Failed to write cross-invariants.toml: {error}"), ) })?; } - // Delegate to load-dir logic with merge=true so agent-submitted specs - // are added to the existing tenant config instead of replacing it. - let cedar_policies = body.cedar_policies.clone(); let dir_request = LoadDirRequest { tenant: tenant.clone(), specs_dir: specs_root.to_string_lossy().to_string(), merge: true, }; - let result = handle_load_dir(State(state.clone()), Json(dir_request)).await; - - if result.is_ok() - && let Some(ref cedar_text) = cedar_policies - && !cedar_text.trim().is_empty() - { - if let Err(e) = cedar_text.parse::() { - tracing::warn!(error = %e, "bundled Cedar policies failed to parse, skipping"); - } else { - // Update the in-memory text cache. - if let Ok(mut policies) = state.tenant_policies.write() { - let entry = policies.entry(tenant.clone()).or_default(); - *entry = merge_inline_cedar_policy_text(entry, cedar_text); - } - // Reload the per-tenant Cedar policy set. - let full_text = state - .tenant_policies - .read() - .ok() - .and_then(|p| p.get(&tenant).cloned()) - .unwrap_or_default(); - if let Err(e) = state.authz.reload_tenant_policies(&tenant, &full_text) { - tracing::error!(error = %e, "failed to reload policies with bundled Cedar"); - } else { - tracing::info!(tenant = %tenant, "bundled Cedar policies loaded successfully"); - } - } + let directory = validate_spec_directory(&dir_request.specs_dir)?; + let result = load_specs_from_directory(state.clone(), dir_request, directory).await; + if let Err(error) = temp_dir.close() { + tracing::warn!(error = %error, "failed to remove isolated inline-spec staging directory"); } if result.is_ok() { let warning_context = build_adr_warning_context(&state, &body, &tenant).await; for entity_name in &entity_names { - let traj = TrajectoryEntry { + let trajectory = TrajectoryEntry { timestamp: sim_now().to_rfc3339(), tenant: tenant.clone(), entity_type: entity_name.clone(), @@ -283,18 +263,26 @@ pub(crate) async fn handle_load_inline( to_status: None, error: None, agent_id: Some(security_ctx.principal.id.clone()), - session_id: None, + // Caller-declared session, carried beside the credential rather + // than inside it: `context_attrs` is the Cedar context. + session_id: authenticated.session_id().map(str::to_string), authz_denied: None, denied_resource: None, denied_module: None, source: Some(TrajectorySource::Entity), - spec_governed: None, - agent_type: None, + // Not a governed dispatch: the kernel never ran an action here, + // and the row's session and entity type are caller-chosen. Left + // ungoverned it is walked as an ActorExecution, so a caller could + // post `SubmitSpec` into another run's session and flip that + // run's conformance verdict on an undeclared action. + spec_governed: Some(false), + agent_type: security_ctx.principal.agent_type.clone(), request_body: warning_context.clone(), intent: None, matched_policy_ids: None, + capture_seq: None, }; - if !state.enqueue_trajectory_entry(traj) { + if !state.enqueue_trajectory_entry(trajectory) { tracing::warn!("failed to enqueue spec submission trajectory"); } } @@ -302,303 +290,3 @@ pub(crate) async fn handle_load_inline( result } - -async fn build_adr_warning_context( - state: &ServerState, - body: &LoadInlineRequest, - tenant: &str, -) -> Option { - let namespaces = extract_submitted_namespaces(&body.specs); - let candidate_paths = adr_candidate_paths(body.app_name.as_deref(), &namespaces); - if candidate_paths.is_empty() { - return None; - } - - let hits = find_existing_adr_paths(state, tenant, &candidate_paths).await; - if !hits.is_empty() { - return None; - } - - tracing::warn!( - tenant, - app_name = body.app_name.as_deref().unwrap_or(""), - namespaces = ?namespaces, - candidate_paths = ?candidate_paths, - "Spec submitted with no ADRs — design decisions should be recorded under /apps//adrs/" - ); - - Some(json!({ - "warnings": [{ - "code": "missing_adrs", - "message": "Spec submitted with no ADRs — design decisions should be recorded under /apps//adrs/.", - "candidate_paths": candidate_paths, - "namespaces": namespaces, - "app_name": body.app_name, - }] - })) -} - -fn extract_submitted_namespaces(specs: &std::collections::BTreeMap) -> Vec { - let mut namespaces = std::collections::BTreeSet::new(); - for (filename, content) in specs { - if !filename.ends_with(".csdl.xml") { - continue; - } - if let Ok(document) = parse_csdl(content) { - for schema in document.schemas { - if schema.namespace.ends_with(".Vocab") || schema.namespace == "Temper.Vocab" { - continue; - } - namespaces.insert(schema.namespace); - } - } - } - namespaces.into_iter().collect() -} - -fn resolve_inline_specs_root( - tmp_dir: &Path, - specs: &std::collections::BTreeMap, -) -> Result { - let model_paths: Vec<&str> = specs - .keys() - .filter_map(|path| path.ends_with("model.csdl.xml").then_some(path.as_str())) - .collect(); - - if model_paths.is_empty() { - return Err(( - StatusCode::BAD_REQUEST, - "Inline spec submission must include model.csdl.xml".to_string(), - )); - } - - if model_paths.len() > 1 { - return Err(( - StatusCode::BAD_REQUEST, - format!( - "Inline spec submission must contain exactly one model.csdl.xml, found {}", - model_paths.len() - ), - )); - } - - let model_path = Path::new(model_paths[0]); - let relative_root = model_path.parent().unwrap_or_else(|| Path::new("")); - Ok(if relative_root.as_os_str().is_empty() { - tmp_dir.to_path_buf() - } else { - tmp_dir.join(relative_root) - }) -} - -fn merge_inline_cedar_policy_text(existing: &str, incoming: &str) -> String { - let mut policy_text = existing.trim_end().to_string(); - let incoming = incoming.trim(); - if incoming.is_empty() || policy_text.contains(incoming) { - return policy_text; - } - if !policy_text.is_empty() { - policy_text.push('\n'); - } - policy_text.push_str(incoming); - policy_text -} - -fn adr_candidate_paths(app_name: Option<&str>, namespaces: &[String]) -> Vec { - let mut candidates = std::collections::BTreeSet::new(); - if let Some(app_name) = app_name { - let normalized = normalize_app_slug(app_name); - if !normalized.is_empty() { - candidates.insert(format!("/apps/{normalized}/adrs/")); - } - } - - for namespace in namespaces { - for candidate in namespace_to_app_candidates(namespace) { - if !candidate.is_empty() { - candidates.insert(format!("/apps/{candidate}/adrs/")); - } - } - } - - candidates.into_iter().collect() -} - -fn namespace_to_app_candidates(namespace: &str) -> Vec { - let parts: Vec<&str> = namespace - .split('.') - .filter(|part| !part.is_empty()) - .collect(); - if parts.is_empty() { - return Vec::new(); - } - - let full = normalize_app_slug(namespace); - let remainder = if parts.len() > 1 { - kebab_join(&parts[1..]) - } else { - normalize_app_slug(parts[0]) - }; - - let mut candidates = std::collections::BTreeSet::new(); - if !full.is_empty() { - candidates.insert(full); - } - if !remainder.is_empty() { - candidates.insert(remainder.clone()); - } - match parts[0].to_ascii_lowercase().as_str() { - "openpaw" | "paw" => { - if !remainder.is_empty() { - candidates.insert(format!("paw-{remainder}")); - } - } - "temper" => { - if !remainder.is_empty() { - candidates.insert(format!("temper-{remainder}")); - } - } - _ => {} - } - candidates.into_iter().collect() -} - -fn kebab_join(parts: &[&str]) -> String { - parts - .iter() - .map(|part| normalize_app_slug(part)) - .filter(|part| !part.is_empty()) - .collect::>() - .join("-") -} - -fn normalize_app_slug(value: &str) -> String { - let mut slug = String::new(); - let mut prev_was_sep = true; - let mut prev_was_lower_or_digit = false; - - for ch in value.chars() { - if ch.is_ascii_alphanumeric() { - let is_upper = ch.is_ascii_uppercase(); - if is_upper && prev_was_lower_or_digit && !prev_was_sep { - slug.push('-'); - } - slug.push(ch.to_ascii_lowercase()); - prev_was_sep = false; - prev_was_lower_or_digit = ch.is_ascii_lowercase() || ch.is_ascii_digit(); - } else if !prev_was_sep { - slug.push('-'); - prev_was_sep = true; - prev_was_lower_or_digit = false; - } - } - - slug.trim_matches('-').to_string() -} - -async fn find_existing_adr_paths( - state: &ServerState, - tenant: &str, - candidate_paths: &[String], -) -> Vec { - let tenant_id = TenantId::new(tenant); - let has_files = { - let registry = state.registry.read().unwrap(); // ci-ok: infallible lock - registry.get_spec(&tenant_id, "File").is_some() - }; - if !has_files { - return Vec::new(); - } - - let mut hits = Vec::new(); - for file_id in state.list_entity_ids(&tenant_id, "File") { - let Ok(resp) = state - .get_tenant_entity_state(&tenant_id, "File", &file_id) - .await - else { - continue; - }; - if resp.state.status == "Archived" { - continue; - } - let path = resp - .state - .fields - .get("Path") - .and_then(|value| value.as_str()) - .or_else(|| { - resp.state - .fields - .get("path") - .and_then(|value| value.as_str()) - }); - let Some(path) = path else { continue }; - if candidate_paths - .iter() - .any(|prefix| path.starts_with(prefix)) - { - hits.push(path.to_string()); - } - } - hits.sort(); - hits.dedup(); - hits -} - -#[cfg(test)] -mod tests { - use super::{ - adr_candidate_paths, merge_inline_cedar_policy_text, namespace_to_app_candidates, - normalize_app_slug, - }; - - #[test] - fn normalize_app_slug_kebab_cases_namespaces() { - assert_eq!( - normalize_app_slug("Temper.ProjectManagement"), - "temper-project-management" - ); - assert_eq!(normalize_app_slug("OpenPaw"), "open-paw"); - assert_eq!(normalize_app_slug("llm-wiki"), "llm-wiki"); - } - - #[test] - fn namespace_to_app_candidates_adds_platform_aware_variants() { - let paw = namespace_to_app_candidates("OpenPaw.Foresight"); - assert!(paw.contains(&"open-paw-foresight".to_string())); - assert!(paw.contains(&"foresight".to_string())); - assert!(paw.contains(&"paw-foresight".to_string())); - - let temper = namespace_to_app_candidates("Temper.ProjectManagement"); - assert!(temper.contains(&"temper-project-management".to_string())); - assert!(temper.contains(&"project-management".to_string())); - } - - #[test] - fn adr_candidate_paths_prefers_explicit_app_name() { - let paths = - adr_candidate_paths(Some("llm-wiki"), &["Temper.ProjectManagement".to_string()]); - assert!(paths.contains(&"/apps/llm-wiki/adrs/".to_string())); - assert!(paths.contains(&"/apps/project-management/adrs/".to_string())); - } - - #[test] - fn inline_cedar_policy_merge_deduplicates_repeated_bundle_text() { - let policy = "permit(principal, action, resource);"; - let merged_once = merge_inline_cedar_policy_text("", policy); - let merged_twice = merge_inline_cedar_policy_text(&merged_once, policy); - - assert_eq!(merged_twice, policy); - } - - #[test] - fn inline_cedar_policy_merge_preserves_distinct_existing_policy() { - let existing = "permit(principal, action == Action::\"read\", resource);"; - let incoming = "permit(principal, action == Action::\"write\", resource);"; - - assert_eq!( - merge_inline_cedar_policy_text(existing, incoming), - format!("{existing}\n{incoming}") - ); - } -} diff --git a/crates/temper-server/src/observe/specs/load_inline/support.rs b/crates/temper-server/src/observe/specs/load_inline/support.rs new file mode 100644 index 000000000..f9020295e --- /dev/null +++ b/crates/temper-server/src/observe/specs/load_inline/support.rs @@ -0,0 +1,378 @@ +use std::path::{Path, PathBuf}; + +use axum::http::StatusCode; +use serde_json::json; +use temper_runtime::tenant::TenantId; +use temper_spec::csdl::parse_csdl; + +use super::super::types::LoadInlineRequest; +use crate::state::ServerState; + +pub(super) async fn build_adr_warning_context( + state: &ServerState, + body: &LoadInlineRequest, + tenant: &str, +) -> Option { + let namespaces = extract_submitted_namespaces(&body.specs); + let candidate_paths = adr_candidate_paths(body.app_name.as_deref(), &namespaces); + if candidate_paths.is_empty() { + return None; + } + + let hits = find_existing_adr_paths(state, tenant, &candidate_paths).await; + if !hits.is_empty() { + return None; + } + + tracing::warn!( + tenant, + app_name = body.app_name.as_deref().unwrap_or(""), + namespaces = ?namespaces, + candidate_paths = ?candidate_paths, + "Spec submitted with no ADRs — design decisions should be recorded under /apps//adrs/" + ); + + Some(json!({ + "warnings": [{ + "code": "missing_adrs", + "message": "Spec submitted with no ADRs — design decisions should be recorded under /apps//adrs/.", + "candidate_paths": candidate_paths, + "namespaces": namespaces, + "app_name": body.app_name, + }] + })) +} + +fn extract_submitted_namespaces(specs: &std::collections::BTreeMap) -> Vec { + let mut namespaces = std::collections::BTreeSet::new(); + for (filename, content) in specs { + if !filename.ends_with(".csdl.xml") { + continue; + } + if let Ok(document) = parse_csdl(content) { + for schema in document.schemas { + if schema.namespace.ends_with(".Vocab") || schema.namespace == "Temper.Vocab" { + continue; + } + namespaces.insert(schema.namespace); + } + } + } + namespaces.into_iter().collect() +} + +const INLINE_SPEC_PATH_BUDGET: usize = 512; +const INLINE_SPEC_COMPONENT_BUDGET: usize = 255; + +fn validate_inline_spec_path(path: &str) -> Result<(), (StatusCode, String)> { + use std::path::Component; + + if path.is_empty() || path.len() > INLINE_SPEC_PATH_BUDGET { + return Err(( + StatusCode::BAD_REQUEST, + format!("Invalid inline spec path length: {path:?}"), + )); + } + let mut components = 0usize; + for component in Path::new(path).components() { + let Component::Normal(component) = component else { + return Err(( + StatusCode::BAD_REQUEST, + format!("Inline spec paths must be relative and normalized: {path:?}"), + )); + }; + if component.is_empty() || component.as_encoded_bytes().len() > INLINE_SPEC_COMPONENT_BUDGET + { + return Err(( + StatusCode::BAD_REQUEST, + format!("Inline spec path component exceeds its budget: {path:?}"), + )); + } + components += 1; + } + if components == 0 { + return Err(( + StatusCode::BAD_REQUEST, + format!("Inline spec path has no filename: {path:?}"), + )); + } + Ok(()) +} + +pub(super) fn resolve_inline_specs_root( + tmp_dir: &Path, + specs: &std::collections::BTreeMap, +) -> Result { + for path in specs.keys() { + validate_inline_spec_path(path)?; + } + + let model_paths: Vec<&str> = specs + .keys() + .filter_map(|path| { + (Path::new(path).file_name().and_then(|name| name.to_str()) == Some("model.csdl.xml")) + .then_some(path.as_str()) + }) + .collect(); + + if model_paths.is_empty() { + return Err(( + StatusCode::BAD_REQUEST, + "Inline spec submission must include model.csdl.xml".to_string(), + )); + } + + if model_paths.len() > 1 { + return Err(( + StatusCode::BAD_REQUEST, + format!( + "Inline spec submission must contain exactly one model.csdl.xml, found {}", + model_paths.len() + ), + )); + } + + let model_path = Path::new(model_paths[0]); + let relative_root = model_path.parent().unwrap_or_else(|| Path::new("")); + for path in specs.keys().map(Path::new) { + if !path.starts_with(relative_root) { + return Err(( + StatusCode::BAD_REQUEST, + format!( + "Inline spec path {path:?} is outside the model.csdl.xml directory {relative_root:?}" + ), + )); + } + } + Ok(if relative_root.as_os_str().is_empty() { + tmp_dir.to_path_buf() + } else { + tmp_dir.join(relative_root) + }) +} + +fn adr_candidate_paths(app_name: Option<&str>, namespaces: &[String]) -> Vec { + let mut candidates = std::collections::BTreeSet::new(); + if let Some(app_name) = app_name { + let normalized = normalize_app_slug(app_name); + if !normalized.is_empty() { + candidates.insert(format!("/apps/{normalized}/adrs/")); + } + } + + for namespace in namespaces { + for candidate in namespace_to_app_candidates(namespace) { + if !candidate.is_empty() { + candidates.insert(format!("/apps/{candidate}/adrs/")); + } + } + } + + candidates.into_iter().collect() +} + +fn namespace_to_app_candidates(namespace: &str) -> Vec { + let parts: Vec<&str> = namespace + .split('.') + .filter(|part| !part.is_empty()) + .collect(); + if parts.is_empty() { + return Vec::new(); + } + + let full = normalize_app_slug(namespace); + let remainder = if parts.len() > 1 { + kebab_join(&parts[1..]) + } else { + normalize_app_slug(parts[0]) + }; + + let mut candidates = std::collections::BTreeSet::new(); + if !full.is_empty() { + candidates.insert(full); + } + if !remainder.is_empty() { + candidates.insert(remainder.clone()); + } + match parts[0].to_ascii_lowercase().as_str() { + "openpaw" | "paw" => { + if !remainder.is_empty() { + candidates.insert(format!("paw-{remainder}")); + } + } + "temper" => { + if !remainder.is_empty() { + candidates.insert(format!("temper-{remainder}")); + } + } + _ => {} + } + candidates.into_iter().collect() +} + +fn kebab_join(parts: &[&str]) -> String { + parts + .iter() + .map(|part| normalize_app_slug(part)) + .filter(|part| !part.is_empty()) + .collect::>() + .join("-") +} + +fn normalize_app_slug(value: &str) -> String { + let mut slug = String::new(); + let mut prev_was_sep = true; + let mut prev_was_lower_or_digit = false; + + for ch in value.chars() { + if ch.is_ascii_alphanumeric() { + let is_upper = ch.is_ascii_uppercase(); + if is_upper && prev_was_lower_or_digit && !prev_was_sep { + slug.push('-'); + } + slug.push(ch.to_ascii_lowercase()); + prev_was_sep = false; + prev_was_lower_or_digit = ch.is_ascii_lowercase() || ch.is_ascii_digit(); + } else if !prev_was_sep { + slug.push('-'); + prev_was_sep = true; + prev_was_lower_or_digit = false; + } + } + + slug.trim_matches('-').to_string() +} + +async fn find_existing_adr_paths( + state: &ServerState, + tenant: &str, + candidate_paths: &[String], +) -> Vec { + let tenant_id = TenantId::new(tenant); + let has_files = { + let registry = state.registry.read().unwrap(); // ci-ok: infallible lock + registry.get_spec(&tenant_id, "File").is_some() + }; + if !has_files { + return Vec::new(); + } + + let mut hits = Vec::new(); + for file_id in state.list_entity_ids(&tenant_id, "File") { + let Ok(resp) = state + .get_tenant_entity_state(&tenant_id, "File", &file_id) + .await + else { + continue; + }; + if resp.state.status == "Archived" { + continue; + } + let path = resp + .state + .fields + .get("Path") + .and_then(|value| value.as_str()) + .or_else(|| { + resp.state + .fields + .get("path") + .and_then(|value| value.as_str()) + }); + let Some(path) = path else { continue }; + if candidate_paths + .iter() + .any(|prefix| path.starts_with(prefix)) + { + hits.push(path.to_string()); + } + } + hits.sort(); + hits.dedup(); + hits +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use super::{ + adr_candidate_paths, namespace_to_app_candidates, normalize_app_slug, + resolve_inline_specs_root, + }; + + #[test] + fn normalize_app_slug_kebab_cases_namespaces() { + assert_eq!( + normalize_app_slug("Temper.ProjectManagement"), + "temper-project-management" + ); + assert_eq!(normalize_app_slug("OpenPaw"), "open-paw"); + assert_eq!(normalize_app_slug("llm-wiki"), "llm-wiki"); + } + + #[test] + fn namespace_to_app_candidates_adds_platform_aware_variants() { + let paw = namespace_to_app_candidates("OpenPaw.Foresight"); + assert!(paw.contains(&"open-paw-foresight".to_string())); + assert!(paw.contains(&"foresight".to_string())); + assert!(paw.contains(&"paw-foresight".to_string())); + + let temper = namespace_to_app_candidates("Temper.ProjectManagement"); + assert!(temper.contains(&"temper-project-management".to_string())); + assert!(temper.contains(&"project-management".to_string())); + } + + #[test] + fn adr_candidate_paths_prefers_explicit_app_name() { + let paths = + adr_candidate_paths(Some("llm-wiki"), &["Temper.ProjectManagement".to_string()]); + assert!(paths.contains(&"/apps/llm-wiki/adrs/".to_string())); + assert!(paths.contains(&"/apps/project-management/adrs/".to_string())); + } + + #[test] + fn inline_spec_paths_cannot_escape_isolated_staging() { + for malicious in [ + "../model.csdl.xml", + "/tmp/model.csdl.xml", + "nested/../../model.csdl.xml", + "./model.csdl.xml", + ] { + let specs = BTreeMap::from([(malicious.to_string(), String::new())]); + let error = resolve_inline_specs_root(std::path::Path::new("/tmp/staging"), &specs) + .expect_err("escaping path should be rejected"); + assert_eq!(error.0, axum::http::StatusCode::BAD_REQUEST); + } + } + + #[test] + fn inline_spec_paths_must_share_the_model_directory() { + let specs = BTreeMap::from([ + ("app/model.csdl.xml".to_string(), String::new()), + ("other/order.ioa.toml".to_string(), String::new()), + ]); + let error = resolve_inline_specs_root(std::path::Path::new("/tmp/staging"), &specs) + .expect_err("out-of-root file should be rejected"); + assert_eq!(error.0, axum::http::StatusCode::BAD_REQUEST); + } + + #[test] + fn inline_spec_model_name_is_exact_not_a_suffix() { + let specs = BTreeMap::from([("not-model.csdl.xml".to_string(), String::new())]); + let error = resolve_inline_specs_root(std::path::Path::new("/tmp/staging"), &specs) + .expect_err("model filename suffix should not be accepted"); + assert_eq!(error.0, axum::http::StatusCode::BAD_REQUEST); + } + + #[test] + fn inline_spec_paths_allow_one_normalized_nested_root() { + let specs = BTreeMap::from([ + ("app/model.csdl.xml".to_string(), String::new()), + ("app/order.ioa.toml".to_string(), String::new()), + ]); + let root = resolve_inline_specs_root(std::path::Path::new("/tmp/staging"), &specs) + .expect("normalized paths should be accepted"); + assert_eq!(root, std::path::Path::new("/tmp/staging/app")); + } +} diff --git a/crates/temper-server/src/observe/specs/types.rs b/crates/temper-server/src/observe/specs/types.rs index 9747b90ca..262dd0c1d 100644 --- a/crates/temper-server/src/observe/specs/types.rs +++ b/crates/temper-server/src/observe/specs/types.rs @@ -27,7 +27,8 @@ pub(crate) struct LoadInlineRequest { /// Optional inline `cross-invariants.toml` source. #[serde(default)] pub(crate) cross_invariants_toml: Option, - /// Optional Cedar policy text to bundle with the spec deployment. + /// Legacy bundled Cedar text. Non-empty values are rejected; callers must + /// use the separately authorized tenant policy-management API. #[serde(default)] pub(crate) cedar_policies: Option, } diff --git a/crates/temper-server/src/observe/specs/validate_ioa.rs b/crates/temper-server/src/observe/specs/validate_ioa.rs index d836b42ef..b93326cae 100644 --- a/crates/temper-server/src/observe/specs/validate_ioa.rs +++ b/crates/temper-server/src/observe/specs/validate_ioa.rs @@ -1,9 +1,10 @@ -use axum::extract::State; -use axum::http::{HeaderMap, StatusCode}; +use axum::extract::{Extension, State}; +use axum::http::StatusCode; use axum::response::Json; +use temper_authz::AuthenticatedRequestContext; use tracing::instrument; -use crate::authz::require_observe_auth; +use crate::authz::{require_authenticated_context, require_observe_auth}; use crate::state::ServerState; use super::types::ValidateIoaRequest; @@ -21,10 +22,12 @@ const MAX_IOA_SOURCE_BYTES: usize = 1_048_576; #[instrument(skip_all, fields(otel.name = "POST /api/specs/validate-ioa"))] pub(crate) async fn handle_validate_ioa( State(state): State, - headers: HeaderMap, + authenticated: Option>, Json(body): Json, ) -> Result, (StatusCode, String)> { - require_observe_auth(&state, &headers, "run_verification", "Verification") + let authenticated = require_authenticated_context(authenticated.as_deref()) + .map_err(|status| (status, "authentication required".to_string()))?; + require_observe_auth(&state, authenticated, "run_verification", "Verification") .map_err(|status| (status, "verification authorization failed".to_string()))?; let ioa_source = body.ioa_source; diff --git a/crates/temper-server/src/observe/subprocess_verify.rs b/crates/temper-server/src/observe/subprocess_verify.rs index 2284c6c6b..7e2671c6f 100644 --- a/crates/temper-server/src/observe/subprocess_verify.rs +++ b/crates/temper-server/src/observe/subprocess_verify.rs @@ -29,11 +29,17 @@ pub async fn verify_in_subprocess( bin: &Path, ioa_source: &str, ) -> Result { - let mut child = tokio::process::Command::new(bin) + let mut command = tokio::process::Command::new(bin); + command .arg("verify-ioa") + // Verification processes parse untrusted specs and need no server + // credentials. Do not inherit deployment, database, or provider keys. + .env_clear() + .kill_on_drop(true) .stdin(std::process::Stdio::piped()) .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()); + let mut child = command .spawn() .map_err(|e| format!("failed to spawn verification subprocess: {e}"))?; @@ -62,3 +68,7 @@ pub async fn verify_in_subprocess( serde_json::from_slice::(&output.stdout) .map_err(|e| format!("failed to parse verification subprocess output: {e}")) } + +#[cfg(all(test, unix))] +#[path = "subprocess_verify_test.rs"] +mod tests; diff --git a/crates/temper-server/src/observe/subprocess_verify_test.rs b/crates/temper-server/src/observe/subprocess_verify_test.rs new file mode 100644 index 000000000..b5e4f6952 --- /dev/null +++ b/crates/temper-server/src/observe/subprocess_verify_test.rs @@ -0,0 +1,57 @@ +use std::os::unix::fs::PermissionsExt as _; + +use super::*; + +const PARENT_ENV_SENTINEL: &str = "TEMPER_VERIFY_PARENT_SECRET"; + +struct ParentEnvSentinel; + +impl ParentEnvSentinel { + fn install() -> Self { + // SAFETY: this test owns a unique Temper-specific environment key and + // removes it before returning. The child must not inherit it because + // `verify_in_subprocess` calls `env_clear()`. + unsafe { std::env::set_var(PARENT_ENV_SENTINEL, "must-not-leak") }; + Self + } +} + +impl Drop for ParentEnvSentinel { + fn drop(&mut self) { + // SAFETY: see `install`; this is the matching cleanup for the same + // unique test-only key. + unsafe { std::env::remove_var(PARENT_ENV_SENTINEL) }; + } +} + +#[tokio::test] +async fn verifier_subprocess_does_not_inherit_parent_environment() { + let _sentinel = ParentEnvSentinel::install(); + assert!( + std::env::var_os(PARENT_ENV_SENTINEL).is_some(), + "test runner should provide the parent-only environment sentinel" + ); + let directory = tempfile::tempdir().expect("create verifier test directory"); + let script = directory.path().join("verify-env.sh"); + std::fs::write( + &script, + format!( + "#!/bin/sh\nif [ -n \"${{{PARENT_ENV_SENTINEL}+x}}\" ]; then echo inherited-parent-secret >&2; exit 42; fi\ncat >/dev/null\necho '{{}}'\n" + ), + ) + .expect("write verifier test script"); + let mut permissions = std::fs::metadata(&script) + .expect("read verifier script metadata") + .permissions(); + permissions.set_mode(0o700); + std::fs::set_permissions(&script, permissions).expect("make verifier script executable"); + + let error = verify_in_subprocess(&script, "[automaton]") + .await + .expect_err("empty JSON should not decode as a cascade result"); + assert!( + error.contains("failed to parse verification subprocess output"), + "unexpected subprocess verifier error: {error}" + ); + assert!(!error.contains("inherited-parent-secret")); +} diff --git a/crates/temper-server/src/observe/verification/cascade.rs b/crates/temper-server/src/observe/verification/cascade.rs index 146fd0efb..849f51b7a 100644 --- a/crates/temper-server/src/observe/verification/cascade.rs +++ b/crates/temper-server/src/observe/verification/cascade.rs @@ -1,12 +1,13 @@ //! POST /observe/verify/{entity} -- run verification cascade on a spec. -use axum::extract::{Path, State}; -use axum::http::{HeaderMap, StatusCode}; +use axum::extract::{Extension, Path, State}; +use axum::http::StatusCode; use axum::response::Json; +use temper_authz::AuthenticatedRequestContext; use temper_runtime::scheduler::sim_now; use tracing::instrument; -use crate::authz::require_observe_auth; +use crate::authz::{require_authenticated_context, require_observe_auth}; use crate::registry::VerificationStatus; use crate::state::ServerState; @@ -17,10 +18,11 @@ use crate::state::ServerState; #[instrument(skip_all, fields(entity, otel.name = "POST /observe/verify/{entity}"))] pub(crate) async fn handle_run_verification( State(state): State, - headers: HeaderMap, + authenticated: Option>, Path(entity): Path, ) -> Result, StatusCode> { - require_observe_auth(&state, &headers, "run_verification", "Verification")?; + let authenticated = require_authenticated_context(authenticated.as_deref())?; + require_observe_auth(&state, authenticated, "run_verification", "Verification")?; let Some((tenant_id, ioa_source)) = state.find_entity_ioa_source(&entity) else { tracing::warn!("entity spec not found for verification"); diff --git a/crates/temper-server/src/observe/verification/paths.rs b/crates/temper-server/src/observe/verification/paths.rs index 026fc1573..f029674f1 100644 --- a/crates/temper-server/src/observe/verification/paths.rs +++ b/crates/temper-server/src/observe/verification/paths.rs @@ -1,12 +1,13 @@ //! GET /observe/paths/{entity} -- extract state machine paths. -use axum::extract::{Path, Query, State}; -use axum::http::{HeaderMap, StatusCode}; +use axum::extract::{Extension, Path, Query, State}; +use axum::http::StatusCode; use axum::response::Json; use serde::Deserialize; +use temper_authz::AuthenticatedRequestContext; use tracing::instrument; -use crate::authz::require_observe_auth; +use crate::authz::{require_authenticated_context, require_observe_auth}; use crate::state::ServerState; /// Query parameters for path extraction. @@ -20,11 +21,12 @@ pub(crate) struct PathsQueryParams { #[instrument(skip_all, fields(entity, otel.name = "GET /observe/paths/{entity}"))] pub(crate) async fn handle_get_paths( State(state): State, - headers: HeaderMap, + authenticated: Option>, Path(entity): Path, Query(params): Query, ) -> Result, StatusCode> { - require_observe_auth(&state, &headers, "read_verification", "Verification")?; + let authenticated = require_authenticated_context(authenticated.as_deref())?; + require_observe_auth(&state, authenticated, "read_verification", "Verification")?; let Some((_tenant_id, ioa_source)) = state.find_entity_ioa_source(&entity) else { tracing::warn!("entity spec not found for path extraction"); diff --git a/crates/temper-server/src/observe/verification/simulation.rs b/crates/temper-server/src/observe/verification/simulation.rs index 64a71bc7b..8d3d9ba21 100644 --- a/crates/temper-server/src/observe/verification/simulation.rs +++ b/crates/temper-server/src/observe/verification/simulation.rs @@ -1,11 +1,12 @@ //! GET /observe/simulation/{entity} -- run deterministic simulation. -use axum::extract::{Path, Query, State}; -use axum::http::{HeaderMap, StatusCode}; +use axum::extract::{Extension, Path, Query, State}; +use axum::http::StatusCode; use axum::response::Json; +use temper_authz::AuthenticatedRequestContext; use tracing::instrument; -use crate::authz::require_observe_auth; +use crate::authz::{require_authenticated_context, require_observe_auth}; use crate::observe::SimQueryParams; use crate::state::ServerState; @@ -15,11 +16,12 @@ use crate::state::ServerState; #[instrument(skip_all, fields(entity, otel.name = "GET /observe/simulation/{entity}"))] pub(crate) async fn handle_run_simulation( State(state): State, - headers: HeaderMap, + authenticated: Option>, Path(entity): Path, Query(params): Query, ) -> Result, StatusCode> { - require_observe_auth(&state, &headers, "read_verification", "Verification")?; + let authenticated = require_authenticated_context(authenticated.as_deref())?; + require_observe_auth(&state, authenticated, "read_verification", "Verification")?; let Some((_tenant_id, ioa_source)) = state.find_entity_ioa_source(&entity) else { tracing::warn!("entity spec not found for simulation"); diff --git a/crates/temper-server/src/observe/verification/status.rs b/crates/temper-server/src/observe/verification/status.rs index c81440816..8965fdf59 100644 --- a/crates/temper-server/src/observe/verification/status.rs +++ b/crates/temper-server/src/observe/verification/status.rs @@ -1,12 +1,13 @@ //! GET /observe/verification-status -- all entity verification statuses. -use axum::extract::State; -use axum::http::{HeaderMap, StatusCode}; +use axum::extract::{Extension, State}; +use axum::http::StatusCode; use axum::response::Json; use serde::Serialize; +use temper_authz::AuthenticatedRequestContext; use tracing::instrument; -use crate::authz::{observe_tenant_scope, require_observe_auth}; +use crate::authz::{observe_tenant_scope, require_authenticated_context, require_observe_auth}; use crate::registry::VerificationStatus; use crate::state::ServerState; @@ -37,10 +38,11 @@ struct EntityVerificationStatusResponse { #[instrument(skip_all, fields(otel.name = "GET /observe/verification-status"))] pub(crate) async fn handle_verification_status( State(state): State, - headers: HeaderMap, + authenticated: Option>, ) -> Result, StatusCode> { - require_observe_auth(&state, &headers, "read_verification", "Spec")?; - let tenant_scope = observe_tenant_scope(&state, &headers)?; + let authenticated = require_authenticated_context(authenticated.as_deref())?; + require_observe_auth(&state, authenticated, "read_verification", "Spec")?; + let tenant_scope = observe_tenant_scope(authenticated); let registry = state.registry.read().unwrap(); // ci-ok: infallible lock let mut pending = 0usize; let mut running = 0usize; @@ -50,9 +52,7 @@ pub(crate) async fn handle_verification_status( let mut entities = Vec::new(); for tenant_id in registry.tenant_ids() { - if let Some(ref scope) = tenant_scope - && tenant_id != scope - { + if tenant_id != tenant_scope { continue; } if let Some(statuses) = registry.verification_statuses(tenant_id) { diff --git a/crates/temper-server/src/observe/verification/stream.rs b/crates/temper-server/src/observe/verification/stream.rs index 121609396..f8adc155b 100644 --- a/crates/temper-server/src/observe/verification/stream.rs +++ b/crates/temper-server/src/observe/verification/stream.rs @@ -2,14 +2,15 @@ use std::convert::Infallible; -use axum::extract::State; -use axum::http::{HeaderMap, StatusCode}; +use axum::extract::{Extension, State}; +use axum::http::StatusCode; use axum::response::sse::{Event, KeepAlive, Sse}; +use temper_authz::AuthenticatedRequestContext; use tokio_stream::StreamExt; use tokio_stream::wrappers::BroadcastStream; use tracing::instrument; -use crate::authz::{observe_tenant_scope, require_observe_auth}; +use crate::authz::{observe_tenant_scope, require_authenticated_context, require_observe_auth}; use crate::state::ServerState; /// GET /observe/design-time/stream -- SSE stream of design-time events. @@ -19,19 +20,17 @@ use crate::state::ServerState; #[instrument(skip_all, fields(otel.name = "GET /observe/design-time/stream"))] pub(crate) async fn handle_design_time_stream( State(state): State, - headers: HeaderMap, + authenticated: Option>, ) -> Result>>, StatusCode> { - require_observe_auth(&state, &headers, "read_events", "Event")?; - let tenant_scope = observe_tenant_scope(&state, &headers)?; - let filter_tenant = tenant_scope.map(|t| t.as_str().to_string()); + let authenticated = require_authenticated_context(authenticated.as_deref())?; + require_observe_auth(&state, authenticated, "read_events", "Event")?; + let filter_tenant = observe_tenant_scope(authenticated).as_str().to_string(); let rx = state.design_time_tx.subscribe(); let stream = BroadcastStream::new(rx).filter_map(move |result| { match result { Ok(event) => { - if let Some(ref tenant) = filter_tenant - && event.tenant != *tenant - { + if event.tenant != filter_tenant { return None; } let data = serde_json::to_string(&event).unwrap_or_default(); diff --git a/crates/temper-server/src/observe/verification/workflow.rs b/crates/temper-server/src/observe/verification/workflow.rs index a7ae65ad5..c5611895e 100644 --- a/crates/temper-server/src/observe/verification/workflow.rs +++ b/crates/temper-server/src/observe/verification/workflow.rs @@ -1,13 +1,14 @@ //! GET /observe/workflows -- full workflow view per app/tenant. -use axum::extract::State; -use axum::http::{HeaderMap, StatusCode}; +use axum::extract::{Extension, State}; +use axum::http::StatusCode; use axum::response::Json; use serde::Serialize; +use temper_authz::AuthenticatedRequestContext; use temper_runtime::tenant::TenantId; use tracing::instrument; -use crate::authz::{observe_tenant_scope, require_observe_auth}; +use crate::authz::{observe_tenant_scope, require_authenticated_context, require_observe_auth}; use crate::registry::{SpecRegistry, VerificationStatus}; use crate::state::ServerState; @@ -44,11 +45,16 @@ pub(in crate::observe) struct WorkflowsResponse { } /// Fetch design-time events from the durable metadata backend. -async fn fetch_event_log(state: &ServerState) -> Vec { - let stores = state.collect_all_metadata_stores().await; +async fn fetch_event_log( + state: &ServerState, + tenant: &TenantId, +) -> Vec { let mut all_events = Vec::new(); - for store in &stores { - match store.list_design_time_events(None, 10_000).await { + if let Some(store) = state.metadata_store_for_tenant(tenant.as_str()).await { + match store + .list_design_time_events(Some(tenant.as_str()), 10_000) + .await + { Ok(rows) => { all_events.extend(rows.into_iter().map(|r| crate::state::DesignTimeEvent { kind: r.kind, @@ -73,8 +79,23 @@ async fn fetch_event_log(state: &ServerState) -> Vec std::collections::BTreeMap { - state.count_trajectories_by_tenant().await +async fn fetch_runtime_counts( + state: &ServerState, + tenant: &TenantId, +) -> std::collections::BTreeMap { + let Some(store) = state.metadata_store_for_tenant(tenant.as_str()).await else { + return std::collections::BTreeMap::new(); + }; + match store.count_trajectories_by_tenant().await { + Ok(counts) => counts + .into_iter() + .filter(|(stored_tenant, _)| stored_tenant == tenant.as_str()) + .collect(), + Err(error) => { + tracing::warn!(error = %error, backend = store.backend_name(), "failed to count workflow trajectories"); + std::collections::BTreeMap::new() + } + } } /// Build a step from an event log entry matching a given kind. @@ -212,20 +233,19 @@ fn build_entity_workflow( #[instrument(skip_all, fields(otel.name = "GET /observe/workflows"))] pub(crate) async fn handle_workflows( State(state): State, - headers: HeaderMap, + authenticated: Option>, ) -> Result, StatusCode> { - require_observe_auth(&state, &headers, "read_events", "Event")?; - let tenant_scope = observe_tenant_scope(&state, &headers)?; + let authenticated = require_authenticated_context(authenticated.as_deref())?; + require_observe_auth(&state, authenticated, "read_events", "Event")?; + let tenant_scope = observe_tenant_scope(authenticated); - let event_log = fetch_event_log(&state).await; - let runtime_counts = fetch_runtime_counts(&state).await; + let event_log = fetch_event_log(&state, tenant_scope).await; + let runtime_counts = fetch_runtime_counts(&state, tenant_scope).await; let registry = state.registry.read().unwrap(); // ci-ok: infallible lock let mut workflows = Vec::new(); for tenant_id in registry.tenant_ids() { - if let Some(ref scope) = tenant_scope - && tenant_id != scope - { + if tenant_id != tenant_scope { continue; } let tenant_str = tenant_id.as_str().to_string(); diff --git a/crates/temper-server/src/observe/wasm.rs b/crates/temper-server/src/observe/wasm.rs index 43e61585c..5378e913e 100644 --- a/crates/temper-server/src/observe/wasm.rs +++ b/crates/temper-server/src/observe/wasm.rs @@ -5,19 +5,19 @@ use std::collections::BTreeMap; use axum::extract::Path; -use axum::extract::{Query, State}; +use axum::extract::{Extension, Query, State}; use axum::http::{HeaderMap, StatusCode}; use axum::response::Json; use base64::Engine; use serde::{Deserialize, Serialize}; +use temper_authz::AuthenticatedRequestContext; use tracing::instrument; use crate::authz::{ - GovernedMutationAuth, observe_tenant_scope, require_governed_mutation_auth, - require_observe_auth, + GovernedMutationAuth, observe_tenant_scope, require_authenticated_context, + require_governed_mutation_auth, require_observe_auth, }; -use crate::odata::extract_tenant; use crate::state::ServerState; #[derive(Deserialize)] @@ -90,15 +90,18 @@ pub struct WasmInvocationResponse { /// POST /api/wasm/modules/{module_name} — upload a WASM binary. /// -/// Admin principals bypass Cedar; other principals require "manage_wasm" on "WasmModule". +/// Requires `manage_wasm` on the exact `WasmModule` resource. #[instrument(skip_all, fields(module_name, otel.name = "POST /api/wasm/modules/{module_name}"))] pub async fn handle_upload_wasm_module( State(state): State, headers: HeaderMap, + authenticated: Option>, Path(module_name): Path, body: axum::body::Bytes, ) -> Result, (StatusCode, String)> { - let tenant = extract_tenant(&headers, &state)?; + let authenticated = require_authenticated_context(authenticated.as_deref()) + .map_err(|status| (status, "authentication required".to_string()))?; + let tenant = authenticated.tenant().clone(); let mut resource_attrs = BTreeMap::new(); resource_attrs.insert( "id".to_string(), @@ -110,7 +113,7 @@ pub async fn handle_upload_wasm_module( ); if let Some(resp) = require_governed_mutation_auth( &state, - &headers, + authenticated, GovernedMutationAuth { tenant: tenant.as_str(), action: "manage_wasm", @@ -233,10 +236,12 @@ fn decode_wasm_upload_body( #[instrument(skip_all, fields(module_name, otel.name = "GET /observe/wasm/modules/{module_name}"))] pub async fn handle_get_wasm_module_info( State(state): State, - headers: HeaderMap, + authenticated: Option>, Path(module_name): Path, ) -> Result, StatusCode> { - let tenant = extract_tenant(&headers, &state).map_err(|(s, _)| s)?; + let authenticated = require_authenticated_context(authenticated.as_deref())?; + require_observe_auth(&state, authenticated, "read_wasm", "WasmModule")?; + let tenant = authenticated.tenant().clone(); let hash = { let wasm_reg = state.wasm_module_registry.read().unwrap(); @@ -261,14 +266,16 @@ pub async fn handle_get_wasm_module_info( /// DELETE /api/wasm/modules/{module_name} — remove a module. /// -/// Admin principals bypass Cedar; other principals require "manage_wasm" on "WasmModule". +/// Requires `manage_wasm` on the exact `WasmModule` resource. #[instrument(skip_all, fields(module_name, otel.name = "DELETE /api/wasm/modules/{module_name}"))] pub async fn handle_delete_wasm_module( State(state): State, - headers: HeaderMap, + authenticated: Option>, Path(module_name): Path, ) -> Result, (StatusCode, String)> { - let tenant = extract_tenant(&headers, &state)?; + let authenticated = require_authenticated_context(authenticated.as_deref()) + .map_err(|status| (status, "authentication required".to_string()))?; + let tenant = authenticated.tenant().clone(); let mut resource_attrs = BTreeMap::new(); resource_attrs.insert( "id".to_string(), @@ -280,7 +287,7 @@ pub async fn handle_delete_wasm_module( ); if let Some(resp) = require_governed_mutation_auth( &state, - &headers, + authenticated, GovernedMutationAuth { tenant: tenant.as_str(), action: "manage_wasm", @@ -348,23 +355,24 @@ pub async fn handle_delete_wasm_module( }))) } -/// GET /observe/wasm/modules — list all modules (with stats). -/// -/// Admin/System principals see all tenants; others are scoped to `X-Tenant-Id`. +/// GET /observe/wasm/modules — list modules in the credential-bound tenant. #[instrument(skip_all, fields(otel.name = "GET /observe/wasm/modules"))] pub async fn handle_list_wasm_modules( State(state): State, - headers: HeaderMap, + authenticated: Option>, ) -> Result, StatusCode> { - require_observe_auth(&state, &headers, "read_wasm", "WasmModule")?; - let tenant_scope = observe_tenant_scope(&state, &headers)?; + let authenticated = require_authenticated_context(authenticated.as_deref())?; + require_observe_auth(&state, authenticated, "read_wasm", "WasmModule")?; + let tenant_scope = observe_tenant_scope(authenticated); // Collect invocation stats via fan-out across all tenant stores. let invocation_stats: std::collections::BTreeMap)> = { let mut stats: std::collections::BTreeMap)> = std::collections::BTreeMap::new(); - let stores = state.collect_all_metadata_stores().await; - for store in &stores { + let store = state + .metadata_store_for_tenant(authenticated.tenant().as_str()) + .await; + for store in store.iter() { if let Ok(rows) = store.load_recent_wasm_invocations(10_000).await { for row in rows { let module = row.module_name.clone(); @@ -411,11 +419,7 @@ pub async fn handle_list_wasm_modules( let mut entries: Vec = wasm_reg .all_modules() .into_iter() - .filter(|(tenant, _, _)| { - tenant_scope - .as_ref() - .is_none_or(|scope| scope.as_str() == *tenant) - }) + .filter(|(tenant, _, _)| tenant_scope.as_str() == *tenant) .map(|(tenant, name, hash)| make_entry(tenant, name, hash)) .collect(); @@ -438,15 +442,18 @@ pub async fn handle_list_wasm_modules( #[instrument(skip_all, fields(otel.name = "GET /observe/wasm/invocations"))] pub async fn handle_list_wasm_invocations( State(state): State, - headers: HeaderMap, + authenticated: Option>, Query(params): Query, ) -> Result, StatusCode> { - require_observe_auth(&state, &headers, "read_wasm", "WasmModule")?; + let authenticated = require_authenticated_context(authenticated.as_deref())?; + require_observe_auth(&state, authenticated, "read_wasm", "WasmModule")?; let limit = params.limit.unwrap_or(100).min(10_000); - let stores = state.collect_all_metadata_stores().await; let mut all_filtered: Vec = Vec::new(); - for store in &stores { + let store = state + .metadata_store_for_tenant(authenticated.tenant().as_str()) + .await; + for store in store.iter() { match store.load_recent_wasm_invocations(limit as i64).await { Ok(rows) => { let filtered: Vec = rows diff --git a/crates/temper-server/src/odata/authz.rs b/crates/temper-server/src/odata/authz.rs index c77018745..e1fcb6ad1 100644 --- a/crates/temper-server/src/odata/authz.rs +++ b/crates/temper-server/src/odata/authz.rs @@ -2,13 +2,13 @@ use std::collections::BTreeMap; -use axum::http::{HeaderMap, StatusCode}; +use axum::extract::Extension; +use axum::http::StatusCode; use axum::response::{IntoResponse, Response}; -use temper_authz::SecurityContext; +use temper_authz::{AuthenticatedRequestContext, SecurityContext}; use temper_runtime::tenant::TenantId; -use crate::authz::{DenialInput, record_authz_denial, security_context_from_headers}; -use crate::identity::ResolvedIdentity; +use crate::authz::{DenialInput, record_authz_denial}; use crate::request_context::AgentContext; use crate::response::odata_error; use crate::state::ServerState; @@ -19,20 +19,44 @@ pub(crate) const READ_ACTION: &str = "read"; pub(super) const UPDATE_ACTION: &str = "update"; pub(super) const DELETE_ACTION: &str = "delete"; -/// Build the authoritative Cedar principal for an external OData request. -pub(super) fn request_security_context( - headers: &HeaderMap, - agent_ctx: &AgentContext, - resolved_identity: Option<&ResolvedIdentity>, -) -> SecurityContext { - if let Some(identity) = resolved_identity { - SecurityContext::from_resolved_identity( - &identity.agent_instance_id, - &identity.agent_type_name, - agent_ctx.session_id.as_deref(), +/// Require the immutable context installed by credential authentication. +#[derive(Clone, Copy, Debug)] +pub(super) struct AuthenticationRequired; + +impl IntoResponse for AuthenticationRequired { + fn into_response(self) -> Response { + odata_error( + StatusCode::UNAUTHORIZED, + "AuthenticationRequired", + "A valid tenant credential is required", ) - } else { - security_context_from_headers(headers, None, agent_ctx.session_id.as_deref(), None) + .into_response() + } +} + +pub(super) fn require_authenticated_context( + context: Option>, +) -> Result { + context + .map(|Extension(context)| context) + .ok_or(AuthenticationRequired) +} + +/// Attach exact authenticated authority to downstream dispatch context. +/// +/// Correlation metadata remains header-derived, but no principal field is +/// reconstructed or enriched from headers. +pub(super) fn apply_authenticated_context( + agent_context: &mut AgentContext, + security_context: &SecurityContext, +) { + agent_context.security_ctx = Some(security_context.clone()); + if matches!( + security_context.principal.kind, + temper_authz::PrincipalKind::Agent | temper_authz::PrincipalKind::Admin + ) { + agent_context.agent_id = Some(security_context.principal.id.clone()); + agent_context.agent_type = security_context.principal.agent_type.clone(); } } @@ -49,11 +73,6 @@ pub(super) fn resource_attrs_from_body( body: &serde_json::Value, ) -> BTreeMap { let mut attrs = BTreeMap::new(); - attrs.insert( - "id".to_string(), - serde_json::Value::String(resource_id.to_string()), - ); - let status = body .get("status") .or_else(|| body.get("Status")) @@ -61,23 +80,34 @@ pub(super) fn resource_attrs_from_body( .or_else(|| body.get("fields").and_then(|fields| fields.get("Status"))) .cloned() .unwrap_or_else(|| serde_json::Value::String(String::new())); - attrs.insert("status".to_string(), status); if let Some(fields) = body.get("fields").and_then(serde_json::Value::as_object) { for (key, value) in fields { - attrs.insert(key.clone(), value.clone()); + if !temper_spec::automaton::is_server_derived_field_name(key) { + attrs.insert(key.clone(), value.clone()); + } } } else if let Some(fields) = body.as_object() { for (key, value) in fields { - if !key.starts_with('@') { + if !key.starts_with('@') && !temper_spec::automaton::is_server_derived_field_name(key) { attrs.insert(key.clone(), value.clone()); } } } + for key in ["id", "Id"] { + attrs.insert( + key.to_string(), + serde_json::Value::String(resource_id.to_string()), + ); + } + for key in ["status", "Status"] { + attrs.insert(key.to_string(), status.clone()); + } + let has_spec = state .has_registered_spec(tenant, entity_type) - .unwrap_or(false); + .expect("registry lock poisoned while building OData authorization attributes"); attrs.insert("has_spec".to_string(), serde_json::Value::Bool(has_spec)); attrs } @@ -164,6 +194,11 @@ pub(super) async fn authorize_mutation( .get("status") .and_then(serde_json::Value::as_str) .map(str::to_string), + intent: agent_ctx.intent.clone(), + session_id: agent_ctx.session_id.clone(), + // A genuine attempted dispatch of a registered action: walked by + // conformance, matching both parents' behavior. + spec_governed: None, }, ) .await; diff --git a/crates/temper-server/src/odata/bindings.rs b/crates/temper-server/src/odata/bindings.rs index ffe35b806..a98d267b9 100644 --- a/crates/temper-server/src/odata/bindings.rs +++ b/crates/temper-server/src/odata/bindings.rs @@ -1,6 +1,6 @@ //! Bound action helpers for OData write handlers. -use axum::http::{HeaderMap, StatusCode}; +use axum::http::StatusCode; use axum::response::IntoResponse; use opentelemetry::KeyValue as OtelKeyValue; use opentelemetry::trace::{Span, Status, Tracer}; @@ -14,12 +14,11 @@ use super::account_verification::enforce_commons_account_verified_for_action; use super::common::run_write_prechecks; use super::rate_limit::{enforce_commons_write_rate_limit, owner_id_from_action}; use super::response::annotate_entity; -use crate::authz::{DenialInput, record_authz_denial, security_context_from_headers}; +use crate::authz::{DenialInput, record_authz_denial}; use crate::blobs::hydrate_blob_refs_for_tenant; -use crate::identity::ResolvedIdentity; use crate::request_context::AgentContext; use crate::response::{ODataResponse, odata_error}; -use crate::state::{BoundActionHookContext, DispatchError, DispatchExtOptions, ServerState}; +use crate::state::{BoundActionHookContext, DispatchCommand, DispatchError, ServerState}; fn idempotency_actor_key(tenant: &TenantId, entity_type: &str, entity_id: &str) -> String { format!("{tenant}:{entity_type}:{entity_id}") @@ -35,10 +34,9 @@ pub(super) async fn dispatch_bound_action( action: &str, body_json: serde_json::Value, agent_ctx: &AgentContext, - headers: &HeaderMap, await_integration: bool, idempotency_key: Option, - resolved_identity: Option<&ResolvedIdentity>, + security_ctx: &SecurityContext, ) -> axum::response::Response { let http_start = sim_now(); let tracer = opentelemetry::global::tracer("temper"); @@ -72,33 +70,6 @@ pub(super) async fn dispatch_bound_action( )); } - // Build SecurityContext — credential-resolved identity (ADR-0033) or - // operator identity for global API key access. - let security_ctx = if let Some(identity) = resolved_identity { - http_span.set_attribute(OtelKeyValue::new( - "agent.id", - identity.agent_instance_id.clone(), - )); - http_span.set_attribute(OtelKeyValue::new( - "agent.type", - identity.agent_type_name.clone(), - )); - SecurityContext::from_resolved_identity( - &identity.agent_instance_id, - &identity.agent_type_name, - agent_ctx.session_id.as_deref(), - ) - } else { - // No credential resolved — operator/admin access via global API key. - // Build SecurityContext from X-Temper-Principal-Kind header (admin/system) - // without trusting self-declared identity fields. - security_context_from_headers( - headers, - None, // No self-declared agent_id - agent_ctx.session_id.as_deref(), - None, // No self-declared agent_type - ) - }; let mut dispatch_agent_ctx = agent_ctx.clone(); dispatch_agent_ctx.security_ctx = Some(security_ctx.clone()); @@ -148,45 +119,15 @@ pub(super) async fn dispatch_bound_action( return odata_error(StatusCode::INTERNAL_SERVER_ERROR, code, &e).into_response(); } }; + let expected_authorization_precondition = + crate::entity_actor::effects::entity_authorization_precondition( + &authz_snapshot.current_state.state, + ); let current_state = authz_snapshot.current_state; let resource_attrs = authz_snapshot.resource_attrs; - if let Err(resp) = enforce_commons_account_verified_for_action( - state, - tenant, - entity_type, - ¤t_state.state.fields, - &body_json, - ) - .await - { - http_span.set_status(Status::error("AccountVerificationRequired")); - http_span.set_attribute(OtelKeyValue::new("http.status_code", 403i64)); - let end_time: std::time::SystemTime = sim_now().into(); - http_span.end_with_timestamp(end_time); - return *resp; - } - - if let Err(resp) = enforce_commons_write_rate_limit( - state, - tenant, - entity_type, - owner_id_from_action(¤t_state.state.fields, &body_json), - headers, - agent_ctx, - resolved_identity, - ) - .await - { - http_span.set_status(Status::error("RateLimitExceeded")); - http_span.set_attribute(OtelKeyValue::new("http.status_code", 429i64)); - let end_time: std::time::SystemTime = sim_now().into(); - http_span.end_with_timestamp(end_time); - return resp; - } - let authz_result = state.authorize_with_context( - &security_ctx, + security_ctx, action, entity_type, &resource_attrs, @@ -198,7 +139,7 @@ pub(super) async fn dispatch_bound_action( state, DenialInput { tenant: tenant.as_str(), - security_ctx: &security_ctx, + security_ctx, agent_id_override: agent_ctx.agent_id.as_deref(), action, resource_type: entity_type, @@ -207,6 +148,11 @@ pub(super) async fn dispatch_bound_action( reason: &reason, module_name: None, from_status: Some(current_state.state.status.clone()), + intent: agent_ctx.intent.clone(), + session_id: agent_ctx.session_id.clone(), + // A genuine attempted dispatch of a registered action: walked by + // conformance, matching both parents' behavior. + spec_governed: None, }, ) .await; @@ -223,6 +169,38 @@ pub(super) async fn dispatch_bound_action( .into_response(); } + if let Err(resp) = enforce_commons_account_verified_for_action( + state, + tenant, + entity_type, + ¤t_state.state.fields, + &body_json, + ) + .await + { + http_span.set_status(Status::error("AccountVerificationRequired")); + http_span.set_attribute(OtelKeyValue::new("http.status_code", 403i64)); + let end_time: std::time::SystemTime = sim_now().into(); + http_span.end_with_timestamp(end_time); + return *resp; + } + + if let Err(resp) = enforce_commons_write_rate_limit( + state, + tenant, + entity_type, + owner_id_from_action(¤t_state.state.fields, &body_json), + security_ctx, + ) + .await + { + http_span.set_status(Status::error("RateLimitExceeded")); + http_span.set_attribute(OtelKeyValue::new("http.status_code", 429i64)); + let end_time: std::time::SystemTime = sim_now().into(); + http_span.end_with_timestamp(end_time); + return resp; + } + let current_fields = current_state.state.fields.clone(); if let Err(resp) = run_write_prechecks( state, @@ -267,17 +245,18 @@ pub(super) async fn dispatch_bound_action( } let result = state - .dispatch_tenant_action_ext_typed( - tenant, - entity_type, - key_str, - action, - body_json.clone(), - DispatchExtOptions { + .dispatch_tenant_action_ext_typed_if_current( + DispatchCommand { + tenant, + entity_type, + entity_id: key_str, + action, + params: body_json.clone(), agent_ctx: &dispatch_agent_ctx, await_integration, await_reactions: true, }, + expected_authorization_precondition, ) .await; diff --git a/crates/temper-server/src/odata/blob_media.rs b/crates/temper-server/src/odata/blob_media.rs new file mode 100644 index 000000000..4462f9d87 --- /dev/null +++ b/crates/temper-server/src/odata/blob_media.rs @@ -0,0 +1,236 @@ +//! Authenticated binary responses for Blob primitive values. + +use axum::body::Body; +use axum::http::header::{CONTENT_LENGTH, CONTENT_TYPE, ETAG}; +use axum::http::{HeaderValue, StatusCode}; +use axum::response::{IntoResponse, Response}; +use base64::Engine as _; +use temper_authz::SecurityContext; +use temper_runtime::tenant::TenantId; + +use super::authz::{READ_ACTION, authorize_read}; +use super::common::resolve_entity_type; +use super::read::{entity_set_not_found_response, load_existing_entity_descriptor_body}; +use crate::blobs::field_overflow_descriptor; +use crate::response::odata_error; +use crate::state::ServerState; + +const MAX_INLINE_BLOB_MEDIA_BYTES: u64 = 128 * 1024; + +pub(super) async fn handle_blob_primitive_stream( + state: &ServerState, + tenant: &TenantId, + security_ctx: &SecurityContext, + set_name: &str, + key: &str, + property: &str, +) -> Response { + let entity_type = match resolve_entity_type(state, tenant, set_name) { + Some(entity_type) if entity_type == "Blob" => entity_type, + Some(_) => { + return odata_error( + StatusCode::BAD_REQUEST, + "UnsupportedPrimitiveStream", + "primitive $value streaming is supported only for Blob binary fields", + ) + .into_response(); + } + None => return entity_set_not_found_response(state, tenant, set_name).await, + }; + if !matches!(property, "Content" | "CanonicalBytes") { + return odata_error( + StatusCode::BAD_REQUEST, + "UnsupportedBlobProperty", + "Blob primitive $value supports only Content and CanonicalBytes", + ) + .into_response(); + } + + let body = match load_existing_entity_descriptor_body( + state, + tenant, + &entity_type, + set_name, + key, + ) + .await + { + Ok(body) => body, + Err(response) => return response, + }; + if let Err(response) = authorize_read( + state, + tenant, + security_ctx, + READ_ACTION, + &entity_type, + key, + &body, + ) { + return *response; + } + + let fields = body.get("fields").unwrap_or(&body); + let Some(raw_size) = fields.get("Size").and_then(serde_json::Value::as_u64) else { + return odata_error( + StatusCode::CONFLICT, + "InvalidBlobMetadata", + "Blob.Size is missing or invalid", + ) + .into_response(); + }; + let Some(field) = fields.get(property) else { + return odata_error( + StatusCode::NOT_FOUND, + "BlobPropertyNotFound", + &format!("Blob('{key}').{property} is not available"), + ) + .into_response(); + }; + let decoded_size = if property == "Content" { + raw_size + } else { + let prefix_len = format!("blob {raw_size}\0").len() as u64; + match raw_size.checked_add(prefix_len) { + Some(size) => size, + None => { + return odata_error( + StatusCode::CONFLICT, + "InvalidBlobMetadata", + "Blob canonical size overflowed u64", + ) + .into_response(); + } + } + }; + let Some(expected_encoded_size) = decoded_size + .checked_add(2) + .map(|bytes| bytes / 3) + .and_then(|groups| groups.checked_mul(4)) + .and_then(|base64_bytes| base64_bytes.checked_add(2)) + else { + return odata_error( + StatusCode::CONFLICT, + "InvalidBlobMetadata", + "Blob encoded size overflowed u64", + ) + .into_response(); + }; + + if let Some(inline) = field.as_str() { + if decoded_size > MAX_INLINE_BLOB_MEDIA_BYTES { + return odata_error( + StatusCode::CONFLICT, + "InlineBlobMediaTooLarge", + "large Blob media must use a field-overflow descriptor", + ) + .into_response(); + } + if inline.len() as u64 + 2 != expected_encoded_size { + return odata_error( + StatusCode::CONFLICT, + "BlobMediaSizeMismatch", + "inline Blob media length does not match Blob.Size", + ) + .into_response(); + } + let decoded = match base64::engine::general_purpose::STANDARD.decode(inline) { + Ok(decoded) if decoded.len() as u64 == decoded_size => decoded, + Ok(_) => { + return odata_error( + StatusCode::CONFLICT, + "BlobMediaSizeMismatch", + "decoded inline Blob media length does not match Blob.Size", + ) + .into_response(); + } + Err(error) => { + return odata_error( + StatusCode::CONFLICT, + "InvalidBlobMedia", + &format!("inline Blob media is not valid base64: {error}"), + ) + .into_response(); + } + }; + return media_response(Body::from(decoded), decoded_size, key); + } + + let Some(descriptor) = field_overflow_descriptor(field) else { + return odata_error( + StatusCode::CONFLICT, + "InvalidBlobDescriptor", + &format!("Blob('{key}').{property} has an invalid media descriptor"), + ) + .into_response(); + }; + let blob_key = descriptor.key; + let encoded_size = descriptor.serialized_bytes; + if encoded_size != expected_encoded_size { + return odata_error( + StatusCode::CONFLICT, + "BlobMediaSizeMismatch", + "Blob media descriptor length does not match the decoded field size", + ) + .into_response(); + } + + let encoded = match state + .stream_blob_object(tenant, blob_key, encoded_size) + .await + { + Ok(crate::blob_store::BlobStreamRead::Found(stream)) => stream, + Ok(crate::blob_store::BlobStreamRead::Missing) => { + return odata_error( + StatusCode::NOT_FOUND, + "BlobMediaMissing", + &format!("Blob media object '{blob_key}' was not found"), + ) + .into_response(); + } + Ok(crate::blob_store::BlobStreamRead::TooLarge { .. }) => { + return odata_error( + StatusCode::CONFLICT, + "BlobMediaSizeMismatch", + "Blob media object exceeds its descriptor size", + ) + .into_response(); + } + Err(error) => { + tracing::error!(%error, %blob_key, "Blob media object-store read failed"); + return odata_error( + StatusCode::SERVICE_UNAVAILABLE, + "BlobMediaUnavailable", + "Blob media is temporarily unavailable", + ) + .into_response(); + } + }; + if encoded.content_length() != encoded_size { + return odata_error( + StatusCode::CONFLICT, + "BlobMediaSizeMismatch", + "Blob media object length does not match its descriptor", + ) + .into_response(); + } + let encoded = encoded.verify_sha256(descriptor.sha256); + let decoded = crate::blob_store::decode_json_base64_stream(encoded, decoded_size); + media_response(Body::from_stream(decoded.into_stream()), decoded_size, key) +} + +fn media_response(body: Body, decoded_size: u64, key: &str) -> Response { + let mut response = Response::new(body); + *response.status_mut() = StatusCode::OK; + response.headers_mut().insert( + CONTENT_TYPE, + HeaderValue::from_static("application/octet-stream"), + ); + if let Ok(value) = HeaderValue::from_str(&decoded_size.to_string()) { + response.headers_mut().insert(CONTENT_LENGTH, value); + } + if let Ok(value) = HeaderValue::from_str(&format!("\"{key}\"")) { + response.headers_mut().insert(ETAG, value); + } + response +} diff --git a/crates/temper-server/src/odata/common.rs b/crates/temper-server/src/odata/common.rs index f62465d1b..8ef35a60e 100644 --- a/crates/temper-server/src/odata/common.rs +++ b/crates/temper-server/src/odata/common.rs @@ -21,11 +21,21 @@ pub(crate) fn extract_tenant( headers: &HeaderMap, state: &ServerState, ) -> Result { - if let Some(val) = headers.get("x-tenant-id") - && let Ok(s) = val.to_str() - && !s.is_empty() - { - return Ok(TenantId::new(s)); + if let Some(value) = headers.get("x-tenant-id") { + let tenant = value.to_str().map(str::trim).map_err(|_| { + ( + StatusCode::BAD_REQUEST, + "Invalid X-Tenant-Id header encoding".to_string(), + ) + })?; + if !tenant.is_empty() { + return TenantId::try_new(tenant).map_err(|error| { + ( + StatusCode::BAD_REQUEST, + format!("Invalid X-Tenant-Id header: {error}"), + ) + }); + } } // Multi-tenant mode: require explicit tenant header. @@ -198,30 +208,6 @@ pub(super) async fn run_write_prechecks( Ok(()) } -/// Load an entity's current state or return a 404 response. -/// -/// Consolidates the repeated pattern of calling `get_tenant_entity_state` -/// and mapping errors to OData error responses. -pub(super) async fn load_entity_or_404( - state: &ServerState, - tenant: &TenantId, - entity_type: &str, - set_name: &str, - key: &str, -) -> Result { - state - .get_tenant_entity_state(tenant, entity_type, key) - .await - .map_err(|e| { - crate::response::odata_error( - StatusCode::NOT_FOUND, - "ResourceNotFound", - &format!("Entity '{set_name}' with key '{key}' not found: {e}"), - ) - .into_response() - }) -} - /// Resolve the parent of a `$value` path to `(set_name, entity_id)`. /// /// Returns 400 if the parent is not an entity instance. diff --git a/crates/temper-server/src/odata/content_addressed.rs b/crates/temper-server/src/odata/content_addressed.rs index e7e2cafea..521a203f2 100644 --- a/crates/temper-server/src/odata/content_addressed.rs +++ b/crates/temper-server/src/odata/content_addressed.rs @@ -23,23 +23,30 @@ use axum::body::Body; use axum::extract::{Extension, State}; use axum::http::{HeaderMap, StatusCode}; use axum::response::IntoResponse; -use sha1::Digest; -use tokio_stream::StreamExt as _; +use futures_util::TryStreamExt as _; +use temper_authz::AuthenticatedRequestContext; +use temper_runtime::scheduler::sim_now; use super::account_verification::enforce_commons_account_verified_for_write; use super::authz::{ - CREATE_ACTION, MutationResource, authorize_mutation, request_security_context, - resource_attrs_from_body, + CREATE_ACTION, MutationResource, apply_authenticated_context, authorize_mutation, + require_authenticated_context, }; -use super::common::{extract_tenant, run_write_prechecks}; +use super::common::run_write_prechecks; use super::response::annotate_entity; -use super::storage_guardrails::enforce_commons_storage_cap; -use crate::identity::ResolvedIdentity; +use super::storage_guardrails::storage_cap_error_response; +use crate::blob_store::{BlobByteStream, BlobIngestAdmissionError, MAX_RAW_BLOB_BYTES}; +use crate::blobs::{FIELD_OVERFLOW_BLOB_PREFIX, blob_ref_value}; use crate::request_context::extract_agent_context; use crate::response::{ODataResponse, odata_error}; use crate::state::ServerState; -const MAX_OBJECT_BYTES: usize = 2 * 1024 * 1024 * 1024; +const EXPECTED_OBJECT_ID_HEADER: &str = "x-expected-object-id"; + +mod responses; +use responses::{ + blob_store_error_response, remove_binary_fields_from_create_response, stage_error_response, +}; /// `POST /tdata/Blobs/Temper.IngestRaw` — stream raw blob bytes, /// hash them, persist a `Blob` row keyed by the SHA-1 of the @@ -50,43 +57,37 @@ const MAX_OBJECT_BYTES: usize = 2 * 1024 * 1024 * 1024; /// canonical hash prefix and as a defence against open-ended /// streams. /// * `X-Repository-Id` — foreign key back to the parent repo. +/// * `X-Expected-Object-Id` — lowercase SHA-1 of the canonical object, +/// required so Cedar and quota admission use the exact resource before +/// the request body is polled. /// -/// Optional: -/// * `X-Tenant-Id`, principal headers — same as any OData write. +/// Authority and tenant are supplied by the authenticated typed request +/// context, exactly like every protected OData write. pub async fn handle_blob_ingest_raw( State(state): State, - resolved_id: Option>, + authenticated: Option>, headers: HeaderMap, body: Body, ) -> impl IntoResponse { - ingest_raw_inner( - state, - resolved_id.map(|Extension(identity)| identity), - headers, - body, - "Blob", - "blob", - ) - .await + let authenticated = match require_authenticated_context(authenticated) { + Ok(context) => context, + Err(error) => return error.into_response(), + }; + ingest_raw_inner(state, authenticated, headers, body, "Blob", "blob").await } async fn ingest_raw_inner( state: ServerState, - resolved_identity: Option, + authenticated: AuthenticatedRequestContext, headers: HeaderMap, body: Body, entity_type: &str, kind_tag: &str, ) -> axum::response::Response { - let tenant = match extract_tenant(&headers, &state) { - Ok(t) => t, - Err(e) => return e.into_response(), - }; + let tenant = authenticated.tenant().clone(); + let security_ctx = authenticated.security_context().clone(); let mut agent_ctx = extract_agent_context(&headers); - if let Some(ref identity) = resolved_identity { - agent_ctx.agent_id = Some(identity.agent_instance_id.clone()); - agent_ctx.agent_type = Some(identity.agent_type_name.clone()); - } + apply_authenticated_context(&mut agent_ctx, &security_ctx); let repository_id = match headers .get("x-repository-id") @@ -105,6 +106,11 @@ async fn ingest_raw_inner( } }; + let expected_object_id = match expected_object_id(&headers) { + Ok(object_id) => object_id, + Err(response) => return *response, + }; + let declared_len = match headers .get("content-length") .and_then(|v| v.to_str().ok()) @@ -120,117 +126,253 @@ async fn ingest_raw_inner( .into_response(); } }; - if declared_len > MAX_OBJECT_BYTES { - return odata_error( - StatusCode::PAYLOAD_TOO_LARGE, - "ObjectTooLarge", - &format!("declared size {declared_len} exceeds {MAX_OBJECT_BYTES}"), + let now = sim_now().to_rfc3339(); + let admission_fields = serde_json::json!({ + "Id": expected_object_id, + "RepositoryId": repository_id, + "Size": declared_len as i64, + "Status": "Durable", + "CreatedAt": now, + }); + let attrs = match state + .build_create_authz_resource_attrs( + &tenant, + entity_type, + &expected_object_id, + &admission_fields, ) - .into_response(); + .await + { + Ok(attrs) => attrs, + Err(error) => { + return odata_error(StatusCode::INTERNAL_SERVER_ERROR, "ReadError", &error) + .into_response(); + } + }; + if let Err(response) = authorize_mutation( + &state, + &tenant, + &security_ctx, + &agent_ctx, + CREATE_ACTION, + MutationResource { + entity_type, + entity_id: &expected_object_id, + attrs: &attrs, + }, + ) + .await + { + return response; } - // Hash the canonical bytes incrementally. We need the body - // around to write into the row, but the hasher runs in - // streaming fashion regardless — Vec is just the sink. - let mut hasher = sha1::Sha1::new(); - let prefix = format!("{kind_tag} {declared_len}\0"); - hasher.update(prefix.as_bytes()); - - let mut content = Vec::with_capacity(declared_len); - let mut stream = body.into_data_stream(); - while let Some(chunk_result) = stream.next().await { - let chunk = match chunk_result { - Ok(c) => c, - Err(e) => { - return odata_error( - StatusCode::BAD_REQUEST, - "BodyStreamError", - &format!("body stream failed: {e}"), - ) - .into_response(); - } - }; - if content.len() + chunk.len() > declared_len { + // Reserve staging capacity only after Cedar admits the exact expected + // object ID/repository/size. Otherwise denied credentials could occupy the + // tenant or global upload permits without ever being allowed to poll a body. + let mut staging_admission = match state + .raw_blob_ingest_budget + .try_reserve(&tenant, declared_len) + { + Ok(permit) => permit, + Err(BlobIngestAdmissionError::ObjectTooLarge) => { return odata_error( - StatusCode::BAD_REQUEST, - "BodyExceedsContentLength", - "body bytes exceed declared Content-Length", + StatusCode::PAYLOAD_TOO_LARGE, + "ObjectTooLarge", + &format!( + "declared size {declared_len} exceeds the {} byte object or {} byte staging budget", + MAX_RAW_BLOB_BYTES, + state.raw_blob_ingest_budget.capacity_bytes() + ), + ) + .into_response(); + } + Err(BlobIngestAdmissionError::BudgetExhausted) => { + return odata_error( + StatusCode::TOO_MANY_REQUESTS, + "BlobIngestBudgetExhausted", + "Concurrent raw Blob staging has exhausted its admission budget", ) .into_response(); } - hasher.update(&chunk); - content.extend_from_slice(&chunk); + Err(BlobIngestAdmissionError::TenantBusy) => { + return odata_error( + StatusCode::TOO_MANY_REQUESTS, + "BlobIngestTenantBusy", + "This tenant already has a raw Blob upload in progress", + ) + .into_response(); + } + }; + + // Snapshot repository/account/quota admission under the existing commons + // mutation lock, then keep only the owner-byte reservation across I/O. A + // slow upload must not hold the coarse cross-tenant lock. + let admission_guard = state.acquire_commons_write_guardrail_lock(&tenant).await; + + if let Err(response) = run_write_prechecks( + &state, + &tenant, + entity_type, + &expected_object_id, + "Create", + "create", + &admission_fields, + ) + .await + { + return response; + } + if let Err(response) = + enforce_commons_account_verified_for_write(&state, &tenant, entity_type, &admission_fields) + .await + { + return *response; } - if content.len() != declared_len { + let mut storage_reservation = match state + .reserve_commons_blob_storage( + &tenant, + &expected_object_id, + &repository_id, + declared_len as i64, + ) + .await + { + Ok(reservation) => reservation, + Err(error) => return storage_cap_error_response(error), + }; + drop(admission_guard); + + let blob_store = match state.blob_store_for_tenant(&tenant) { + Ok(store) => store, + Err(error) => { + return odata_error( + StatusCode::SERVICE_UNAVAILABLE, + "BlobStoreUnavailable", + &error, + ) + .into_response(); + } + }; + let canonical_prefix = format!("{kind_tag} {declared_len}\0"); + let stream: BlobByteStream = Box::pin( + body.into_data_stream() + .map_err(|error| std::io::Error::other(error.to_string())), + ); + let staged = match blob_store + .stage_canonical_stream( + stream, + declared_len, + canonical_prefix.as_bytes(), + state.raw_blob_ingest_budget.progress_policy(), + &mut staging_admission, + ) + .await + { + Ok(staged) => staged, + Err(error) => return stage_error_response(error), + }; + if staged.canonical_sha1() != expected_object_id { return odata_error( - StatusCode::BAD_REQUEST, - "BodyShorterThanContentLength", - &format!("expected {declared_len} body bytes, got {}", content.len()), + StatusCode::UNPROCESSABLE_ENTITY, + "ObjectDigestMismatch", + &format!( + "expected object ID {expected_object_id}, computed {}", + staged.canonical_sha1() + ), ) .into_response(); } - let sha = hex_lower(&hasher.finalize()); + let content_descriptor = match staged.base64_json_descriptor(&[]).await { + Ok(descriptor) => descriptor, + Err(error) => return blob_store_error_response(&error), + }; + let content_key = format!( + "{FIELD_OVERFLOW_BLOB_PREFIX}{}.json", + content_descriptor.sha256 + ); + if let Err(error) = blob_store + .put_staged_base64_json( + &content_key, + &staged, + &[], + content_descriptor.serialized_len, + ) + .await + { + return blob_store_error_response(&error); + } - use base64::Engine; - let b64 = base64::engine::general_purpose::STANDARD; - let canonical_b64 = { - let mut buf = Vec::with_capacity(prefix.len() + content.len()); - buf.extend_from_slice(prefix.as_bytes()); - buf.extend_from_slice(&content); - b64.encode(&buf) + let canonical_descriptor = match staged + .base64_json_descriptor(canonical_prefix.as_bytes()) + .await + { + Ok(descriptor) => descriptor, + Err(error) => return blob_store_error_response(&error), }; - let content_b64 = b64.encode(&content); + let canonical_key = format!( + "{FIELD_OVERFLOW_BLOB_PREFIX}{}.json", + canonical_descriptor.sha256 + ); + if let Err(error) = blob_store + .put_staged_base64_json( + &canonical_key, + &staged, + canonical_prefix.as_bytes(), + canonical_descriptor.serialized_len, + ) + .await + { + return blob_store_error_response(&error); + } - let now = chrono::Utc::now().to_rfc3339(); let initial_fields = serde_json::json!({ - "Id": sha, + "Id": expected_object_id, "RepositoryId": repository_id, "Size": declared_len as i64, - "Content": content_b64, - "CanonicalBytes": canonical_b64, + "Content": blob_ref_value(&content_key, content_descriptor.serialized_len), + "CanonicalBytes": blob_ref_value(&canonical_key, canonical_descriptor.serialized_len), "Status": "Durable", "CreatedAt": now, }); - - let _commons_guardrail_lock = state.acquire_commons_write_guardrail_lock(&tenant).await; - - if let Err(resp) = run_write_prechecks( + let final_guard = state.acquire_commons_write_guardrail_lock(&tenant).await; + // Convert the pending reservation back into a final cap check while the + // mutation lock prevents another writer from taking the released bytes. + drop(storage_reservation.take()); + if let Err(error) = state + .enforce_commons_storage_cap_for_write( + &tenant, + entity_type, + &expected_object_id, + "Create", + &initial_fields, + ) + .await + { + drop(final_guard); + return storage_cap_error_response(error); + } + if let Err(response) = run_write_prechecks( &state, &tenant, entity_type, - &sha, + &expected_object_id, "Create", "create", &initial_fields, ) .await { - return resp; + drop(final_guard); + return response; } - - if let Err(resp) = + if let Err(response) = enforce_commons_account_verified_for_write(&state, &tenant, entity_type, &initial_fields) .await { - return *resp; - } - - if let Err(resp) = enforce_commons_storage_cap( - &state, - &tenant, - entity_type, - &sha, - "Create", - &initial_fields, - ) - .await - { - return resp; + drop(final_guard); + return *response; } - - let security_ctx = request_security_context(&headers, &agent_ctx, resolved_identity.as_ref()); - let attrs = resource_attrs_from_body(&state, &tenant, entity_type, &sha, &initial_fields); if let Err(response) = authorize_mutation( &state, &tenant, @@ -239,28 +381,30 @@ async fn ingest_raw_inner( CREATE_ACTION, MutationResource { entity_type, - entity_id: &sha, + entity_id: &expected_object_id, attrs: &attrs, }, ) .await { + drop(final_guard); return response; } - match state - .get_or_create_tenant_entity(&tenant, entity_type, &sha, initial_fields) - .await - { + let create_result = state + .get_or_create_tenant_entity(&tenant, entity_type, &expected_object_id, initial_fields) + .await; + match create_result { Ok(response) => { let _ = agent_ctx; state.clear_commons_storage_projection_cache_for_entity(entity_type); + drop(final_guard); let mut state_json = serde_json::to_value(&response.state).unwrap_or_default(); - crate::blobs::hydrate_blob_refs_for_tenant(&state, &tenant, &mut state_json).await; + remove_binary_fields_from_create_response(&mut state_json); let body = annotate_entity( state_json, format!("$metadata#{entity_type}s/$entity"), - Some(format!("{entity_type}s('{sha}')")), + Some(format!("{entity_type}s('{expected_object_id}')")), ); ODataResponse { status: StatusCode::CREATED, @@ -268,21 +412,55 @@ async fn ingest_raw_inner( } .into_response() } - Err(e) => odata_error( - StatusCode::INTERNAL_SERVER_ERROR, - "EntityCreateFailed", - &e.to_string(), - ) - .into_response(), + Err(error) => { + drop(final_guard); + odata_error( + StatusCode::INTERNAL_SERVER_ERROR, + "EntityCreateFailed", + &error.to_string(), + ) + .into_response() + } } } -fn hex_lower(bytes: &[u8]) -> String { - const HEX: &[u8; 16] = b"0123456789abcdef"; - let mut s = String::with_capacity(bytes.len() * 2); - for &b in bytes { - s.push(HEX[(b >> 4) as usize] as char); - s.push(HEX[(b & 0x0f) as usize] as char); +fn expected_object_id(headers: &HeaderMap) -> Result> { + let Some(value) = headers.get(EXPECTED_OBJECT_ID_HEADER) else { + return Err(Box::new( + odata_error( + StatusCode::PRECONDITION_REQUIRED, + "MissingExpectedObjectId", + "X-Expected-Object-Id header required for pre-body authorization", + ) + .into_response(), + )); + }; + let object_id = value.to_str().map(str::trim).map_err(|_| { + Box::new( + odata_error( + StatusCode::BAD_REQUEST, + "InvalidExpectedObjectId", + "X-Expected-Object-Id must be visible ASCII", + ) + .into_response(), + ) + })?; + if object_id.len() != 40 + || !object_id + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return Err(Box::new( + odata_error( + StatusCode::BAD_REQUEST, + "InvalidExpectedObjectId", + "X-Expected-Object-Id must be 40 lowercase hexadecimal characters", + ) + .into_response(), + )); } - s + Ok(object_id.to_string()) } + +#[cfg(test)] +mod tests; diff --git a/crates/temper-server/src/odata/content_addressed/responses.rs b/crates/temper-server/src/odata/content_addressed/responses.rs new file mode 100644 index 000000000..41299b720 --- /dev/null +++ b/crates/temper-server/src/odata/content_addressed/responses.rs @@ -0,0 +1,60 @@ +//! Error and metadata-only response helpers for raw Blob ingest. + +use axum::http::StatusCode; +use axum::response::IntoResponse as _; + +use crate::blob_store::BlobStageError; +use crate::response::odata_error; + +pub(super) fn stage_error_response(error: BlobStageError) -> axum::response::Response { + let (status, code) = match &error { + BlobStageError::BodyStream(_) => (StatusCode::BAD_REQUEST, "BodyStreamError"), + BlobStageError::BodyExceedsDeclaredLength { .. } => { + (StatusCode::BAD_REQUEST, "BodyExceedsContentLength") + } + BlobStageError::BodyShorterThanDeclaredLength { .. } => { + (StatusCode::BAD_REQUEST, "BodyShorterThanContentLength") + } + BlobStageError::IdleTimeout { .. } => { + (StatusCode::REQUEST_TIMEOUT, "BlobIngestIdleTimeout") + } + BlobStageError::TotalDeadline { .. } => { + (StatusCode::REQUEST_TIMEOUT, "BlobIngestDeadlineExceeded") + } + BlobStageError::ThroughputTooLow { .. } => { + (StatusCode::REQUEST_TIMEOUT, "BlobIngestThroughputTooLow") + } + BlobStageError::StagingBudgetExhausted { .. } => ( + StatusCode::TOO_MANY_REQUESTS, + "BlobIngestStagingBudgetExhausted", + ), + BlobStageError::Storage(_) => (StatusCode::INSUFFICIENT_STORAGE, "BlobStagingFailed"), + }; + let message = if matches!(&error, BlobStageError::Storage(_)) { + tracing::error!(%error, "raw Blob staging failed"); + "Blob staging failed".to_string() + } else { + error.to_string() + }; + odata_error(status, code, &message).into_response() +} + +pub(super) fn blob_store_error_response(error: &str) -> axum::response::Response { + tracing::error!(%error, "raw Blob object-store write failed"); + odata_error( + StatusCode::SERVICE_UNAVAILABLE, + "BlobStoreWriteFailed", + "Blob object-store write failed", + ) + .into_response() +} + +pub(super) fn remove_binary_fields_from_create_response(state: &mut serde_json::Value) { + if let Some(fields) = state + .get_mut("fields") + .and_then(serde_json::Value::as_object_mut) + { + fields.remove("Content"); + fields.remove("CanonicalBytes"); + } +} diff --git a/crates/temper-server/src/odata/content_addressed/tests.rs b/crates/temper-server/src/odata/content_addressed/tests.rs new file mode 100644 index 000000000..075f14a76 --- /dev/null +++ b/crates/temper-server/src/odata/content_addressed/tests.rs @@ -0,0 +1,445 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Duration; + +use axum::Router; +use axum::body::{Body, Bytes}; +use axum::http::{Request, StatusCode}; +use sha1::Digest as _; +use temper_authz::{AuthenticatedRequestContext, Principal, PrincipalKind, SecurityContext}; +use temper_runtime::ActorSystem; +use temper_runtime::tenant::TenantId; +use temper_spec::csdl::parse_csdl; +use tower::ServiceExt as _; +use wiremock::matchers::method; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +use crate::blob_store::{BlobIngestBudget, BlobIngestProgressPolicy}; +use crate::blobs::FIELD_OVERFLOW_REF_KEY; +use crate::secrets::vault::SecretsVault; +use crate::state::ServerState; + +const BLOB_CSDL: &str = r#" + + + + + + + + + + + + + + + + + + +"#; + +const BLOB_IOA: &str = r#" +[automaton] +name = "Blob" +states = ["Durable"] +initial = "Durable" + +[[action]] +name = "Create" +kind = "input" +from = ["Durable"] +to = "Durable" +params = ["RepositoryId", "Size", "Content", "CanonicalBytes", "CreatedAt"] +"#; + +fn security_context() -> SecurityContext { + SecurityContext { + principal: Principal { + id: "raw-ingest-test".to_string(), + kind: PrincipalKind::Customer, + role: None, + acting_for: None, + agent_type: None, + attributes: Default::default(), + }, + context_attrs: Default::default(), + correlation_id: "raw-ingest-test".to_string(), + } +} + +async fn authenticate( + mut request: axum::extract::Request, + next: axum::middleware::Next, +) -> axum::response::Response { + request + .extensions_mut() + .insert(AuthenticatedRequestContext::new( + TenantId::default(), + security_context(), + )); + next.run(request).await +} + +fn test_state() -> (ServerState, tempfile::TempDir) { + let csdl = parse_csdl(BLOB_CSDL).expect("Blob test CSDL"); + let mut specs = std::collections::BTreeMap::new(); + specs.insert("Blob".to_string(), BLOB_IOA.to_string()); + let mut state = ServerState::with_specs( + ActorSystem::new("raw-blob-ingest-http"), + csdl, + BLOB_CSDL.to_string(), + specs, + ) + .expect("Blob test state"); + let data_dir = tempfile::tempdir().expect("raw Blob data dir"); + state.data_dir = data_dir.path().to_path_buf(); + allow_blob_create(&state); + (state, data_dir) +} + +fn allow_blob_create(state: &ServerState) { + state + .authz + .reload_tenant_policies( + "default", + r#" +permit(principal, action == Action::"create", resource is Blob); +permit(principal, action == Action::"read", resource is Blob); +"#, + ) + .expect("install Blob create policy"); +} + +fn app(state: ServerState) -> Router { + crate::router::build_router(state).layer(axum::middleware::from_fn(authenticate)) +} + +fn git_blob_id(body: &[u8]) -> String { + let mut hasher = sha1::Sha1::new(); + hasher.update(format!("blob {}\0", body.len()).as_bytes()); + hasher.update(body); + format!("{:x}", hasher.finalize()) +} + +fn ingest_request(body: Body, declared_len: usize, expected_id: &str) -> Request { + Request::post("/tdata/Blobs/Temper.IngestRaw") + .header("Content-Type", "application/octet-stream") + .header("Content-Length", declared_len.to_string()) + .header("X-Expected-Object-Id", expected_id) + .header("X-Repository-Id", "repository-1") + .body(body) + .expect("raw Blob request") +} + +fn counted_body(bytes: &'static [u8], polls: Arc) -> Body { + Body::from_stream(futures_util::stream::once(async move { + polls.fetch_add(1, Ordering::SeqCst); + Ok::<_, std::io::Error>(Bytes::from_static(bytes)) + })) +} + +#[tokio::test] +async fn raw_ingest_cors_preflight_allows_required_headers() { + let (state, _data_dir) = test_state(); + let response = app(state) + .oneshot( + Request::builder() + .method("OPTIONS") + .uri("/tdata/Blobs/Temper.IngestRaw") + .header("Origin", "https://client.example") + .header("Access-Control-Request-Method", "POST") + .header( + "Access-Control-Request-Headers", + "content-type,x-repository-id,x-expected-object-id", + ) + .body(Body::empty()) + .expect("preflight request"), + ) + .await + .expect("preflight response"); + + assert_eq!(response.status(), StatusCode::OK); + let allowed = response + .headers() + .get("access-control-allow-headers") + .and_then(|value| value.to_str().ok()) + .expect("allowed CORS headers") + .to_ascii_lowercase(); + assert!(allowed.contains("x-repository-id")); + assert!(allowed.contains("x-expected-object-id")); +} + +async fn staging_entry_count(path: &std::path::Path) -> usize { + if !tokio::fs::try_exists(path).await.unwrap_or(false) { + return 0; + } + let mut entries = tokio::fs::read_dir(path).await.expect("read staging dir"); + let mut count = 0; + while entries + .next_entry() + .await + .expect("next staging entry") + .is_some() + { + count += 1; + } + count +} + +#[tokio::test] +async fn raw_ingest_streams_overflow_fields_and_returns_metadata_only() { + let (state, _data_dir) = test_state(); + let body = b"abc"; + let object_id = git_blob_id(body); + let response = app(state.clone()) + .oneshot(ingest_request( + Body::from(body.as_slice()), + body.len(), + &object_id, + )) + .await + .expect("raw Blob response"); + + assert_eq!(response.status(), StatusCode::CREATED); + let response_bytes = axum::body::to_bytes(response.into_body(), 1024 * 1024) + .await + .expect("response body"); + let response_json: serde_json::Value = + serde_json::from_slice(&response_bytes).expect("response JSON"); + assert_eq!(response_json["fields"]["Id"], object_id); + assert!(response_json["fields"].get("Content").is_none()); + assert!(response_json["fields"].get("CanonicalBytes").is_none()); + + let persisted = state + .get_tenant_entity_state(&TenantId::default(), "Blob", &object_id) + .await + .expect("persisted Blob"); + assert!( + persisted.state.fields["Content"] + .get(FIELD_OVERFLOW_REF_KEY) + .is_some() + ); + assert!( + persisted.state.fields["CanonicalBytes"] + .get(FIELD_OVERFLOW_REF_KEY) + .is_some() + ); + + let mut fields = persisted.state.fields.clone(); + crate::blobs::hydrate_blob_refs_for_tenant(&state, &TenantId::default(), &mut fields).await; + assert_eq!(fields["Content"], "YWJj"); + assert_eq!(fields["CanonicalBytes"], "YmxvYiAzAGFiYw=="); +} + +#[tokio::test] +async fn blob_property_value_preserves_small_inline_media() { + let (state, _data_dir) = test_state(); + let body = b"abc"; + let object_id = git_blob_id(body); + state + .get_or_create_tenant_entity( + &TenantId::default(), + "Blob", + &object_id, + serde_json::json!({ + "Id": object_id, + "RepositoryId": "repository-1", + "Size": body.len() as i64, + "Content": "YWJj", + "CanonicalBytes": "YmxvYiAzAGFiYw==", + "Status": "Durable", + "CreatedAt": "2026-01-01T00:00:00Z", + }), + ) + .await + .expect("create inline Blob"); + let router = app(state); + + let content = router + .clone() + .oneshot( + Request::get(format!("/tdata/Blobs('{object_id}')/Content/$value")) + .body(Body::empty()) + .expect("content request"), + ) + .await + .expect("content response"); + assert_eq!(content.status(), StatusCode::OK); + assert_eq!( + axum::body::to_bytes(content.into_body(), 4) + .await + .expect("inline content"), + body.as_slice() + ); + + let canonical = router + .oneshot( + Request::get(format!("/tdata/Blobs('{object_id}')/CanonicalBytes/$value")) + .body(Body::empty()) + .expect("canonical request"), + ) + .await + .expect("canonical response"); + assert_eq!(canonical.status(), StatusCode::OK); + assert_eq!( + axum::body::to_bytes(canonical.into_body(), 11) + .await + .expect("inline canonical"), + b"blob 3\0abc".as_slice() + ); +} + +#[tokio::test] +async fn unauthorized_raw_ingest_does_not_poll_body() { + let (state, _data_dir) = test_state(); + state + .authz + .reload_tenant_policies( + "default", + r#"permit(principal, action == Action::"read", resource is Blob);"#, + ) + .expect("install deny-create policy"); + let polls = Arc::new(AtomicUsize::new(0)); + let response = app(state.clone()) + .oneshot(ingest_request( + counted_body(b"abc", polls.clone()), + 3, + &git_blob_id(b"abc"), + )) + .await + .expect("authorization response"); + + assert_eq!(response.status(), StatusCode::FORBIDDEN); + assert_eq!(polls.load(Ordering::SeqCst), 0); + assert!( + state + .list_entity_ids(&TenantId::default(), "Blob") + .is_empty() + ); +} + +#[tokio::test] +async fn declared_size_over_budget_does_not_poll_body() { + let (mut state, _data_dir) = test_state(); + state.raw_blob_ingest_budget = BlobIngestBudget::new(2, 1); + let polls = Arc::new(AtomicUsize::new(0)); + let response = app(state.clone()) + .oneshot(ingest_request( + counted_body(b"abc", polls.clone()), + 3, + &git_blob_id(b"abc"), + )) + .await + .expect("budget response"); + + assert_eq!(response.status(), StatusCode::PAYLOAD_TOO_LARGE); + assert_eq!(polls.load(Ordering::SeqCst), 0); + assert!( + state + .list_entity_ids(&TenantId::default(), "Blob") + .is_empty() + ); +} + +#[tokio::test] +async fn wrong_digest_cleans_staging_and_creates_no_entity() { + let (state, data_dir) = test_state(); + let response = app(state.clone()) + .oneshot(ingest_request(Body::from("abc"), 3, &git_blob_id(b"abd"))) + .await + .expect("digest response"); + + assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY); + assert!( + state + .list_entity_ids(&TenantId::default(), "Blob") + .is_empty() + ); + assert_eq!( + staging_entry_count(&data_dir.path().join("blobs/.ingest-staging")).await, + 0 + ); +} + +#[tokio::test] +async fn short_and_long_bodies_clean_staging_and_create_no_entity() { + for (body, declared_len, expected_status) in [ + ("ab", 3usize, StatusCode::BAD_REQUEST), + ("abc", 2usize, StatusCode::BAD_REQUEST), + ] { + let (state, data_dir) = test_state(); + let expected_id = if declared_len == 3 { + git_blob_id(b"abc") + } else { + git_blob_id(b"ab") + }; + let response = app(state.clone()) + .oneshot(ingest_request( + Body::from(body.to_string()), + declared_len, + &expected_id, + )) + .await + .expect("length response"); + assert_eq!(response.status(), expected_status); + assert!( + state + .list_entity_ids(&TenantId::default(), "Blob") + .is_empty() + ); + assert_eq!( + staging_entry_count(&data_dir.path().join("blobs/.ingest-staging")).await, + 0 + ); + } +} + +#[tokio::test] +async fn concurrent_declared_bytes_cannot_exceed_budget() { + let (mut state, data_dir) = test_state(); + state.raw_blob_ingest_budget = BlobIngestBudget::new(3, 1); + let router = app(state.clone()); + let first_polls = Arc::new(AtomicUsize::new(0)); + let first_counter = first_polls.clone(); + let first_body = Body::from_stream(async_stream::stream! { + first_counter.fetch_add(1, Ordering::SeqCst); + yield Ok::<_, std::io::Error>(Bytes::from_static(b"a")); + std::future::pending::<()>().await; + }); + let first = tokio::spawn(router.clone().oneshot(ingest_request( + first_body, + 3, + &git_blob_id(b"abc"), + ))); + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while first_polls.load(Ordering::SeqCst) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("first upload should start"); + + let rejected_polls = Arc::new(AtomicUsize::new(0)); + let second = router + .oneshot(ingest_request( + counted_body(b"xyz", rejected_polls.clone()), + 3, + &git_blob_id(b"xyz"), + )) + .await + .expect("concurrent budget response"); + assert_eq!(second.status(), StatusCode::TOO_MANY_REQUESTS); + assert_eq!(rejected_polls.load(Ordering::SeqCst), 0); + + first.abort(); + let _ = first.await; + tokio::task::yield_now().await; + assert_eq!( + staging_entry_count(&data_dir.path().join("blobs/.ingest-staging")).await, + 0 + ); +} + +mod failure_and_streaming; diff --git a/crates/temper-server/src/odata/content_addressed/tests/failure_and_streaming.rs b/crates/temper-server/src/odata/content_addressed/tests/failure_and_streaming.rs new file mode 100644 index 000000000..5a5ad7611 --- /dev/null +++ b/crates/temper-server/src/odata/content_addressed/tests/failure_and_streaming.rs @@ -0,0 +1,205 @@ +use super::*; + +#[tokio::test] +async fn object_store_failure_creates_no_entity_and_cleans_staging() { + let server = MockServer::start().await; + Mock::given(method("PUT")) + .respond_with(ResponseTemplate::new(500)) + .mount(&server) + .await; + let (mut state, data_dir) = test_state(); + let vault = SecretsVault::new(&[7u8; 32]); + vault + .cache_secret("default", "blob_endpoint", server.uri()) + .expect("cache endpoint"); + vault + .cache_secret("default", "blob_bucket", "test-bucket".to_string()) + .expect("cache bucket"); + state.secrets_vault = Some(Arc::new(vault)); + + let response = app(state.clone()) + .oneshot(ingest_request(Body::from("abc"), 3, &git_blob_id(b"abc"))) + .await + .expect("backend failure response"); + + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert!( + state + .list_entity_ids(&TenantId::default(), "Blob") + .is_empty() + ); + assert_eq!( + staging_entry_count(&data_dir.path().join("blob-ingest-staging")).await, + 0 + ); +} + +#[tokio::test] +async fn stalled_upload_times_out_releases_admission_and_cleans_staging() { + let (mut state, data_dir) = test_state(); + state.raw_blob_ingest_budget = BlobIngestBudget::with_limits( + 16, + 1, + 2, + 1, + BlobIngestProgressPolicy::new( + Duration::from_millis(25), + Duration::from_secs(1), + Duration::from_secs(1), + Duration::from_millis(50), + 1, + ), + ); + let router = app(state.clone()); + let stalled = + Body::from_stream(futures_util::stream::pending::>()); + let response = tokio::time::timeout( + Duration::from_secs(1), + router + .clone() + .oneshot(ingest_request(stalled, 3, &git_blob_id(b"abc"))), + ) + .await + .expect("stalled upload must be bounded") + .expect("stalled upload response"); + assert_eq!(response.status(), StatusCode::REQUEST_TIMEOUT); + assert_eq!( + staging_entry_count(&data_dir.path().join("blobs/.ingest-staging")).await, + 0 + ); + + let retry = router + .oneshot(ingest_request(Body::from("abc"), 3, &git_blob_id(b"abc"))) + .await + .expect("retry response"); + assert_eq!(retry.status(), StatusCode::CREATED); +} + +#[tokio::test] +async fn trickled_upload_fails_minimum_throughput_and_cleans_staging() { + let (mut state, data_dir) = test_state(); + state.raw_blob_ingest_budget = BlobIngestBudget::with_limits( + 16, + 1, + 2, + 1, + BlobIngestProgressPolicy::new( + Duration::from_millis(250), + Duration::from_secs(1), + Duration::from_millis(20), + Duration::from_millis(20), + 10_000, + ), + ); + let body = Body::from_stream(async_stream::stream! { + yield Ok::<_, std::io::Error>(Bytes::from_static(b"a")); + tokio::time::sleep(Duration::from_millis(100)).await; + yield Ok::<_, std::io::Error>(Bytes::from_static(b"bc")); + }); + let response = app(state.clone()) + .oneshot(ingest_request(body, 3, &git_blob_id(b"abc"))) + .await + .expect("trickle response"); + assert_eq!(response.status(), StatusCode::REQUEST_TIMEOUT); + assert_eq!( + staging_entry_count(&data_dir.path().join("blobs/.ingest-staging")).await, + 0 + ); + assert!( + state + .list_entity_ids(&TenantId::default(), "Blob") + .is_empty() + ); +} + +#[tokio::test] +async fn large_blob_json_stays_descriptor_and_property_value_streams_exact_bytes() { + let (state, _data_dir) = test_state(); + let body = vec![0x5au8; 256 * 1024]; + let object_id = git_blob_id(&body); + let router = app(state.clone()); + let created = router + .clone() + .oneshot(ingest_request( + Body::from(body.clone()), + body.len(), + &object_id, + )) + .await + .expect("large ingest response"); + assert_eq!(created.status(), StatusCode::CREATED); + + let entity = router + .clone() + .oneshot( + Request::get(format!("/tdata/Blobs('{object_id}')")) + .body(Body::empty()) + .expect("entity request"), + ) + .await + .expect("entity response"); + assert_eq!(entity.status(), StatusCode::OK); + let entity_bytes = axum::body::to_bytes(entity.into_body(), 2 * 1024 * 1024) + .await + .expect("entity body"); + let entity_json: serde_json::Value = + serde_json::from_slice(&entity_bytes).expect("entity JSON"); + assert!( + entity_json["fields"]["Content"] + .get(FIELD_OVERFLOW_REF_KEY) + .is_some(), + "large JSON reads must retain a bounded media descriptor" + ); + + let content = router + .clone() + .oneshot( + Request::get(format!("/tdata/Blobs('{object_id}')/Content/$value")) + .body(Body::empty()) + .expect("content request"), + ) + .await + .expect("content response"); + assert_eq!(content.status(), StatusCode::OK); + assert_eq!(content.headers()["content-length"], body.len().to_string()); + let content_bytes = axum::body::to_bytes(content.into_body(), body.len() + 1) + .await + .expect("streamed content"); + assert_eq!(content_bytes.as_ref(), body.as_slice()); + + let canonical = router + .oneshot( + Request::get(format!("/tdata/Blobs('{object_id}')/CanonicalBytes/$value")) + .body(Body::empty()) + .expect("canonical request"), + ) + .await + .expect("canonical response"); + assert_eq!(canonical.status(), StatusCode::OK); + let canonical_bytes = axum::body::to_bytes( + canonical.into_body(), + body.len() + "blob 262144\0".len() + 1, + ) + .await + .expect("streamed canonical bytes"); + let mut expected_canonical = format!("blob {}\0", body.len()).into_bytes(); + expected_canonical.extend_from_slice(&body); + assert_eq!(canonical_bytes.as_ref(), expected_canonical.as_slice()); + + state + .authz + .reload_tenant_policies( + "default", + r#"permit(principal, action == Action::"create", resource is Blob);"#, + ) + .expect("remove Blob read policy"); + let denied = app(state) + .oneshot( + Request::get(format!("/tdata/Blobs('{object_id}')/Content/$value")) + .body(Body::empty()) + .expect("denied content request"), + ) + .await + .expect("denied content response"); + assert_eq!(denied.status(), StatusCode::FORBIDDEN); +} diff --git a/crates/temper-server/src/odata/mod.rs b/crates/temper-server/src/odata/mod.rs index 7675e97f7..03f23070a 100644 --- a/crates/temper-server/src/odata/mod.rs +++ b/crates/temper-server/src/odata/mod.rs @@ -4,6 +4,7 @@ mod account_verification; mod app_uniqueness; pub(crate) mod authz; mod bindings; +mod blob_media; mod common; pub(crate) mod constraints; mod content_addressed; @@ -19,8 +20,6 @@ mod stream_fast_path; mod stream_put; mod write; -#[cfg(feature = "observe")] -pub(crate) use common::extract_tenant; pub use content_addressed::handle_blob_ingest_raw; pub use read::handle_hints; pub use read::handle_metadata; diff --git a/crates/temper-server/src/odata/query_plane_read/tests/proof.rs b/crates/temper-server/src/odata/query_plane_read/tests/proof.rs index 09d0d7194..3a7b18995 100644 --- a/crates/temper-server/src/odata/query_plane_read/tests/proof.rs +++ b/crates/temper-server/src/odata/query_plane_read/tests/proof.rs @@ -851,7 +851,6 @@ async fn session_entry_chain_parent_lookup_uses_bounded_native_page() { .await .expect("create local turso db"); let mut state = build_order_state("query-plane-session-parent"); - state.set_storage_stack(StorageStack::from_turso(store.clone())); let tenant = TenantId::default(); for index in 0usize..1200 { @@ -867,6 +866,10 @@ async fn session_entry_chain_parent_lookup_uses_bounded_native_page() { ) .await; } + // Keep fixture materialization single-owner. Installing the shared store + // starts query-plane maintenance workers, which must not race the bulk + // fixture transaction loop on libSQL's local connection. + state.set_storage_stack(StorageStack::from_turso(store.clone())); let security_ctx = SecurityContext::system(); let query_options = QueryOptions { @@ -928,7 +931,6 @@ async fn session_entry_leaf_id_lookup_uses_bounded_native_page() { .await .expect("create local turso db"); let mut state = build_order_state("query-plane-session-leaf"); - state.set_storage_stack(StorageStack::from_turso(store.clone())); let tenant = TenantId::default(); for index in 0usize..1200 { @@ -944,6 +946,7 @@ async fn session_entry_leaf_id_lookup_uses_bounded_native_page() { ) .await; } + state.set_storage_stack(StorageStack::from_turso(store.clone())); let security_ctx = SecurityContext::system(); let query_options = QueryOptions { diff --git a/crates/temper-server/src/odata/rate_limit.rs b/crates/temper-server/src/odata/rate_limit.rs index 2706e0182..55ed2efd4 100644 --- a/crates/temper-server/src/odata/rate_limit.rs +++ b/crates/temper-server/src/odata/rate_limit.rs @@ -1,12 +1,9 @@ -use axum::http::{HeaderMap, HeaderValue, StatusCode}; +use axum::http::{HeaderValue, StatusCode}; use axum::response::IntoResponse; use serde_json::Value; -use temper_authz::{PrincipalKind, SecurityContext}; +use temper_authz::SecurityContext; use temper_runtime::tenant::TenantId; -use crate::authz::security_context_from_headers; -use crate::identity::ResolvedIdentity; -use crate::request_context::AgentContext; use crate::response::odata_error; use crate::state::ServerState; @@ -29,29 +26,18 @@ pub(super) fn owner_id_from_action(fields: &Value, params: &Value) -> Option, - headers: &HeaderMap, - agent_ctx: &AgentContext, - resolved_identity: Option<&ResolvedIdentity>, + security_context: &SecurityContext, ) -> Result<(), axum::response::Response> { if !state.commons_guardrails_enabled(tenant) || rate_limit_exempt_entity(entity_type) { return Ok(()); } - let security_ctx = request_security_context(headers, agent_ctx, resolved_identity); - if matches!( - security_ctx.principal.kind, - PrincipalKind::Admin | PrincipalKind::System - ) { - return Ok(()); - } - - let owner_id = owner_id.unwrap_or_else(|| security_ctx.principal.id.clone()); + let owner_id = owner_id.unwrap_or_else(|| security_context.principal.id.clone()); if owner_id.trim().is_empty() || owner_id == "anonymous" { return Ok(()); } @@ -87,22 +73,6 @@ pub(super) async fn enforce_commons_write_rate_limit( } } -fn request_security_context( - headers: &HeaderMap, - agent_ctx: &AgentContext, - resolved_identity: Option<&ResolvedIdentity>, -) -> SecurityContext { - if let Some(identity) = resolved_identity { - SecurityContext::from_resolved_identity( - &identity.agent_instance_id, - &identity.agent_type_name, - agent_ctx.session_id.as_deref(), - ) - } else { - security_context_from_headers(headers, None, agent_ctx.session_id.as_deref(), None) - } -} - fn rate_limit_exempt_entity(entity_type: &str) -> bool { matches!(entity_type, "Owner" | "RateLimit") } diff --git a/crates/temper-server/src/odata/read.rs b/crates/temper-server/src/odata/read.rs index 3a7e36e10..817efe72a 100644 --- a/crates/temper-server/src/odata/read.rs +++ b/crates/temper-server/src/odata/read.rs @@ -5,7 +5,7 @@ use std::sync::{Arc, RwLock}; use axum::extract::{Extension, Query, State}; use axum::http::{HeaderMap, StatusCode}; use axum::response::{IntoResponse, Response}; -use temper_authz::SecurityContext; +use temper_authz::{AuthenticatedRequestContext, SecurityContext}; use temper_odata::path::{KeyValue, ODataPath, parse_path}; use temper_odata::query::parse_query_options; use temper_odata::query::types::{ @@ -15,7 +15,8 @@ use temper_runtime::tenant::TenantId; use temper_wasm::{StreamRegistry, WasmInvocationContext}; use tracing::instrument; -use super::authz::{READ_ACTION, authorize_read, request_security_context}; +use super::authz::{READ_ACTION, authorize_read, require_authenticated_context}; +use super::blob_media::handle_blob_primitive_stream; use super::common::{ check_has_stream_or_400, extract_key, extract_tenant, has_expand_options, resolve_entity_type, resolve_value_parent, tenant_csdl_xml, tenant_entity_sets, @@ -29,10 +30,8 @@ use super::read_support::{ }; use super::response::annotate_entity; use super::stream_fast_path::try_file_stream_fast_path; -use crate::blobs::hydrate_blob_refs_for_tenant; -use crate::identity::ResolvedIdentity; +use crate::blobs::{BlobHydrationBudget, hydrate_blob_refs_for_tenant_with_budget}; use crate::query_eval::{expand_entity, select_fields}; -use crate::request_context::extract_agent_context; use crate::response::{ODataResponse, ODataStreamResponse, ODataXmlResponse, odata_error}; use crate::state::ServerState; use crate::storage::{QueryFieldIndexOrder, QueryFieldIndexOrderDirection}; @@ -47,6 +46,7 @@ async fn resolve_parent_entity( state: &ServerState, tenant: &TenantId, security_ctx: &SecurityContext, + hydration_budget: &BlobHydrationBudget, ) -> Result<(String, String, String), (StatusCode, String)> { match path { ODataPath::Entity(set_name, key) => { @@ -60,8 +60,14 @@ async fn resolve_parent_entity( Ok((entity_type, key_str, set_name.clone())) } ODataPath::NavigationProperty { parent, property } => { - let (parent_type, parent_key, _parent_set) = - Box::pin(resolve_parent_entity(parent, state, tenant, security_ctx)).await?; + let (parent_type, parent_key, _parent_set) = Box::pin(resolve_parent_entity( + parent, + state, + tenant, + security_ctx, + hydration_budget, + )) + .await?; // Use expand to resolve the nav property let parent_set = resolve_entity_set_name(state, tenant, &parent_type); @@ -72,6 +78,7 @@ async fn resolve_parent_entity( &parent_set, &parent_key, security_ctx, + hydration_budget, ) .await .map_err(|_| { @@ -91,6 +98,7 @@ async fn resolve_parent_entity( state, tenant, security_ctx, + hydration_budget, ) .await .map_err(|_| { @@ -137,8 +145,14 @@ async fn resolve_parent_entity( key, } => { // Resolve the parent, then the keyed entity in the nav collection - let (parent_type, _parent_key, _parent_set) = - Box::pin(resolve_parent_entity(parent, state, tenant, security_ctx)).await?; + let (parent_type, _parent_key, _parent_set) = Box::pin(resolve_parent_entity( + parent, + state, + tenant, + security_ctx, + hydration_budget, + )) + .await?; let target_type = resolve_navigation_target_type(state, tenant, &parent_type, property)?; @@ -183,7 +197,7 @@ fn service_document_body(state: &ServerState, tenant: &TenantId) -> serde_json:: serde_json::json!({"@odata.context": "$metadata", "value": entity_sets}) } -async fn entity_set_not_found_response( +pub(super) async fn entity_set_not_found_response( state: &ServerState, tenant: &TenantId, set_name: &str, @@ -206,20 +220,7 @@ fn resource_not_found_response(set_name: &str, key: &str) -> Response { .into_response() } -/// Load a single entity body for OData read handlers. -/// -/// Tries the durable `entity_catalog` projection first when a query-plane -/// store is configured or the `TEMPER_ODATA_CATALOG_FAST_READ` flag is on. On -/// a hit, returns a body whose shape matches the actor's serialized -/// `EntityState` — blob refs already hydrated. On miss (no catalog row, -/// catalog disabled, or backend error), falls back to the actor path: validate -/// the entity exists in the in-memory index, then load via -/// `get_tenant_entity_state`. -/// -/// The catalog-first path bypasses the in-memory index, so it survives -/// cold starts and bulk-imported entities (data on disk but not yet -/// indexed in the running process). See ADR-0077. -async fn load_existing_entity_body( +pub(super) async fn load_existing_entity_descriptor_body( state: &ServerState, tenant: &TenantId, entity_type: &str, @@ -233,24 +234,16 @@ async fn load_existing_entity_body( { return Ok(body); } - - if !state.entity_exists(tenant, entity_type, key) { + if !state.entity_exists(tenant, entity_type, key) + && !state.ensure_entity_loaded(tenant, entity_type, key).await + { return Err(resource_not_found_response(set_name, key)); } - - let response = state + state .get_tenant_entity_state(tenant, entity_type, key) .await - .map_err(|_| resource_not_found_response(set_name, key))?; - let mut body = serde_json::to_value(&response.state).unwrap_or_default(); - hydrate_blob_refs_for_tenant(state, tenant, &mut body).await; - if let Some(obj) = body.as_object_mut() { - obj.insert( - "@odata.id".into(), - serde_json::json!(format!("{set_name}('{key}')")), - ); - } - Ok(body) + .map(|response| serde_json::to_value(&response.state).unwrap_or_default()) + .map_err(|_| resource_not_found_response(set_name, key)) } async fn load_authorized_entity_body( @@ -260,8 +253,13 @@ async fn load_authorized_entity_body( set_name: &str, key: &str, security_ctx: &SecurityContext, + hydration_budget: &BlobHydrationBudget, ) -> Result { - let body = load_existing_entity_body(state, tenant, entity_type, set_name, key).await?; + // Authorization precedes object-store reads. Overflow descriptors contain + // the inline ownership/relationship metadata Cedar needs without granting + // an unauthorized caller a storage-I/O amplification primitive. + let mut body = + load_existing_entity_descriptor_body(state, tenant, entity_type, set_name, key).await?; authorize_read( state, tenant, @@ -272,6 +270,13 @@ async fn load_authorized_entity_body( &body, ) .map_err(|response| *response)?; + hydrate_blob_refs_for_tenant_with_budget(state, tenant, &mut body, hydration_budget).await; + if let Some(object) = body.as_object_mut() { + object.insert( + "@odata.id".into(), + serde_json::json!(format!("{set_name}('{key}')")), + ); + } Ok(body) } @@ -391,12 +396,34 @@ async fn resolve_entity_request_key( } } +#[derive(Clone, Copy)] +struct ReadContext<'a> { + state: &'a ServerState, + tenant: &'a TenantId, + security: &'a SecurityContext, + hydration: &'a BlobHydrationBudget, +} + +impl<'a> ReadContext<'a> { + fn new( + state: &'a ServerState, + tenant: &'a TenantId, + security: &'a SecurityContext, + hydration: &'a BlobHydrationBudget, + ) -> Self { + Self { + state, + tenant, + security, + hydration, + } + } +} + async fn apply_entity_query_options( mut body: serde_json::Value, entity_type: &str, - state: &ServerState, - tenant: &TenantId, - security_ctx: &SecurityContext, + context: ReadContext<'_>, query_options: &QueryOptions, select_before_expand: bool, ) -> Result { @@ -409,9 +436,10 @@ async fn apply_entity_query_options( &mut body, expand_items, entity_type, - state, - tenant, - security_ctx, + context.state, + context.tenant, + context.security, + context.hydration, ) .await?; } @@ -433,24 +461,36 @@ struct EntityBodyOptions<'a> { } async fn build_entity_body( - state: &ServerState, - tenant: &TenantId, + context: ReadContext<'_>, entity_type: &str, set_name: &str, key: &str, - security_ctx: &SecurityContext, options: EntityBodyOptions<'_>, ) -> Result { - let mut state_json = - load_authorized_entity_body(state, tenant, entity_type, set_name, key, security_ctx) - .await?; + let mut state_json = load_authorized_entity_body( + context.state, + context.tenant, + entity_type, + set_name, + key, + context.security, + context.hydration, + ) + .await?; if let Some(obj) = state_json.as_object_mut() { obj.remove("@odata.id"); } let mut body = annotate_entity(state_json, options.context, options.odata_id); if options.enrich { - enrich_entity_response(&mut body, entity_type, set_name, key, state, tenant); + enrich_entity_response( + &mut body, + entity_type, + set_name, + key, + context.state, + context.tenant, + ); } if let Some(name) = options.function @@ -462,9 +502,7 @@ async fn build_entity_body( apply_entity_query_options( body, entity_type, - state, - tenant, - security_ctx, + context, options.query_options, options.select_before_expand, ) @@ -589,6 +627,7 @@ pub(super) async fn handle_odata_get_for_tenant( .into_response(); } }; + let hydration_budget = BlobHydrationBudget::generic_response(); match odata_path { ODataPath::Metadata => ODataXmlResponse { @@ -610,6 +649,7 @@ pub(super) async fn handle_odata_get_for_tenant( &name, &query_options, &query_params, + &hydration_budget, ) .await } @@ -622,6 +662,7 @@ pub(super) async fn handle_odata_get_for_tenant( &set_name, &key, &query_options, + &hydration_budget, ) .await } @@ -637,6 +678,7 @@ pub(super) async fn handle_odata_get_for_tenant( parent, property, &query_options, + &hydration_budget, ) .await } @@ -647,9 +689,7 @@ pub(super) async fn handle_odata_get_for_tenant( ref key, } => { handle_navigation_entity( - &state, - &tenant, - &security_ctx, + ReadContext::new(&state, &tenant, &security_ctx, &hydration_budget), parent, property, key, @@ -684,13 +724,14 @@ pub(super) async fn handle_odata_get_for_tenant( &parent, &function, &query_options, + &hydration_budget, ) .await } } ODataPath::Value { ref parent } => { - handle_stream_get(&state, &tenant, &security_ctx, parent).await + handle_stream_get(&state, &tenant, &security_ctx, parent, &hydration_budget).await } _ => odata_error( @@ -733,6 +774,7 @@ async fn handle_entity_set( name: &str, query_options: &QueryOptions, query_params: &std::collections::BTreeMap, + hydration_budget: &BlobHydrationBudget, ) -> axum::response::Response { tracing::debug!(name = %name, tenant = %tenant, "handle_entity_set"); let entity_type = match resolve_entity_type(state, tenant, name) { @@ -761,6 +803,9 @@ async fn handle_entity_set( read_result.telemetry.record(&span); let mut result = read_result.entities; + for entity in &mut result { + hydrate_blob_refs_for_tenant_with_budget(state, tenant, entity, hydration_budget).await; + } if let Some(ref expand_items) = query_options.expand { for entity in &mut result { if let Err(response) = expand_entity( @@ -770,6 +815,7 @@ async fn handle_entity_set( state, tenant, security_ctx, + hydration_budget, ) .await { @@ -845,6 +891,7 @@ async fn handle_entity( set_name: &str, key: &temper_odata::path::KeyValue, query_options: &QueryOptions, + hydration_budget: &BlobHydrationBudget, ) -> axum::response::Response { let entity_type = match resolve_entity_type(state, tenant, set_name) { Some(t) => t, @@ -904,12 +951,10 @@ async fn handle_entity( } match build_entity_body( - state, - tenant, + ReadContext::new(state, tenant, security_ctx, hydration_budget), &entity_type, set_name, &key_str, - security_ctx, EntityBodyOptions { context: format!("$metadata#{set_name}/$entity"), odata_id: Some(format!("{set_name}('{key_str}')")), @@ -938,9 +983,10 @@ async fn handle_navigation_property( parent: &ODataPath, property: &str, query_options: &QueryOptions, + hydration_budget: &BlobHydrationBudget, ) -> axum::response::Response { let (parent_type, parent_key, parent_set) = - match resolve_parent_entity(parent, state, tenant, security_ctx).await { + match resolve_parent_entity(parent, state, tenant, security_ctx, hydration_budget).await { Ok(r) => r, Err((status, msg)) => { return odata_error(status, "InvalidPath", &msg).into_response(); @@ -954,6 +1000,7 @@ async fn handle_navigation_property( &parent_set, &parent_key, security_ctx, + hydration_budget, ) .await { @@ -986,6 +1033,7 @@ async fn handle_navigation_property( state, tenant, security_ctx, + hydration_budget, ) .await { @@ -1036,35 +1084,42 @@ async fn handle_navigation_property( /// Handle `NavigationEntity` path: resolve parent, then fetch keyed child. async fn handle_navigation_entity( - state: &ServerState, - tenant: &TenantId, - security_ctx: &SecurityContext, + context: ReadContext<'_>, parent: &ODataPath, property: &str, key: &temper_odata::path::KeyValue, query_options: &QueryOptions, ) -> axum::response::Response { - let (parent_type, parent_key, parent_set) = - match resolve_parent_entity(parent, state, tenant, security_ctx).await { - Ok(r) => r, - Err((status, msg)) => { - return odata_error(status, "InvalidPath", &msg).into_response(); - } - }; + let (parent_type, parent_key, parent_set) = match resolve_parent_entity( + parent, + context.state, + context.tenant, + context.security, + context.hydration, + ) + .await + { + Ok(r) => r, + Err((status, msg)) => { + return odata_error(status, "InvalidPath", &msg).into_response(); + } + }; if let Err(response) = load_authorized_entity_body( - state, - tenant, + context.state, + context.tenant, &parent_type, &parent_set, &parent_key, - security_ctx, + context.security, + context.hydration, ) .await { return response; } - let Ok(target_type) = resolve_navigation_target_type(state, tenant, &parent_type, property) + let Ok(target_type) = + resolve_navigation_target_type(context.state, context.tenant, &parent_type, property) else { return odata_error( StatusCode::NOT_FOUND, @@ -1075,15 +1130,13 @@ async fn handle_navigation_entity( }; let key_str = extract_key(key); - let target_set = resolve_entity_set_name(state, tenant, &target_type); + let target_set = resolve_entity_set_name(context.state, context.tenant, &target_type); match build_entity_body( - state, - tenant, + context, &target_type, &target_set, &key_str, - security_ctx, EntityBodyOptions { context: format!("$metadata#{target_set}/$entity"), odata_id: Some(format!("{target_set}('{key_str}')")), @@ -1112,6 +1165,7 @@ async fn handle_bound_function( parent: &ODataPath, function: &str, query_options: &QueryOptions, + hydration_budget: &BlobHydrationBudget, ) -> axum::response::Response { let (parent_set, parent_key) = match parent { ODataPath::Entity(set_name, key) => (set_name.clone(), extract_key(key)), @@ -1138,12 +1192,10 @@ async fn handle_bound_function( }; match build_entity_body( - state, - tenant, + ReadContext::new(state, tenant, security_ctx, hydration_budget), &entity_type, &parent_set, &parent_key, - security_ctx, EntityBodyOptions { context: format!("$metadata#{entity_type}"), odata_id: None, @@ -1168,18 +1220,17 @@ async fn handle_bound_function( #[instrument(skip_all, fields(otel.name = "GET /odata/{path}"))] pub async fn handle_odata_get( State(state): State, - resolved_id: Option>, - headers: HeaderMap, + authenticated: Option>, + _headers: HeaderMap, axum::extract::Path(path): axum::extract::Path, Query(query_params): Query>, ) -> impl IntoResponse { - let tenant = match extract_tenant(&headers, &state) { - Ok(t) => t, - Err(e) => return e.into_response(), + let authenticated = match require_authenticated_context(authenticated) { + Ok(context) => context, + Err(error) => return error.into_response(), }; - let agent_ctx = extract_agent_context(&headers); - let resolved_identity = resolved_id.map(|Extension(identity)| identity); - let security_ctx = request_security_context(&headers, &agent_ctx, resolved_identity.as_ref()); + let tenant = authenticated.tenant().clone(); + let security_ctx = authenticated.security_context().clone(); handle_odata_get_for_tenant(state, tenant, security_ctx, path, query_params).await } @@ -1215,16 +1266,26 @@ pub async fn handle_metadata( } #[instrument(skip_all, fields(otel.name = "GET /odata/hints"))] -pub async fn handle_hints(State(state): State) -> impl IntoResponse { +pub async fn handle_hints( + State(state): State, + authenticated: Option>, +) -> impl IntoResponse { + let authenticated = match require_authenticated_context(authenticated) { + Ok(context) => context, + Err(error) => return error.into_response(), + }; let hints = state .agent_hints .read() .expect("agent hints lock should not be poisoned") - .clone(); + .get(authenticated.tenant()) + .cloned() + .unwrap_or_default(); ODataResponse { status: StatusCode::OK, body: serde_json::to_value(&hints).unwrap_or_default(), } + .into_response() } /// Handle GET on `$value` — return binary content for stream-backed entities. @@ -1242,7 +1303,20 @@ async fn handle_stream_get( tenant: &TenantId, security_ctx: &SecurityContext, parent: &ODataPath, + hydration_budget: &BlobHydrationBudget, ) -> axum::response::Response { + if let Some((set_name, key, property)) = resolve_blob_primitive_value_parent(parent) { + return handle_blob_primitive_stream( + state, + tenant, + security_ctx, + &set_name, + &key, + &property, + ) + .await; + } + // 1. Resolve parent to (set_name, entity_id) let (set_name, key) = match resolve_value_parent(parent) { Ok(pair) => pair, @@ -1266,6 +1340,7 @@ async fn handle_stream_get( &set_name, &key, security_ctx, + hydration_budget, ) .await { @@ -1326,7 +1401,13 @@ async fn handle_stream_get( }; let wasm_result = match state - .invoke_wasm_direct(tenant, "blob_adapter", inv_ctx, streams.clone()) + .invoke_wasm_direct( + tenant, + "blob_adapter", + inv_ctx, + streams.clone(), + security_ctx, + ) .await { Ok(r) => r, @@ -1382,6 +1463,16 @@ async fn handle_stream_get( .into_response() } +fn resolve_blob_primitive_value_parent(parent: &ODataPath) -> Option<(String, String, String)> { + let ODataPath::NavigationProperty { parent, property } = parent else { + return None; + }; + let ODataPath::Entity(set_name, key) = parent.as_ref() else { + return None; + }; + Some((set_name.clone(), extract_key(key), property.clone())) +} + #[cfg(test)] mod next_link_tests { use super::{encode_query_component, next_link}; diff --git a/crates/temper-server/src/odata/read_support.rs b/crates/temper-server/src/odata/read_support.rs index 59d6780ec..9f26ce8a6 100644 --- a/crates/temper-server/src/odata/read_support.rs +++ b/crates/temper-server/src/odata/read_support.rs @@ -5,7 +5,6 @@ use std::collections::{BTreeMap, BTreeSet}; use futures_util::stream::{self, StreamExt}; use temper_runtime::tenant::TenantId; -use crate::blobs::hydrate_blob_refs_for_tenant; use crate::state::ServerState; use crate::storage::{ CatalogRowsLoad, EntityCatalogRow, load_catalog_rows_by_id, load_selected_catalog_rows_by_id, @@ -234,9 +233,11 @@ pub(super) async fn try_load_entity_body_from_catalog( let rows = try_load_catalog_rows(state, tenant, entity_type, &ids).await; let row = rows.into_iter().next().map(|(_, r)| r)?; maybe_spawn_catalog_shadow_check(state, tenant, entity_type, &row); - let mut body = catalog_row_to_entity_body(entity_type, entity_set_name, row); - hydrate_blob_refs_for_tenant(state, tenant, &mut body).await; - Some(body) + Some(catalog_row_to_entity_body( + entity_type, + entity_set_name, + row, + )) } pub(super) async fn materialize_entity_set_entities( @@ -285,7 +286,7 @@ pub(super) async fn materialize_entity_set_entities( let selected_catalog_fields = selected_catalog_fields_owned.clone(); async move { if let Some(row) = catalog_row { - let mut entity = match selected_catalog_fields.as_deref() { + let entity = match selected_catalog_fields.as_deref() { Some(select) => catalog_row_to_selected_entity_body( &entity_type, &entity_set_name, @@ -294,7 +295,6 @@ pub(super) async fn materialize_entity_set_entities( ), None => catalog_row_to_entity_body(&entity_type, &entity_set_name, row), }; - hydrate_blob_refs_for_tenant(&state, &tenant, &mut entity).await; return Some(entity); } match state @@ -333,7 +333,6 @@ pub(super) async fn materialize_entity_set_entities( } } let mut entity = serde_json::to_value(&response.state).unwrap_or_default(); - hydrate_blob_refs_for_tenant(&state, &tenant, &mut entity).await; if let Some(obj) = entity.as_object_mut() { obj.insert( "@odata.id".into(), diff --git a/crates/temper-server/src/odata/storage_guardrails.rs b/crates/temper-server/src/odata/storage_guardrails.rs index b8abc26c5..c00c04cd3 100644 --- a/crates/temper-server/src/odata/storage_guardrails.rs +++ b/crates/temper-server/src/odata/storage_guardrails.rs @@ -20,7 +20,15 @@ pub(super) async fn enforce_commons_storage_cap( .await { Ok(()) => Ok(()), - Err(CommonsStorageCapError::Exceeded(exceeded)) => Err(odata_error( + Err(error) => Err(storage_cap_error_response(error)), + } +} + +pub(super) fn storage_cap_error_response( + error: CommonsStorageCapError, +) -> axum::response::Response { + match error { + CommonsStorageCapError::Exceeded(exceeded) => odata_error( StatusCode::PAYLOAD_TOO_LARGE, "StorageCapExceeded", &format!( @@ -31,24 +39,24 @@ pub(super) async fn enforce_commons_storage_cap( exceeded.cap_bytes ), ) - .into_response()), - Err(CommonsStorageCapError::OwnerSuspended(owner_id)) => Err(odata_error( + .into_response(), + CommonsStorageCapError::ReservationCapacityExhausted => odata_error( + StatusCode::TOO_MANY_REQUESTS, + "StorageReservationCapacityExhausted", + "Too many storage-reserved writes are already in flight", + ) + .into_response(), + CommonsStorageCapError::OwnerSuspended(owner_id) => odata_error( StatusCode::FORBIDDEN, "OwnerSuspended", &format!("Owner '{owner_id}' is suspended"), ) - .into_response()), - Err(CommonsStorageCapError::MissingAttribution(msg)) => { - Err( - odata_error(StatusCode::CONFLICT, "StorageAttributionMissing", &msg) - .into_response(), - ) + .into_response(), + CommonsStorageCapError::MissingAttribution(msg) => { + odata_error(StatusCode::CONFLICT, "StorageAttributionMissing", &msg).into_response() } - Err(CommonsStorageCapError::Internal(msg)) => { - Err( - odata_error(StatusCode::INTERNAL_SERVER_ERROR, "StorageCapError", &msg) - .into_response(), - ) + CommonsStorageCapError::Internal(msg) => { + odata_error(StatusCode::INTERNAL_SERVER_ERROR, "StorageCapError", &msg).into_response() } } } diff --git a/crates/temper-server/src/odata/stream_put.rs b/crates/temper-server/src/odata/stream_put.rs index b578139cf..b3c839a8e 100644 --- a/crates/temper-server/src/odata/stream_put.rs +++ b/crates/temper-server/src/odata/stream_put.rs @@ -18,7 +18,7 @@ use crate::blobs::hydrate_blob_refs_for_tenant; use crate::entity_actor::EntityResponse; use crate::request_context::AgentContext; use crate::response::odata_error; -use crate::state::{FileStreamContentError, ServerState}; +use crate::state::{DispatchCommand, FileStreamContentError, ServerState}; type ODataStreamPutError = Box; @@ -172,10 +172,21 @@ pub(super) async fn handle_stream_put( { return response; } + let expected_authorization_precondition = + crate::entity_actor::effects::entity_authorization_precondition( + &snapshot.current_state.state, + ); if entity_type == "File" { return match state - .put_file_stream_content_checked(tenant, &key, body.as_ref(), &content_type, agent_ctx) + .put_file_stream_content_checked( + tenant, + &key, + body.as_ref(), + &content_type, + agent_ctx, + Some(expected_authorization_precondition), + ) .await { Ok(entity_resp) => stream_put_success_response(&entity_resp), @@ -183,20 +194,8 @@ pub(super) async fn handle_stream_put( }; } - let entity_state = match state - .get_tenant_entity_state(tenant, &entity_type, &key) - .await - { - Ok(resp) => { - let mut entity_state = serde_json::to_value(&resp.state).unwrap_or_default(); - hydrate_blob_refs_for_tenant(state, tenant, &mut entity_state).await; - entity_state - } - Err(e) => { - return odata_error(StatusCode::INTERNAL_SERVER_ERROR, "StateError", &e) - .into_response(); - } - }; + let mut entity_state = serde_json::to_value(&snapshot.current_state.state).unwrap_or_default(); + hydrate_blob_refs_for_tenant(state, tenant, &mut entity_state).await; let size_bytes = body.len() as i64; let stream_id = format!("upload-{}", temper_runtime::scheduler::sim_uuid()); @@ -240,7 +239,7 @@ pub(super) async fn handle_stream_put( }; let wasm_result = match state - .invoke_wasm_direct(tenant, "blob_adapter", inv_ctx, streams) + .invoke_wasm_direct(tenant, "blob_adapter", inv_ctx, streams, security_ctx) .await { Ok(r) => r, @@ -269,18 +268,25 @@ pub(super) async fn handle_stream_put( if !wasm_result.callback_action.is_empty() { match state - .dispatch_tenant_action( - tenant, - &entity_type, - &key, - &wasm_result.callback_action, - wasm_result.callback_params, - agent_ctx, + .dispatch_tenant_action_ext_typed_if_current( + DispatchCommand { + tenant, + entity_type: &entity_type, + entity_id: &key, + action: &wasm_result.callback_action, + params: wasm_result.callback_params, + agent_ctx, + await_integration: false, + await_reactions: true, + }, + expected_authorization_precondition, ) .await { Ok(entity_resp) => stream_put_success_response(&entity_resp), - Err(e) => odata_error(StatusCode::CONFLICT, "ActionRejected", &e).into_response(), + Err(e) => { + odata_error(StatusCode::CONFLICT, "ActionRejected", &e.to_string()).into_response() + } } } else { StatusCode::NO_CONTENT.into_response() diff --git a/crates/temper-server/src/odata/write.rs b/crates/temper-server/src/odata/write.rs index d7e610c95..a1d80b375 100644 --- a/crates/temper-server/src/odata/write.rs +++ b/crates/temper-server/src/odata/write.rs @@ -4,6 +4,7 @@ use axum::extract::Query; use axum::extract::State; use axum::http::{HeaderMap, StatusCode}; use axum::response::IntoResponse; +use temper_authz::AuthenticatedRequestContext; use temper_odata::path::{ODataPath, parse_path}; use temper_runtime::scheduler::sim_now; use temper_runtime::tenant::TenantId; @@ -15,13 +16,13 @@ use axum::Extension; use super::account_verification::enforce_commons_account_verified_for_write; use super::app_uniqueness::enforce_commons_app_name_unique_for_write; use super::authz::{ - CREATE_ACTION, DELETE_ACTION, MutationResource, UPDATE_ACTION, authorize_mutation, - request_security_context, resource_attrs_from_body, + CREATE_ACTION, DELETE_ACTION, MutationResource, UPDATE_ACTION, apply_authenticated_context, + authorize_mutation, require_authenticated_context, resource_attrs_from_body, }; use super::bindings::dispatch_bound_action; use super::common::{ - constraint_violation_response, extract_key, extract_tenant, load_entity_or_404, - resolve_entity_type, run_write_prechecks, verification_gate_response, + constraint_violation_response, extract_key, resolve_entity_type, run_write_prechecks, + verification_gate_response, }; use super::constraints::pre_delete_relation_checks; use super::rate_limit::{enforce_commons_write_rate_limit, owner_id_from_fields}; @@ -29,7 +30,6 @@ use super::response::annotate_entity; use super::storage_guardrails::enforce_commons_storage_cap; use super::stream_put::handle_stream_put; use crate::blobs::hydrate_blob_refs_for_tenant; -use crate::identity::ResolvedIdentity; use crate::request_context::{AgentContext, extract_agent_context, remote_parent_context}; use crate::response::{ODataResponse, odata_error}; use crate::state::ServerState; @@ -58,6 +58,79 @@ fn parse_json_body_or_400(body: &axum::body::Bytes) -> Result ODataWriteError { + Box::new(odata_error(StatusCode::BAD_REQUEST, "InvalidBody", message).into_response()) +} + +fn prepare_collection_create_fields( + body: serde_json::Value, + entity_type: &str, + initial_status: &str, +) -> Result<(String, serde_json::Value), ODataWriteError> { + let mut fields = body + .as_object() + .cloned() + .ok_or_else(|| invalid_create_body("Entity create body must be a JSON object"))?; + + let lower_id = match fields.get("id") { + Some(value) => Some( + value + .as_str() + .filter(|value| !value.is_empty()) + .ok_or_else(|| invalid_create_body("id must be a non-empty string"))?, + ), + None => None, + }; + let upper_id = match fields.get("Id") { + Some(value) => Some( + value + .as_str() + .filter(|value| !value.is_empty()) + .ok_or_else(|| invalid_create_body("Id must be a non-empty string"))?, + ), + None => None, + }; + if let (Some(lower), Some(upper)) = (lower_id, upper_id) + && lower != upper + { + return Err(invalid_create_body( + "id and Id must identify the same entity", + )); + } + let entity_id = lower_id + .or(upper_id) + .map(str::to_string) + .unwrap_or_else(|| { + let prefix = entity_type_prefix(entity_type); + format!("{prefix}{}", temper_runtime::scheduler::sim_uuid()) + }); + + for key in ["status", "Status"] { + if let Some(value) = fields.get(key) + && value.as_str() != Some(initial_status) + { + return Err(invalid_create_body(&format!( + "{key} must equal the spec-defined initial state '{initial_status}'" + ))); + } + } + + fields.retain(|key, _| !temper_spec::automaton::is_server_derived_field_name(key)); + for key in ["id", "Id"] { + fields.insert( + key.to_string(), + serde_json::Value::String(entity_id.clone()), + ); + } + for key in ["status", "Status"] { + fields.insert( + key.to_string(), + serde_json::Value::String(initial_status.to_string()), + ); + } + Ok((entity_id, serde_json::Value::Object(fields))) +} + fn resolve_entity_type_or_404( state: &ServerState, tenant: &TenantId, @@ -109,6 +182,7 @@ fn resolve_entity_type_or_record_404( request_body, intent, matched_policy_ids: None, + capture_seq: None, }; if !state.enqueue_trajectory_entry(entry) { tracing::warn!( @@ -147,7 +221,14 @@ async fn authorize_collection_create( security_ctx: &temper_authz::SecurityContext, agent_ctx: &AgentContext, ) -> Result<(), ODataWriteError> { - let resource_attrs = resource_attrs_from_body(state, tenant, entity_type, entity_id, fields); + let resource_attrs = state + .build_create_authz_resource_attrs(tenant, entity_type, entity_id, fields) + .await + .map_err(|error| { + Box::new( + odata_error(StatusCode::INTERNAL_SERVER_ERROR, "ReadError", &error).into_response(), + ) + })?; authorize_mutation( state, tenant, @@ -193,7 +274,7 @@ async fn authorize_existing_mutation( action: &str, security_ctx: &temper_authz::SecurityContext, agent_ctx: &AgentContext, -) -> Result<(), ODataWriteError> { +) -> Result { let snapshot = state .load_authz_resource_snapshot(tenant, entity_type, entity_id) .await @@ -215,6 +296,67 @@ async fn authorize_existing_mutation( }, ) .await + .map_err(Box::new)?; + let precondition = crate::entity_actor::effects::entity_authorization_precondition( + &snapshot.current_state.state, + ); + Ok(ExistingMutationResource { + status: snapshot.current_state.state.status, + fields: snapshot.current_state.state.fields, + precondition, + }) +} + +struct ExistingMutationResource { + status: String, + fields: serde_json::Value, + precondition: String, +} + +struct ProspectiveMutationAuthorization<'a> { + tenant: &'a TenantId, + entity_type: &'a str, + entity_id: &'a str, + status: &'a str, + fields: &'a serde_json::Value, + security_ctx: &'a temper_authz::SecurityContext, + agent_ctx: &'a AgentContext, +} + +async fn authorize_prospective_mutation( + state: &ServerState, + request: ProspectiveMutationAuthorization<'_>, +) -> Result<(), ODataWriteError> { + let ProspectiveMutationAuthorization { + tenant, + entity_type, + entity_id, + status, + fields, + security_ctx, + agent_ctx, + } = request; + let attrs = state + .build_authz_resource_attrs(tenant, entity_type, entity_id, status, fields) + .await + .map_err(|error| { + Box::new( + odata_error(StatusCode::INTERNAL_SERVER_ERROR, "ReadError", &error).into_response(), + ) + })?; + authorize_mutation( + state, + tenant, + security_ctx, + agent_ctx, + UPDATE_ACTION, + MutationResource { + entity_type, + entity_id, + attrs: &attrs, + }, + ) + .await .map_err(Box::new) } @@ -222,26 +364,23 @@ async fn authorize_existing_mutation( #[instrument(skip_all, fields(otel.name = "POST /odata/{path}"))] pub async fn handle_odata_post( State(state): State, - resolved_id: Option>, + authenticated: Option>, headers: HeaderMap, axum::extract::Path(path): axum::extract::Path, Query(query_params): Query>, body: axum::body::Bytes, ) -> impl IntoResponse { - let tenant = match extract_tenant(&headers, &state) { - Ok(t) => t, - Err(e) => return e.into_response(), + let authenticated = match require_authenticated_context(authenticated) { + Ok(context) => context, + Err(error) => return error.into_response(), }; + let tenant = authenticated.tenant().clone(); + let security_ctx = authenticated.security_context().clone(); let mut agent_ctx = extract_agent_context(&headers); + apply_authenticated_context(&mut agent_ctx, &security_ctx); if let Some(remote_parent) = remote_parent_context(&agent_ctx) { tracing::Span::current().set_parent(remote_parent); } - let resolved_identity = resolved_id.map(|Extension(id)| id); - // Enrich agent context with credential-resolved identity (ADR-0033). - if let Some(ref identity) = resolved_identity { - agent_ctx.agent_id = Some(identity.agent_instance_id.clone()); - agent_ctx.agent_type = Some(identity.agent_type_name.clone()); - } let await_integration = query_params .get("await_integration") .map(|v| v == "true") @@ -279,17 +418,31 @@ pub async fn handle_odata_post( Err(resp) => return *resp, }; - let entity_id = body_json - .get("id") - .or_else(|| body_json.get("Id")) - .and_then(|v| v.as_str()) - .map(String::from) - .unwrap_or_else(|| { - let prefix = entity_type_prefix(&entity_type); - format!("{prefix}{}", temper_runtime::scheduler::sim_uuid()) - }); - - let initial_fields = body_json.clone(); + let initial_status = match state.initial_entity_status(&tenant, &entity_type) { + Ok(status) => status, + Err(error) => { + return odata_error(StatusCode::INTERNAL_SERVER_ERROR, "ReadError", &error) + .into_response(); + } + }; + let (entity_id, initial_fields) = + match prepare_collection_create_fields(body_json, &entity_type, &initial_status) { + Ok(prepared) => prepared, + Err(response) => return *response, + }; + if let Err(resp) = authorize_collection_create( + &state, + &tenant, + &entity_type, + &entity_id, + &initial_fields, + &security_ctx, + &agent_ctx, + ) + .await + { + return *resp; + } let _commons_guardrail_lock = state.acquire_commons_write_guardrail_lock(&tenant).await; if let Err(resp) = run_write_prechecks( @@ -347,29 +500,11 @@ pub async fn handle_odata_post( &tenant, &entity_type, owner_id_from_fields(&initial_fields), - &headers, - &agent_ctx, - resolved_identity.as_ref(), - ) - .await - { - return resp; - } - - let security_ctx = - request_security_context(&headers, &agent_ctx, resolved_identity.as_ref()); - if let Err(resp) = authorize_collection_create( - &state, - &tenant, - &entity_type, - &entity_id, - &initial_fields, &security_ctx, - &agent_ctx, ) .await { - return *resp; + return resp; } // ToolDefinition: forward tool metadata to the session's ToolRegistry. @@ -612,8 +747,6 @@ pub async fn handle_odata_post( }); let attrs = resource_attrs_from_body(&state, &tenant, &entity_type, &key_str, &authz_body); - let security_ctx = - request_security_context(&headers, &agent_ctx, resolved_identity.as_ref()); if let Err(response) = authorize_mutation( &state, &tenant, @@ -684,10 +817,9 @@ pub async fn handle_odata_post( &action, body_json, &agent_ctx, - &headers, await_integration, idempotency_key.clone(), - resolved_identity.as_ref(), + &security_ctx, ) .await } @@ -705,25 +837,23 @@ pub async fn handle_odata_post( #[instrument(skip_all, fields(otel.name = "PATCH /odata/{path}"))] pub async fn handle_odata_patch( State(state): State, - resolved_id: Option>, + authenticated: Option>, headers: HeaderMap, axum::extract::Path(path): axum::extract::Path, body: axum::body::Bytes, ) -> impl IntoResponse { - let tenant = match extract_tenant(&headers, &state) { - Ok(t) => t, - Err(e) => return e.into_response(), + let authenticated = match require_authenticated_context(authenticated) { + Ok(context) => context, + Err(error) => return error.into_response(), }; + let tenant = authenticated.tenant().clone(); + let security_ctx = authenticated.security_context().clone(); let odata_path = match parse_odata_path_or_400(&path) { Ok(p) => p, Err(resp) => return *resp, }; - let resolved_identity = resolved_id.map(|Extension(identity)| identity); let mut agent_ctx = extract_agent_context(&headers); - if let Some(ref identity) = resolved_identity { - agent_ctx.agent_id = Some(identity.agent_instance_id.clone()); - agent_ctx.agent_type = Some(identity.agent_type_name.clone()); - } + apply_authenticated_context(&mut agent_ctx, &security_ctx); match odata_path { ODataPath::Entity(set_name, key) => { @@ -741,9 +871,7 @@ pub async fn handle_odata_patch( { return *resp; } - let security_ctx = - request_security_context(&headers, &agent_ctx, resolved_identity.as_ref()); - if let Err(resp) = authorize_existing_mutation( + let existing = match authorize_existing_mutation( &state, &tenant, &entity_type, @@ -754,27 +882,23 @@ pub async fn handle_odata_patch( ) .await { - return *resp; - } + Ok(existing) => existing, + Err(resp) => return *resp, + }; let body_json = match parse_json_body_or_400(&body) { Ok(v) => v, Err(resp) => return *resp, }; - let current_state = match load_entity_or_404( - &state, - &tenant, - &entity_type, - &set_name, - &key_str, - ) - .await - { - Ok(v) => v, - Err(resp) => return resp, - }; - - let mut prospective_fields = current_state.state.fields.clone(); + if !body_json.is_object() { + return odata_error( + StatusCode::BAD_REQUEST, + "InvalidBody", + "PATCH body must be a JSON object", + ) + .into_response(); + } + let mut prospective_fields = existing.fields; if let (Some(dst), Some(src)) = (prospective_fields.as_object_mut(), body_json.as_object()) { @@ -785,6 +909,23 @@ pub async fn handle_odata_patch( prospective_fields = body_json.clone(); } + if let Err(response) = authorize_prospective_mutation( + &state, + ProspectiveMutationAuthorization { + tenant: &tenant, + entity_type: &entity_type, + entity_id: &key_str, + status: &existing.status, + fields: &prospective_fields, + security_ctx: &security_ctx, + agent_ctx: &agent_ctx, + }, + ) + .await + { + return *response; + } + let _commons_guardrail_lock = state.acquire_commons_write_guardrail_lock(&tenant).await; if let Err(resp) = run_write_prechecks( @@ -829,9 +970,7 @@ pub async fn handle_odata_patch( &tenant, &entity_type, owner_id_from_fields(&prospective_fields), - &headers, - &agent_ctx, - resolved_identity.as_ref(), + &security_ctx, ) .await { @@ -839,10 +978,17 @@ pub async fn handle_odata_patch( } match state - .update_tenant_entity_fields(&tenant, &entity_type, &key_str, body_json, false) + .update_tenant_entity_fields_if_current( + &tenant, + &entity_type, + &key_str, + body_json, + false, + existing.precondition, + ) .await { - Ok(response) => { + Ok(response) if response.success => { if entity_type == "RateLimit" { state.clear_commons_rate_limit_cache(); } @@ -860,6 +1006,14 @@ pub async fn handle_odata_patch( } .into_response() } + Ok(response) => odata_error( + StatusCode::CONFLICT, + "ConcurrentModification", + response.error.as_deref().unwrap_or( + "entity changed after authorization; retry against current state", + ), + ) + .into_response(), Err(e) => odata_error(StatusCode::INTERNAL_SERVER_ERROR, "UpdateError", &e) .into_response(), } @@ -877,25 +1031,23 @@ pub async fn handle_odata_patch( #[instrument(skip_all, fields(otel.name = "PUT /odata/{path}"))] pub async fn handle_odata_put( State(state): State, - resolved_id: Option>, + authenticated: Option>, headers: HeaderMap, axum::extract::Path(path): axum::extract::Path, body: axum::body::Bytes, ) -> impl IntoResponse { - let tenant = match extract_tenant(&headers, &state) { - Ok(t) => t, - Err(e) => return e.into_response(), + let authenticated = match require_authenticated_context(authenticated) { + Ok(context) => context, + Err(error) => return error.into_response(), }; + let tenant = authenticated.tenant().clone(); + let security_ctx = authenticated.security_context().clone(); let odata_path = match parse_odata_path_or_400(&path) { Ok(p) => p, Err(resp) => return *resp, }; - let resolved_identity = resolved_id.map(|Extension(identity)| identity); let mut agent_ctx = extract_agent_context(&headers); - if let Some(ref identity) = resolved_identity { - agent_ctx.agent_id = Some(identity.agent_instance_id.clone()); - agent_ctx.agent_type = Some(identity.agent_type_name.clone()); - } + apply_authenticated_context(&mut agent_ctx, &security_ctx); match odata_path { ODataPath::Entity(set_name, key) => { @@ -913,9 +1065,7 @@ pub async fn handle_odata_put( { return *resp; } - let security_ctx = - request_security_context(&headers, &agent_ctx, resolved_identity.as_ref()); - if let Err(resp) = authorize_existing_mutation( + let existing = match authorize_existing_mutation( &state, &tenant, &entity_type, @@ -926,13 +1076,39 @@ pub async fn handle_odata_put( ) .await { - return *resp; - } + Ok(existing) => existing, + Err(resp) => return *resp, + }; let body_json = match parse_json_body_or_400(&body) { Ok(v) => v, Err(resp) => return *resp, }; + if !body_json.is_object() { + return odata_error( + StatusCode::BAD_REQUEST, + "InvalidBody", + "PUT body must be a JSON object", + ) + .into_response(); + } + + if let Err(response) = authorize_prospective_mutation( + &state, + ProspectiveMutationAuthorization { + tenant: &tenant, + entity_type: &entity_type, + entity_id: &key_str, + status: &existing.status, + fields: &body_json, + security_ctx: &security_ctx, + agent_ctx: &agent_ctx, + }, + ) + .await + { + return *response; + } let _commons_guardrail_lock = state.acquire_commons_write_guardrail_lock(&tenant).await; @@ -978,9 +1154,7 @@ pub async fn handle_odata_put( &tenant, &entity_type, owner_id_from_fields(&body_json), - &headers, - &agent_ctx, - resolved_identity.as_ref(), + &security_ctx, ) .await { @@ -988,10 +1162,17 @@ pub async fn handle_odata_put( } match state - .update_tenant_entity_fields(&tenant, &entity_type, &key_str, body_json, true) + .update_tenant_entity_fields_if_current( + &tenant, + &entity_type, + &key_str, + body_json, + true, + existing.precondition, + ) .await { - Ok(response) => { + Ok(response) if response.success => { if entity_type == "RateLimit" { state.clear_commons_rate_limit_cache(); } @@ -1009,25 +1190,29 @@ pub async fn handle_odata_put( } .into_response() } + Ok(response) => odata_error( + StatusCode::CONFLICT, + "ConcurrentModification", + response.error.as_deref().unwrap_or( + "entity changed after authorization; retry against current state", + ), + ) + .into_response(), Err(e) => odata_error(StatusCode::INTERNAL_SERVER_ERROR, "UpdateError", &e) .into_response(), } } - ODataPath::Value { parent } => { - let security_ctx = - request_security_context(&headers, &agent_ctx, resolved_identity.as_ref()); - handle_stream_put( - &state, - &tenant, - &parent, - &headers, - body, - &agent_ctx, - &security_ctx, - ) - .await - .into_response() - } + ODataPath::Value { parent } => handle_stream_put( + &state, + &tenant, + &parent, + &headers, + body, + &agent_ctx, + &security_ctx, + ) + .await + .into_response(), _ => odata_error( StatusCode::METHOD_NOT_ALLOWED, "MethodNotAllowed", @@ -1041,24 +1226,22 @@ pub async fn handle_odata_put( #[instrument(skip_all, fields(otel.name = "DELETE /odata/{path}"))] pub async fn handle_odata_delete( State(state): State, - resolved_id: Option>, + authenticated: Option>, headers: HeaderMap, axum::extract::Path(path): axum::extract::Path, ) -> impl IntoResponse { - let tenant = match extract_tenant(&headers, &state) { - Ok(t) => t, - Err(e) => return e.into_response(), + let authenticated = match require_authenticated_context(authenticated) { + Ok(context) => context, + Err(error) => return error.into_response(), }; + let tenant = authenticated.tenant().clone(); + let security_ctx = authenticated.security_context().clone(); let odata_path = match parse_odata_path_or_400(&path) { Ok(p) => p, Err(resp) => return *resp, }; - let resolved_identity = resolved_id.map(|Extension(identity)| identity); let mut agent_ctx = extract_agent_context(&headers); - if let Some(ref identity) = resolved_identity { - agent_ctx.agent_id = Some(identity.agent_instance_id.clone()); - agent_ctx.agent_type = Some(identity.agent_type_name.clone()); - } + apply_authenticated_context(&mut agent_ctx, &security_ctx); match odata_path { ODataPath::Entity(set_name, key) => { @@ -1076,9 +1259,7 @@ pub async fn handle_odata_delete( { return *resp; } - let security_ctx = - request_security_context(&headers, &agent_ctx, resolved_identity.as_ref()); - if let Err(resp) = authorize_existing_mutation( + let existing = match authorize_existing_mutation( &state, &tenant, &entity_type, @@ -1089,25 +1270,14 @@ pub async fn handle_odata_delete( ) .await { - return *resp; - } + Ok(existing) => existing, + Err(resp) => return *resp, + }; if let Err(v) = pre_delete_relation_checks(&state, &tenant, &entity_type, &key_str, "delete").await { return constraint_violation_response(v); } - let current_state = match load_entity_or_404( - &state, - &tenant, - &entity_type, - &set_name, - &key_str, - ) - .await - { - Ok(v) => v, - Err(resp) => return resp, - }; if let Err(resp) = run_write_prechecks( &state, &tenant, @@ -1115,7 +1285,7 @@ pub async fn handle_odata_delete( &key_str, "Delete", "delete", - ¤t_state.state.fields, + &existing.fields, ) .await { @@ -1126,7 +1296,7 @@ pub async fn handle_odata_delete( &state, &tenant, &entity_type, - ¤t_state.state.fields, + &existing.fields, ) .await { @@ -1137,10 +1307,8 @@ pub async fn handle_odata_delete( &state, &tenant, &entity_type, - owner_id_from_fields(¤t_state.state.fields), - &headers, - &agent_ctx, - resolved_identity.as_ref(), + owner_id_from_fields(&existing.fields), + &security_ctx, ) .await { @@ -1148,16 +1316,29 @@ pub async fn handle_odata_delete( } match state - .delete_tenant_entity(&tenant, &entity_type, &key_str) + .delete_tenant_entity_if_current( + &tenant, + &entity_type, + &key_str, + existing.precondition, + ) .await { - Ok(_) => { + Ok(response) if response.success => { if entity_type == "RateLimit" { state.clear_commons_rate_limit_cache(); } state.clear_commons_storage_projection_cache_for_entity(&entity_type); (StatusCode::NO_CONTENT, "").into_response() } + Ok(response) => odata_error( + StatusCode::CONFLICT, + "ConcurrentModification", + response.error.as_deref().unwrap_or( + "entity changed after authorization; retry against current state", + ), + ) + .into_response(), Err(e) => odata_error(StatusCode::INTERNAL_SERVER_ERROR, "DeleteError", &e) .into_response(), } diff --git a/crates/temper-server/src/ots_trajectory_outbox.rs b/crates/temper-server/src/ots_trajectory_outbox.rs index 20f6169a5..9d0c8d1f2 100644 --- a/crates/temper-server/src/ots_trajectory_outbox.rs +++ b/crates/temper-server/src/ots_trajectory_outbox.rs @@ -330,7 +330,7 @@ async fn persist_with_retries( let started_at = Instant::now(); match queued .store - .mark_ots_trajectory_persisted(&queued.item.trajectory_id) + .mark_ots_trajectory_persisted(&queued.item.tenant, &queued.item.trajectory_id) .await { Ok(()) => { @@ -379,7 +379,11 @@ async fn persist_with_retries( failed_total.fetch_add(1, Ordering::Relaxed); if let Err(mark_error) = queued .store - .mark_ots_trajectory_failed(&queued.item.trajectory_id, &error.to_string()) + .mark_ots_trajectory_failed( + &queued.item.tenant, + &queued.item.trajectory_id, + &error.to_string(), + ) .await { tracing::error!( @@ -423,20 +427,22 @@ impl OtsStore for MetadataOtsStore { async fn mark_ots_trajectory_persisted( &self, + tenant: &str, trajectory_id: &str, ) -> Result<(), PersistenceError> { self.inner - .mark_ots_trajectory_persisted(trajectory_id) + .mark_ots_trajectory_persisted(tenant, trajectory_id) .await } async fn mark_ots_trajectory_failed( &self, + tenant: &str, trajectory_id: &str, error: &str, ) -> Result<(), PersistenceError> { self.inner - .mark_ots_trajectory_failed(trajectory_id, error) + .mark_ots_trajectory_failed(tenant, trajectory_id, error) .await } @@ -461,9 +467,10 @@ impl OtsStore for MetadataOtsStore { async fn get_ots_trajectory( &self, + tenant: &str, trajectory_id: &str, - ) -> Result, PersistenceError> { - self.inner.get_ots_trajectory(trajectory_id).await + ) -> Result, PersistenceError> { + self.inner.get_ots_trajectory(tenant, trajectory_id).await } } diff --git a/crates/temper-server/src/ots_trajectory_outbox/tests.rs b/crates/temper-server/src/ots_trajectory_outbox/tests.rs index d6442fb15..9edb4c847 100644 --- a/crates/temper-server/src/ots_trajectory_outbox/tests.rs +++ b/crates/temper-server/src/ots_trajectory_outbox/tests.rs @@ -11,7 +11,10 @@ struct FakeOtsStore { enqueue_attempts: AtomicU64, mark_persisted_attempts: AtomicU64, mark_failed_attempts: AtomicU64, - status: Mutex>, + /// Row status, keyed the way the real stores key the table: the + /// trajectory id comes from the uploading harness, so it is only unique + /// within a tenant. + status: Mutex>, persist_started: Notify, allow_persist: Notify, block_first_persist: AtomicBool, @@ -45,12 +48,16 @@ impl OtsStore for FakeOtsStore { self.status .lock() .expect("fake status mutex poisoned") - .insert(params.trajectory_id.to_string(), "queued".to_string()); + .insert( + (params.tenant.to_string(), params.trajectory_id.to_string()), + "queued".to_string(), + ); Ok(()) } async fn mark_ots_trajectory_persisted( &self, + tenant: &str, trajectory_id: &str, ) -> Result<(), PersistenceError> { self.attempts.fetch_add(1, Ordering::Relaxed); @@ -68,12 +75,16 @@ impl OtsStore for FakeOtsStore { self.status .lock() .expect("fake status mutex poisoned") - .insert(trajectory_id.to_string(), "persisted".to_string()); + .insert( + (tenant.to_string(), trajectory_id.to_string()), + "persisted".to_string(), + ); Ok(()) } async fn mark_ots_trajectory_failed( &self, + tenant: &str, trajectory_id: &str, _error: &str, ) -> Result<(), PersistenceError> { @@ -81,7 +92,10 @@ impl OtsStore for FakeOtsStore { self.status .lock() .expect("fake status mutex poisoned") - .insert(trajectory_id.to_string(), "failed".to_string()); + .insert( + (tenant.to_string(), trajectory_id.to_string()), + "failed".to_string(), + ); Ok(()) } @@ -104,25 +118,34 @@ impl OtsStore for FakeOtsStore { async fn get_ots_trajectory( &self, + _tenant: &str, _trajectory_id: &str, - ) -> Result, PersistenceError> { + ) -> Result, PersistenceError> { Ok(None) } } fn status(store: &FakeOtsStore, id: &str) -> Option { + tenant_status(store, "tenant", id) +} + +fn tenant_status(store: &FakeOtsStore, tenant: &str, id: &str) -> Option { store .status .lock() .expect("fake status mutex poisoned") - .get(id) + .get(&(tenant.to_string(), id.to_string())) .cloned() } fn item(id: &str) -> OtsTrajectoryWrite { + tenant_item("tenant", id) +} + +fn tenant_item(tenant: &str, id: &str) -> OtsTrajectoryWrite { OtsTrajectoryWrite { trajectory_id: id.to_string(), - tenant: "tenant".to_string(), + tenant: tenant.to_string(), agent_id: "agent".to_string(), session_id: "session".to_string(), outcome: "success".to_string(), @@ -234,6 +257,40 @@ async fn exhausted_retries_are_visible() { assert_eq!(status(&store, "traj-fail").as_deref(), Some("failed")); } +#[tokio::test] +async fn a_status_update_is_addressed_to_the_uploading_tenant() { + // Two tenants can upload under the same harness-chosen id. The drainer has + // to carry the tenant with the item, or one tenant's exhausted retries mark + // the other tenant's row failed. + let store = Arc::new(FakeOtsStore { + fail_attempts: AtomicU64::new(3), + ..FakeOtsStore::default() + }); + let outbox = OtsTrajectoryOutbox::start_for_tests(2, 1, 2, Duration::from_millis(1)); + + outbox + .try_enqueue_for_tests(store.clone(), tenant_item("beta", "traj-shared")) + .expect("beta's upload is admitted"); + tokio::time::timeout(Duration::from_millis(200), async { + while outbox.depth() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("beta's terminal failure should release queue depth"); + + assert_eq!( + tenant_status(&store, "beta", "traj-shared").as_deref(), + Some("failed"), + "the failing tenant's row is the one marked" + ); + assert_eq!( + tenant_status(&store, "alpha", "traj-shared"), + None, + "another tenant holding the same id must be untouched" + ); +} + #[tokio::test] async fn full_outbox_rejects_for_caller_retry() { let store = Arc::new(FakeOtsStore { diff --git a/crates/temper-server/src/platform_store.rs b/crates/temper-server/src/platform_store.rs index 6fb57ed66..a4e7e2e8e 100644 --- a/crates/temper-server/src/platform_store.rs +++ b/crates/temper-server/src/platform_store.rs @@ -203,6 +203,17 @@ pub trait PlatformStore: Send + Sync { /// Load all pending decisions (newest first, up to `limit`). async fn load_pending_decisions(&self, limit: usize) -> Result, String>; + /// Load approved decisions whose approved scope names `session_id`. + /// + /// Backs session-grant validation (ADR-0157): a caller-asserted session id + /// becomes a Cedar input only when an approved decision binds that session + /// to the asserting principal. + async fn load_approved_session_decisions( + &self, + tenant: &str, + session_id: &str, + ) -> Result, String>; + // ── WASM modules ───────────────────────────────────────────────── /// Load all WASM modules for a tenant. @@ -444,6 +455,16 @@ impl PlatformStore for TursoEventStore { .map_err(|e| e.to_string()) } + async fn load_approved_session_decisions( + &self, + tenant: &str, + session_id: &str, + ) -> Result, String> { + self.load_approved_session_decisions(tenant, session_id) + .await + .map_err(|e| e.to_string()) + } + async fn load_all_wasm_modules(&self, tenant: &str) -> Result, String> { let rows = self .load_all_wasm_modules(tenant) @@ -704,6 +725,16 @@ impl PlatformStore for PostgresEventStore { .map_err(|e| e.to_string()) } + async fn load_approved_session_decisions( + &self, + tenant: &str, + session_id: &str, + ) -> Result, String> { + self.load_approved_session_decisions(tenant, session_id) + .await + .map_err(|e| e.to_string()) + } + async fn load_all_wasm_modules(&self, tenant: &str) -> Result, String> { let rows = self .load_all_wasm_modules(tenant) @@ -1237,6 +1268,36 @@ mod sim_platform_store { .collect()) } + async fn load_approved_session_decisions( + &self, + tenant: &str, + session_id: &str, + ) -> Result, String> { + let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock + + let prob = inner.faults.decision_read_failure_prob; + if inner.rng.chance(prob) { + return Err("SimPlatformStore: injected decision read failure".into()); + } + + Ok(inner + .pending_decisions + .values() + .filter(|(row_tenant, status, _)| row_tenant == tenant && status == "approved") + .filter(|(_, _, data)| { + serde_json::from_str::(data) + .ok() + .and_then(|v| { + v.pointer("/approved_scope/session_id") + .and_then(|s| s.as_str()) + .map(|s| s == session_id) + }) + .unwrap_or(false) + }) + .map(|(_, _, data)| data.clone()) + .collect()) + } + async fn load_all_wasm_modules(&self, tenant: &str) -> Result, String> { let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock diff --git a/crates/temper-server/src/profiling.rs b/crates/temper-server/src/profiling.rs index 4caedcf75..17a177adf 100644 --- a/crates/temper-server/src/profiling.rs +++ b/crates/temper-server/src/profiling.rs @@ -155,8 +155,14 @@ async fn capture_profile_body( } }; + // `encode` is prost's serializer (pprof's `prost-codec` feature); it emits the + // same pprof protobuf wire format that protobuf-rs `write_to_vec` did — both + // codecs are generated from the same `profile.proto`, so the upload stays + // wire-compatible (ARN-169). Not byte-identical: pprof builds its string table + // from a HashSet, so output ordering already varies run to run under either + // codec. let mut body = Vec::new(); - if let Err(e) = profile.write_to_vec(&mut body) { + if let Err(e) = profile.encode(&mut body) { record_capture_error(profile_type, mode, "pprof_serialize"); return Err(format!("pprof serialize failed: {e}")); } @@ -174,7 +180,7 @@ async fn capture_profile_body( } fn non_empty_env(var_name: &str) -> Option { - std::env::var(var_name) + std::env::var(var_name) // determinism-ok: observability config, read outside simulation .ok() .map(|value| value.trim().to_string()) .filter(|value| !value.is_empty()) diff --git a/crates/temper-server/src/profiling/tests.rs b/crates/temper-server/src/profiling/tests.rs index 66f01dc4b..ff4656faa 100644 --- a/crates/temper-server/src/profiling/tests.rs +++ b/crates/temper-server/src/profiling/tests.rs @@ -149,3 +149,31 @@ fn datadog_profile_event_uses_agent_intake_envelope() { std::env::remove_var("DD_PROFILING_ENABLED"); } } + +#[test] +fn pprof_profile_encodes_to_decodable_protobuf() { + // ARN-169: profile serialization runs through pprof's `prost-codec` feature + // (rather than `protobuf-codec`, which pinned the vulnerable protobuf 2.x). + // Both codecs are generated from the same `profile.proto`, so the emitted + // bytes stay valid pprof protobuf. This guards the codec itself: if a future + // pprof bump silently reverted the feature, `encode`/`decode` would no longer + // resolve to prost's trait methods and this stops compiling — and if the + // encoding ever produced something undecodable, it fails. + use pprof::protos::{Profile, ValueType}; + + let profile = Profile { + string_table: vec![String::new(), "samples".to_string(), "count".to_string()], + sample_type: vec![ValueType { ty: 1, unit: 2 }], + period: 10_000_000, + ..Default::default() + }; + + let mut body = Vec::new(); + profile.encode(&mut body).expect("profile must encode"); + assert!(!body.is_empty(), "encoded pprof profile must not be empty"); + + let decoded = Profile::decode(&body[..]).expect("encoded bytes must be valid pprof protobuf"); + assert_eq!(decoded.period, 10_000_000); + assert_eq!(decoded.sample_type.len(), 1); + assert_eq!(decoded.string_table[1], "samples"); +} diff --git a/crates/temper-server/src/query_eval.rs b/crates/temper-server/src/query_eval.rs index cbd59fb96..3dca89f87 100644 --- a/crates/temper-server/src/query_eval.rs +++ b/crates/temper-server/src/query_eval.rs @@ -7,7 +7,7 @@ use temper_odata::query::types::{ BinaryOperator, FilterExpr, ODataValue, OrderByClause, OrderDirection, QueryOptions, }; -use crate::blobs::hydrate_blob_refs_for_tenant; +use crate::blobs::{BlobHydrationBudget, hydrate_blob_refs_for_tenant_with_budget}; /// Maximum nesting depth for recursive $expand (prevents infinite loops). const MAX_EXPAND_DEPTH: u8 = 3; @@ -396,6 +396,7 @@ struct ExpansionContext<'a> { state: &'a crate::state::ServerState, tenant: &'a temper_runtime::tenant::TenantId, security_ctx: &'a temper_authz::SecurityContext, + hydration_budget: &'a BlobHydrationBudget, } /// Resolve navigation properties for $expand on a single entity. @@ -412,11 +413,13 @@ pub async fn expand_entity( state: &crate::state::ServerState, tenant: &temper_runtime::tenant::TenantId, security_ctx: &temper_authz::SecurityContext, + hydration_budget: &BlobHydrationBudget, ) -> Result<(), axum::response::Response> { let context = ExpansionContext { state, tenant, security_ctx, + hydration_budget, }; expand_entity_recursive(entity, expand_items, entity_type, &context, 0, &mut vec![]).await } @@ -434,6 +437,7 @@ async fn expand_entity_recursive( state, tenant, security_ctx, + hydration_budget, } = context; if depth >= MAX_EXPAND_DEPTH { return Ok(()); @@ -514,8 +518,7 @@ async fn expand_entity_recursive( .get_tenant_entity_state(tenant, &info.target_type, fk) .await { - let mut json = serde_json::to_value(&response.state).unwrap_or_default(); - hydrate_blob_refs_for_tenant(state, tenant, &mut json).await; + let json = serde_json::to_value(&response.state).unwrap_or_default(); related_entities.push(json); } } @@ -528,9 +531,7 @@ async fn expand_entity_recursive( .get_tenant_entity_state(tenant, &info.target_type, related_id) .await { - let mut json = - serde_json::to_value(&response.state).unwrap_or_default(); - hydrate_blob_refs_for_tenant(state, tenant, &mut json).await; + let json = serde_json::to_value(&response.state).unwrap_or_default(); let matches = json .get("fields") .and_then(|f| f.get(target_fk_field.as_str())) @@ -550,9 +551,7 @@ async fn expand_entity_recursive( .get_tenant_entity_state(tenant, &info.target_type, related_id) .await { - let mut json = - serde_json::to_value(&response.state).unwrap_or_default(); - hydrate_blob_refs_for_tenant(state, tenant, &mut json).await; + let json = serde_json::to_value(&response.state).unwrap_or_default(); if matches_parent_reference(&json, entity_type, parent_id) { related_entities.push(json); } @@ -587,6 +586,13 @@ async fn expand_entity_recursive( }) }); + // Keep authorization ahead of object-store I/O. Relationship matching + // and Cedar evaluation use descriptor metadata; only rows the caller + // may read are allowed to consume the shared hydration budget. + for entity in &mut related_entities { + hydrate_blob_refs_for_tenant_with_budget(state, tenant, entity, hydration_budget).await; + } + // Apply nested query options if present if let Some(ref nested_opts) = item.options { let nested_query = QueryOptions { diff --git a/crates/temper-server/src/query_eval_test.rs b/crates/temper-server/src/query_eval_test.rs index 91e9a623c..c79d10468 100644 --- a/crates/temper-server/src/query_eval_test.rs +++ b/crates/temper-server/src/query_eval_test.rs @@ -276,3 +276,125 @@ fn test_find_fk_resolution_no_edge() { let result = find_fk_resolution(&graph, "Foo", "Bar", "Bars", true); assert!(result.is_none()); } + +#[tokio::test] +async fn expand_authorizes_related_rows_before_blob_hydration() { + use temper_authz::{Principal, PrincipalKind, SecurityContext}; + use temper_odata::query::types::ExpandItem; + use temper_runtime::ActorSystem; + use temper_runtime::tenant::TenantId; + use temper_spec::csdl::parse_csdl; + + let csdl_xml = include_str!("../../../test-fixtures/specs/model.csdl.xml"); + let order_ioa = include_str!("../../../test-fixtures/specs/order.ioa.toml"); + let mut specs = std::collections::BTreeMap::new(); + specs.insert("Customer".to_string(), order_ioa.to_string()); + specs.insert("Order".to_string(), order_ioa.to_string()); + let state = crate::state::ServerState::with_specs( + ActorSystem::new("expand-auth-before-hydration"), + parse_csdl(csdl_xml).expect("test CSDL"), + csdl_xml.to_string(), + specs, + ) + .expect("test state"); + let tenant = TenantId::default(); + state + .get_or_create_tenant_entity(&tenant, "Customer", "customer-1", serde_json::json!({})) + .await + .expect("create customer"); + state + .get_or_create_tenant_entity( + &tenant, + "Order", + "order-1", + serde_json::json!({ + "CustomerId": "customer-1", + "PrivatePayload": crate::blobs::blob_ref_value( + "field-overflow/sha256/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.json", + 128, + ), + }), + ) + .await + .expect("create order"); + state + .authz + .reload_tenant_policies( + tenant.as_str(), + r#" + permit(principal, action == Action::"list", resource is Order); + permit(principal, action == Action::"read", resource is Order); + "#, + ) + .expect("install list-and-read policy"); + let security = SecurityContext { + principal: Principal { + id: "reader-1".to_string(), + kind: PrincipalKind::Customer, + role: None, + acting_for: None, + agent_type: None, + attributes: Default::default(), + }, + context_attrs: Default::default(), + correlation_id: "expand-auth-before-hydration".to_string(), + }; + let budget = BlobHydrationBudget::new(1024, 1024, 0, 0); + let attempts_before = budget.read_attempts_remaining(); + let mut customer = serde_json::json!({ + "entity_id": "customer-1", + "fields": {}, + }); + + expand_entity( + &mut customer, + &[ExpandItem { + property: "Orders".to_string(), + options: None, + }], + "Customer", + &state, + &tenant, + &security, + &budget, + ) + .await + .expect("expand list authorization"); + + assert_eq!(customer["Orders"].as_array().map(Vec::len), Some(1)); + assert_eq!(budget.read_attempts_remaining(), attempts_before - 1); + + state + .authz + .reload_tenant_policies( + tenant.as_str(), + r#"permit(principal, action == Action::"list", resource is Order);"#, + ) + .expect("install list-only policy"); + let denied_budget = BlobHydrationBudget::new(1024, 1024, 0, 0); + let denied_attempts_before = denied_budget.read_attempts_remaining(); + let mut customer = serde_json::json!({ + "entity_id": "customer-1", + "fields": {}, + }); + expand_entity( + &mut customer, + &[ExpandItem { + property: "Orders".to_string(), + options: None, + }], + "Customer", + &state, + &tenant, + &security, + &denied_budget, + ) + .await + .expect("expand list authorization"); + + assert_eq!(customer["Orders"], serde_json::json!([])); + assert_eq!( + denied_budget.read_attempts_remaining(), + denied_attempts_before + ); +} diff --git a/crates/temper-server/src/request_context.rs b/crates/temper-server/src/request_context.rs index 88ed2d5cd..1a5517e9d 100644 --- a/crates/temper-server/src/request_context.rs +++ b/crates/temper-server/src/request_context.rs @@ -107,7 +107,7 @@ impl AgentContext { /// `docs/adrs/0046-unified-action-triggers.md`. pub fn for_service(service_name: &str) -> Self { let service_id = format!("service:{service_name}"); - let mut security_ctx = SecurityContext::from_headers(&[]).with_agent_context( + let mut security_ctx = SecurityContext::anonymous().with_agent_context( Some(&service_id), None, Some(service_name), @@ -207,6 +207,22 @@ fn header_string(headers: &HeaderMap, name: &str) -> Option { .map(String::from) } +/// Extract the caller-supplied session id from observability headers. +/// +/// Accepts `X-Temper-Observe-Session-Id` and the shorter `X-Session-Id` alias. +/// Single source of truth so every entrypoint honours both spellings. +pub fn session_id_from_headers(headers: &HeaderMap) -> Option { + header_string(headers, "x-temper-observe-session-id") + .or_else(|| header_string(headers, "x-session-id")) +} + +/// Extract the caller-supplied intent from observability headers. +/// +/// Accepts `X-Temper-Observe-Intent` and the shorter `X-Intent` alias. +pub fn intent_from_headers(headers: &HeaderMap) -> Option { + header_string(headers, "x-temper-observe-intent").or_else(|| header_string(headers, "x-intent")) +} + /// Extract observability context from request headers. /// /// Reads generic session, intent, and observation metadata headers for @@ -215,10 +231,8 @@ fn header_string(headers: &HeaderMap, name: &str) -> Option { /// self-declared headers — they come from credential resolution (ADR-0033) /// or are set to `None` for anonymous/operator access. pub(crate) fn extract_agent_context(headers: &HeaderMap) -> AgentContext { - let session_id = header_string(headers, "x-temper-observe-session-id") - .or_else(|| header_string(headers, "x-session-id")); - let intent = header_string(headers, "x-temper-observe-intent") - .or_else(|| header_string(headers, "x-intent")); + let session_id = session_id_from_headers(headers); + let intent = intent_from_headers(headers); // Extract W3C traceparent: "00-{trace_id}-{parent_span_id}-{flags}" let (trace_id, parent_span_id) = headers .get("traceparent") @@ -288,211 +302,5 @@ pub(crate) fn remote_parent_context(agent_ctx: &AgentContext) -> Option Router { AUTHORIZATION, HeaderName::from_static("x-tenant-id"), HeaderName::from_static("x-session-id"), + HeaderName::from_static("x-repository-id"), + HeaderName::from_static("x-expected-object-id"), HeaderName::from_static("idempotency-key"), - HeaderName::from_static("x-temper-principal-id"), - HeaderName::from_static("x-temper-principal-kind"), - HeaderName::from_static("x-temper-agent-role"), - HeaderName::from_static("x-temper-agent-type"), ]); router @@ -157,80 +155,135 @@ pub fn build_router(state: ServerState) -> Router { ), ) .layer(cors) + // Axum applies the last layer first: strip caller authority before the + // typed-context guard or any route handler observes the request. + .layer(axum::middleware::from_fn( + crate::authz::require_authenticated_request_context, + )) + .layer(axum::middleware::from_fn( + crate::authz::strip_inbound_identity_headers, + )) .with_state(state) } /// Fallback handler for paths not served by any built-in route. -/// Resolves the tenant from `X-Tenant-Id`, consults the tenant's +/// Resolves the tenant from the request's credential, consults that tenant's /// `HttpEndpointTable`, and (in slice 2) returns 501 on match, 404 /// otherwise. Slice 3 of K-1 Phase 2 replaces the 501 with a real /// streaming dispatch into the bound WASM integration. #[tracing::instrument(skip_all, fields(http.method = %method, http.route = %uri.path()))] async fn http_endpoint_fallback( State(state): State, + authenticated: Option>, + admitted: Option>, method: axum::http::Method, uri: Uri, headers: HeaderMap, _body: Body, ) -> Response { - let tenant_header = headers - .get("x-tenant-id") - .and_then(|v| v.to_str().ok()) - .map(str::to_string); - - // Git clients (and GitHub-REST-compat clients) can't send - // X-Tenant-Id; the whole point of the HttpEndpoint surface is - // to terminate foreign wire protocols. When the header is absent - // the fallback prefers the registered `default` tenant — protocol - // endpoint rows live there on single-operator deployments — and - // only then any other non-system tenant. Picking "first - // registered" made resolution depend on registry iteration order: - // a production server whose extra tenants sort before "default" - // resolved header-less git/REST requests to a tenant with an - // empty route table and answered 404. Multi-tenant deployments - // that need strict tenant routing should encode the tenant in the - // path prefix of their HttpEndpoint rows - // (e.g. /{tenant}/{repo}.git/...). - let tenant_id = match tenant_header { - Some(t) if !t.is_empty() => TenantId::new(&t), - _ => { - let Ok(registry) = state.registry.read() else { - return http_404_response(uri.path()); - }; - match http_endpoint_fallback_tenant(®istry.tenant_ids()) { - Some(t) => t.clone(), - None => return http_404_response(uri.path()), - } + let Some(Extension(authenticated)) = authenticated else { + return StatusCode::UNAUTHORIZED.into_response(); + }; + let tenant_id = authenticated.tenant().clone(); + let route = if let Some(Extension(admitted)) = admitted { + let Some(matched) = admitted.into_matched(&tenant_id, method.as_str(), uri.path()) else { + tracing::warn!(tenant = %tenant_id, "HttpEndpoint admission binding mismatch"); + return StatusCode::UNAUTHORIZED.into_response(); + }; + matched + } else { + if authenticated.security_context().principal.id == "anonymous" { + return StatusCode::UNAUTHORIZED.into_response(); } + let Some(table) = state.http_endpoint_tables.get(&tenant_id).await else { + return http_404_response(uri.path()); + }; + let Some(matched) = table.match_request(method.as_str(), uri.path()).await else { + return http_404_response(uri.path()); + }; + matched }; + if route.route.requires_auth && authenticated.security_context().principal.id == "anonymous" { + return StatusCode::UNAUTHORIZED.into_response(); + } - let table = state.http_endpoint_tables.table_for(&tenant_id).await; - let matched = table.match_request(method.as_str(), uri.path()).await; - - let Some(route) = matched else { - return http_404_response(uri.path()); - }; + dispatch_matched_route(state, authenticated, method, uri, headers, _body, route).await +} - dispatch_matched_route(state, tenant_id, method, uri, headers, _body, route).await +/// Request headers that carry an ambient caller credential and must never be +/// forwarded into a WASM guest's invocation context. +/// +/// A guest runs untrusted module code and can read everything it is handed, so any +/// credential here is one it could replay — against this server, an upstream, or an +/// identity-aware proxy fronting us. The list therefore covers direct credentials, +/// the forwarded-auth family injected by such proxies, and cloud instance tokens +/// (ARN-208). +/// +/// This is the *inbound* guard (host → guest). The separate *outbound* sanitizer +/// for guest-supplied headers on internal re-entry lives in +/// `temper_wasm::host_trait::internal_http::internal_header_allowed`, and the two +/// lists are deliberately **not** the same set: that one also drops routing and +/// tenant-identity headers (`host`, `forwarded`, `x-forwarded-for`, `x-tenant-id`, +/// the `x-temper-*` authority namespace) which are legitimate context on the way +/// in. Adding a credential header here does not imply an edit there. +pub(crate) const GUEST_FORBIDDEN_CREDENTIAL_HEADERS: [&str; 16] = [ + "authorization", + "proxy-authorization", + "cookie", + "x-api-key", + // Forwarded-auth family: whatever an identity-aware proxy injects once it has + // authenticated the caller. None of these are used by the current deployment; + // they are stripped defensively so putting Temper behind such a proxy never + // silently starts handing live IdP tokens to guest modules. + "x-forwarded-authorization", + "x-forwarded-access-token", + // Google IAP + "x-goog-iap-jwt-assertion", + // Cloudflare Access + "cf-access-jwt-assertion", + // AWS ALB `authenticate-oidc` + "x-amzn-oidc-accesstoken", + "x-amzn-oidc-data", + "x-amzn-oidc-identity", + // Azure App Service EasyAuth + "x-ms-token-aad-access-token", + "x-ms-token-aad-id-token", + "x-ms-token-aad-refresh-token", + // oauth2-proxy `--set-xauthrequest` + "x-auth-request-access-token", + // Cloud instance credential. + "x-amz-security-token", +]; + +/// Whether a request header carries an ambient caller credential and must not be +/// forwarded into a WASM guest's invocation context. +pub(crate) fn is_credential_header(name: &str) -> bool { + GUEST_FORBIDDEN_CREDENTIAL_HEADERS + .iter() + .any(|candidate| name.eq_ignore_ascii_case(candidate)) } -/// Tenant for a header-less HttpEndpoint request: the registered -/// `default` tenant when present (deterministic — protocol endpoint -/// rows live there on single-operator deployments), else the first -/// non-system tenant, else any tenant at all. -fn http_endpoint_fallback_tenant<'a>(tenant_ids: &[&'a TenantId]) -> Option<&'a TenantId> { - let default_tenant = TenantId::default(); - tenant_ids +/// Build the header list a WASM guest sees for an inbound HTTP dispatch. +/// +/// Today this is the only place inbound headers cross into guest-visible state, so +/// the credential filter lives here rather than inline at the call site: one audited +/// home, and testable against its real output. `pub(crate)` so any future +/// guest-context constructor reuses it instead of reimplementing the filter. +/// +/// Note this covers *headers* only. The request path handed to the guest includes +/// the query string (deliberately — git needs `service=`), so a credential passed as +/// `?access_token=…` is a different carrier of the same invariant; tracked with the +/// outbound mirror in ARN-346 (ARN-208). +pub(crate) fn guest_visible_headers(headers: &HeaderMap) -> Vec<(String, String)> { + headers .iter() - .copied() - .find(|t| **t == default_tenant) - .or_else(|| { - tenant_ids - .iter() - .copied() - .find(|t| t.as_str() != "temper-system") + .filter(|(name, _)| !is_credential_header(name.as_str())) + .filter_map(|(k, v)| { + v.to_str() + .ok() + .map(|s| (k.as_str().to_string(), s.to_string())) }) - .or_else(|| tenant_ids.first().copied()) + .collect() } /// End-to-end dispatch: open an InboundExchange on the shared @@ -241,7 +294,7 @@ fn http_endpoint_fallback_tenant<'a>(tenant_ids: &[&'a TenantId]) -> Option<&'a /// response. async fn dispatch_matched_route( state: ServerState, - tenant_id: TenantId, + authenticated: AuthenticatedRequestContext, method: axum::http::Method, uri: Uri, headers: HeaderMap, @@ -250,6 +303,7 @@ async fn dispatch_matched_route( ) -> Response { use temper_wasm::http_stream::HttpResponseHead; use temper_wasm::types::{HttpDispatchContext, WasmInvocationContext}; + let tenant_id = authenticated.tenant().clone(); // Resolve the integration module hash. The WASM module must // already be registered for this tenant (via app install). @@ -289,7 +343,7 @@ async fn dispatch_matched_route( // ADR-0057 inbound exchange end-to-end streaming — without it, // even SDK-streaming guests are bounded by the buffered limit. let pump_streams = streams.clone(); - tokio::spawn(async move { + let pump_task = async move { use tokio_stream::StreamExt as _; let mut stream = body.into_data_stream(); while let Some(chunk_result) = stream.next().await { @@ -312,17 +366,13 @@ async fn dispatch_matched_route( } } let _ = pump_streams.close(kernel_request_body).await; - }); + }; + tokio::spawn(pump_task); // determinism-ok: detached HTTP body pump, not simulation state - // Build the invocation context. - let header_pairs: Vec<(String, String)> = headers - .iter() - .filter_map(|(k, v)| { - v.to_str() - .ok() - .map(|s| (k.as_str().to_string(), s.to_string())) - }) - .collect(); + // Build the invocation context. Guest-visible headers go through + // `guest_visible_headers` so caller credentials are stripped in one audited + // place (ARN-208). + let header_pairs: Vec<(String, String)> = guest_visible_headers(&headers); let route_params = git_route_params_for_http_dispatch( &route.route.integration_module, uri.path(), @@ -336,7 +386,7 @@ async fn dispatch_matched_route( wasm_module: Some(route.route.integration_module.clone()), trigger_params: serde_json::Value::Null, entity_state: serde_json::Value::Null, - agent_id: None, + agent_id: Some(authenticated.security_context().principal.id.clone()), session_id: None, integration_config: std::collections::BTreeMap::new(), trace_id: String::new(), @@ -353,22 +403,28 @@ async fn dispatch_matched_route( }, params: route_params.clone(), headers: header_pairs, - principal_id: None, + principal_id: Some(authenticated.security_context().principal.id.clone()), request_body_handle: guest_request_body.0, response_body_handle: guest_response_body.0, }), }; - // Build a per-request host that shares the registry. - let secrets: std::collections::BTreeMap = state - .secrets_vault - .as_ref() - .map(|v| v.get_tenant_secrets(tenant_id.as_str())) - .unwrap_or_default(); - let host: std::sync::Arc = std::sync::Arc::new( - temper_wasm::ProductionWasmHost::with_shared_streams(secrets, streams.clone()) - .with_invocation_context(ctx.clone()), - ); + // Build the canonical Cedar-gated host chain while retaining the inbound + // exchange's shared stream registry. + let host = match crate::state::authorized_http_endpoint_host( + &state, + &tenant_id, + &route.route.integration_module, + &ctx, + streams.clone(), + authenticated.security_context(), + ) { + Ok(host) => host, + Err(error) => { + tracing::error!(%error, "failed to construct authorized HttpEndpoint host"); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + }; // Spawn task B: invoke the WASM module. Runs to completion // (guest writes head + body via FFI; we drain on the axum side). @@ -430,17 +486,16 @@ async fn dispatch_matched_route( return dispatch_action_bridge_result( state, - tenant_id, + authenticated, headers, route_params, - route.route.requires_auth, action_bridge, result, ) .await; } - let invoke_task = tokio::spawn(async move { + let invoke_future = async move { match engine .invoke_with_blobs( &invoke_hash, @@ -469,7 +524,8 @@ async fn dispatch_matched_route( ); } } - }); + }; + let invoke_task = tokio::spawn(invoke_future); // determinism-ok: detached WASM invocation task, not simulation state // Await the guest's response head — bounded by the route's // configured timeout so a bad guest doesn't wedge the request. @@ -529,13 +585,13 @@ async fn dispatch_matched_route( async fn dispatch_action_bridge_result( state: ServerState, - tenant_id: TenantId, + authenticated: AuthenticatedRequestContext, headers: HeaderMap, route_params: std::collections::BTreeMap, - requires_auth: bool, bridge: crate::http_endpoint::HttpActionBridge, result: temper_wasm::types::WasmInvocationResult, ) -> Response { + let tenant_id = authenticated.tenant().clone(); let callback_params = result.callback_params.clone(); let refs = bridge_git_receive_pack_refs(&callback_params); let sideband = bridge_git_receive_pack_sideband(&callback_params); @@ -565,27 +621,13 @@ async fn dispatch_action_bridge_result( let action_params = bridge_action_params(&callback_params); let mut agent_ctx = crate::request_context::extract_agent_context(&headers); - let explicit_principal = headers.contains_key("x-temper-principal-kind") - || headers.contains_key("x-temper-principal-id") - || headers.contains_key("x-temper-principal-scopes"); - // Protocol adapters (git wire, REST shims) authenticate callers whose - // credentials are not X-Temper-* headers; their resolved principal - // takes precedence over header context and the system fallback - // (ADR-0138). - agent_ctx.security_ctx = Some( - if let Some(adapter_principal) = bridge_resolved_principal(&callback_params) { - adapter_principal - } else if requires_auth || explicit_principal { - crate::authz::security_context_from_headers( - &headers, - agent_ctx.agent_id.as_deref(), - agent_ctx.session_id.as_deref(), - agent_ctx.agent_type.as_deref(), - ) - } else { - temper_authz::SecurityContext::system() - }, - ); + agent_ctx.security_ctx = Some(authenticated.security_context().clone()); + agent_ctx.agent_id = Some(authenticated.security_context().principal.id.clone()); + agent_ctx.agent_type = authenticated + .security_context() + .principal + .agent_type + .clone(); if agent_ctx.idempotency_key.is_none() { agent_ctx.idempotency_key = action_params .get("ClientRequestId") @@ -771,57 +813,6 @@ fn build_bridge_short_circuit(resp: &serde_json::Value) -> Result Option { - let principal = callback_params.get("bridge_principal")?; - if callback_params.get("action_params").is_none() - && callback_params.get("ActionParams").is_none() - { - tracing::warn!( - "bridge_principal without explicit action_params is ignored (ADR-0138 structured shape required)" - ); - return None; - } - let kind = principal.get("kind").and_then(|v| v.as_str())?.trim(); - let id = principal.get("id").and_then(|v| v.as_str())?.trim(); - if kind.is_empty() || id.is_empty() { - return None; - } - let mut header_pairs = vec![ - ("x-temper-principal-kind".to_string(), kind.to_string()), - ("x-temper-principal-id".to_string(), id.to_string()), - ]; - let scopes = principal - .get("scopes") - .and_then(|v| v.as_array()) - .map(|items| { - items - .iter() - .filter_map(|s| s.as_str()) - .map(str::trim) - .filter(|s| !s.is_empty()) - .collect::>() - .join(",") - }) - .unwrap_or_default(); - if !scopes.is_empty() { - header_pairs.push(("x-temper-principal-scopes".to_string(), scopes)); - } - Some(temper_authz::SecurityContext::from_headers(&header_pairs)) -} - fn bridge_git_receive_pack_refs(callback_params: &serde_json::Value) -> Vec { callback_params .get("git_receive_pack") diff --git a/crates/temper-server/src/router_test.rs b/crates/temper-server/src/router_test.rs index c9b7728a6..1a4d4142e 100644 --- a/crates/temper-server/src/router_test.rs +++ b/crates/temper-server/src/router_test.rs @@ -1,6 +1,10 @@ use super::*; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + use axum::body::Body; use axum::http::{Request, StatusCode}; +use sha1::Digest as _; use temper_runtime::ActorSystem; use temper_runtime::tenant::TenantId; use temper_spec::csdl::parse_csdl; @@ -10,6 +14,96 @@ use tower::ServiceExt; use crate::events::EntityStateChange; use crate::storage::StorageStack; +fn test_security_context() -> temper_authz::SecurityContext { + temper_authz::SecurityContext { + principal: temper_authz::Principal { + id: "test-customer".to_string(), + kind: temper_authz::PrincipalKind::Customer, + role: None, + acting_for: None, + agent_type: None, + attributes: Default::default(), + }, + context_attrs: Default::default(), + correlation_id: "router-test".to_string(), + } +} + +fn claimed_admin_security_context() -> temper_authz::SecurityContext { + temper_authz::SecurityContext { + principal: temper_authz::Principal { + id: "claimed-admin".to_string(), + kind: temper_authz::PrincipalKind::Admin, + role: None, + acting_for: None, + agent_type: None, + attributes: Default::default(), + }, + context_attrs: Default::default(), + correlation_id: "router-admin-side-channel-test".to_string(), + } +} + +async fn authenticate_test_request( + mut request: axum::extract::Request, + next: axum::middleware::Next, +) -> axum::response::Response { + if request + .extensions() + .get::() + .is_none() + { + let tenant = request + .headers() + .get("x-tenant-id") + .and_then(|value| value.to_str().ok()) + .filter(|value| !value.trim().is_empty()) + .map(str::trim) + .map(TenantId::try_new) + .transpose() + .expect("test tenant header should be valid") + .unwrap_or_default(); + request + .extensions_mut() + .insert(temper_authz::AuthenticatedRequestContext::new( + tenant, + test_security_context(), + )); + } + next.run(request).await +} + +fn authenticated_router(state: ServerState) -> Router { + if state + .authz + .get_tenant_policy_text(TenantId::default().as_str()) + .is_none() + { + state + .authz + .reload_tenant_policies( + TenantId::default().as_str(), + "permit(principal, action, resource);", + ) + .expect("functional router tests should install an explicit policy"); + } + super::build_router(state).layer(axum::middleware::from_fn(authenticate_test_request)) +} + +fn git_blob_id(body: &[u8]) -> String { + let mut hasher = sha1::Sha1::new(); + hasher.update(format!("blob {}\0", body.len()).as_bytes()); + hasher.update(body); + format!("{:x}", hasher.finalize()) +} + +fn counted_body(bytes: &'static [u8], polls: Arc) -> Body { + Body::from_stream(futures_util::stream::once(async move { + polls.fetch_add(1, Ordering::SeqCst); + Ok::<_, std::io::Error>(bytes::Bytes::from_static(bytes)) + })) +} + fn test_state() -> ServerState { let csdl_xml = include_str!("../../../test-fixtures/specs/model.csdl.xml"); let csdl = parse_csdl(csdl_xml).unwrap(); @@ -88,7 +182,16 @@ params = ["RepositoryId", "Size", "Content", "CanonicalBytes", "CreatedAt"] let system = ActorSystem::new("test-blob-ingest"); let mut specs = std::collections::BTreeMap::new(); specs.insert("Blob".to_string(), blob_ioa.to_string()); - ServerState::with_specs(system, csdl, csdl_xml.to_string(), specs).unwrap() + let mut state = ServerState::with_specs(system, csdl, csdl_xml.to_string(), specs).unwrap(); + state.data_dir = std::env::temp_dir().join("temper-router-blob-tests"); + state + .authz + .reload_tenant_policies( + "default", + r#"permit(principal, action == Action::"create", resource is Blob);"#, + ) + .expect("install Blob test policy"); + state } fn test_state_with_rate_limit_ioa() -> ServerState { @@ -251,7 +354,13 @@ params = ["RepositoryId", "Size", "Content", "CanonicalBytes", "CreatedAt"] specs.insert("Owner".to_string(), owner_ioa.to_string()); specs.insert("Repository".to_string(), repository_ioa.to_string()); specs.insert("Blob".to_string(), blob_ioa.to_string()); - ServerState::with_specs(system, csdl, csdl_xml.to_string(), specs).unwrap() + let mut state = ServerState::with_specs(system, csdl, csdl_xml.to_string(), specs).unwrap(); + state.data_dir = std::env::temp_dir().join("temper-router-storage-cap-tests"); + state + .authz + .reload_tenant_policies("default", "permit(principal, action, resource);") + .expect("install commons storage test policy"); + state } fn test_state_with_account_verification_ioa() -> ServerState { @@ -476,7 +585,7 @@ initial = "" #[tokio::test] async fn test_service_document() { - let app = build_router(test_state()); + let app = authenticated_router(test_state()); let response = app .oneshot(Request::get("/tdata").body(Body::empty()).unwrap()) .await @@ -491,9 +600,82 @@ async fn test_service_document() { assert_eq!(json["@odata.context"], "$metadata"); } +#[tokio::test] +async fn protected_kernel_route_rejects_forged_headers_without_typed_context() { + let response = super::build_router(test_state()) + .oneshot( + Request::get("/tdata/Orders") + .header("x-tenant-id", "victim") + .header("x-temper-principal-kind", "admin") + .header("x-temper-principal-id", "attacker") + .header("x-temper-agent-role", "supervisor") + .header("x-temper-principal-scopes", "root") + .header("x-temper-attr-owner", "*") + .header("x-temper-action-context", "forged") + .body(Body::empty()) + .expect("request should build"), + ) + .await + .expect("request should run"); + + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); +} + +#[tokio::test] +async fn typed_admin_kind_does_not_bypass_governed_mutation_cedar() { + let mut request = Request::post("/tdata/Orders") + .header("content-type", "application/json") + .body(Body::from(r#"{"Id":"claimed-admin-order"}"#)) + .expect("request should build"); + request + .extensions_mut() + .insert(temper_authz::AuthenticatedRequestContext::new( + TenantId::default(), + claimed_admin_security_context(), + )); + let response = super::build_router(test_state_with_ioa()) + .oneshot(request) + .await + .expect("request should run"); + assert_eq!(response.status(), StatusCode::FORBIDDEN); +} + +#[tokio::test] +async fn hints_require_typed_authentication_and_are_tenant_scoped() { + let state = test_state(); + state.enrich_metadata(&TenantId::default(), "Submit", "default hint"); + state.enrich_metadata(&TenantId::new("tenant-b"), "Approve", "tenant-b hint"); + + let unauthenticated = super::build_router(state.clone()) + .oneshot( + Request::get("/tdata/$hints") + .body(Body::empty()) + .expect("unauthenticated hints request"), + ) + .await + .expect("unauthenticated hints response"); + assert_eq!(unauthenticated.status(), StatusCode::UNAUTHORIZED); + + let response = authenticated_router(state) + .oneshot( + Request::get("/tdata/$hints") + .body(Body::empty()) + .expect("authenticated hints request"), + ) + .await + .expect("authenticated hints response"); + assert_eq!(response.status(), StatusCode::OK); + let body = axum::body::to_bytes(response.into_body(), 64 * 1024) + .await + .expect("hints body"); + let body: serde_json::Value = serde_json::from_slice(&body).expect("hints JSON"); + assert_eq!(body["Submit"], "default hint"); + assert!(body.get("Approve").is_none()); +} + #[tokio::test] async fn test_metadata_endpoint() { - let app = build_router(test_state()); + let app = authenticated_router(test_state()); let response = app .oneshot( Request::get("/tdata/$metadata") @@ -516,7 +698,7 @@ async fn test_metadata_endpoint() { #[tokio::test] async fn test_entity_set_listing() { - let app = build_router(test_state()); + let app = authenticated_router(test_state()); let response = app .oneshot(Request::get("/tdata/Orders").body(Body::empty()).unwrap()) .await @@ -532,7 +714,7 @@ async fn test_entity_set_listing() { #[tokio::test] async fn test_entity_by_key_not_found() { - let app = build_router(test_state()); + let app = authenticated_router(test_state()); let response = app .oneshot( Request::get("/tdata/Orders('abc-123')") @@ -548,7 +730,7 @@ async fn test_entity_by_key_not_found() { #[tokio::test] async fn test_entity_by_key_found() { - let app = build_router(test_state_with_ioa()); + let app = authenticated_router(test_state_with_ioa()); // First create an entity via POST let create_response = app @@ -583,7 +765,7 @@ async fn test_entity_by_key_found() { #[tokio::test] async fn test_unknown_entity_set_returns_404() { - let app = build_router(test_state()); + let app = authenticated_router(test_state()); let response = app .oneshot( Request::get("/tdata/NonExistent") @@ -598,7 +780,7 @@ async fn test_unknown_entity_set_returns_404() { #[tokio::test] async fn test_post_entity_creation() { - let app = build_router(test_state_with_ioa()); + let app = authenticated_router(test_state_with_ioa()); let response = app .oneshot( Request::post("/tdata/Orders") @@ -614,7 +796,7 @@ async fn test_post_entity_creation() { #[tokio::test] async fn test_post_entity_creation_uses_odata_id_property() { - let app = build_router(test_state_with_ioa()); + let app = authenticated_router(test_state_with_ioa()); let create_response = app .clone() .oneshot( @@ -638,10 +820,96 @@ async fn test_post_entity_creation_uses_odata_id_property() { assert_eq!(get_response.status(), StatusCode::OK); } +#[tokio::test] +async fn collection_create_rejects_conflicting_identity_or_lifecycle_aliases() { + let app = authenticated_router(test_state_with_ioa()); + + let conflicting_id = app + .clone() + .oneshot( + Request::post("/tdata/Orders") + .header("Content-Type", "application/json") + .body(Body::from( + r#"{"id":"order-lower","Id":"order-upper","status":"Draft"}"#, + )) + .expect("conflicting ID request"), + ) + .await + .expect("conflicting ID response"); + assert_eq!(conflicting_id.status(), StatusCode::BAD_REQUEST); + + let forged_status = app + .oneshot( + Request::post("/tdata/Orders") + .header("Content-Type", "application/json") + .body(Body::from(r#"{"id":"order-status","Status":"Shipped"}"#)) + .expect("forged status request"), + ) + .await + .expect("forged status response"); + assert_eq!(forged_status.status(), StatusCode::BAD_REQUEST); +} + +#[tokio::test] +async fn collection_create_derives_and_persists_authoritative_compatibility_aliases() { + let state = test_state_with_ioa(); + state + .authz + .reload_tenant_policies( + "default", + r#" +permit(principal, action == Action::"create", resource is Order) +when { + resource.id == "order-trusted" && + resource.Id == "order-trusted" && + resource.status == "Draft" && + resource.Status == "Draft" +}; +forbid(principal, action == Action::"create", resource is Order) +when { resource has ctx_owner_status }; +"#, + ) + .expect("trusted create policy"); + let app = authenticated_router(state); + + let response = app + .oneshot( + Request::post("/tdata/Orders") + .header("Content-Type", "application/json") + .body(Body::from( + r#"{ + "id":"order-trusted", + "Id":"order-trusted", + "status":"Draft", + "Status":"Draft", + "has_spec":true, + "HasSpec":true, + "ctx_owner_status":"Privileged", + "customer":"Alice" + }"#, + )) + .expect("trusted create request"), + ) + .await + .expect("trusted create response"); + assert_eq!(response.status(), StatusCode::CREATED); + let body = axum::body::to_bytes(response.into_body(), 1024 * 1024) + .await + .expect("create response body"); + let body: serde_json::Value = serde_json::from_slice(&body).expect("create response JSON"); + assert_eq!(body["fields"]["id"], "order-trusted"); + assert_eq!(body["fields"]["Id"], "order-trusted"); + assert_eq!(body["fields"]["status"], "Draft"); + assert_eq!(body["fields"]["Status"], "Draft"); + assert!(body["fields"].get("ctx_owner_status").is_none()); + assert!(body["fields"].get("has_spec").is_none()); + assert!(body["fields"].get("HasSpec").is_none()); +} + #[tokio::test] async fn test_data_only_entity_create_fast_path_persists_projection_without_actor_spawn() { let state = test_state_with_data_only_ioa_and_turso().await; - let app = build_router(state.clone()); + let app = authenticated_router(state.clone()); let create_response = app .clone() @@ -710,9 +978,13 @@ async fn test_data_only_create_fast_path_declines_action_bearing_entities() { #[tokio::test] async fn commons_rate_limit_returns_429_per_owner_bucket() { let state = test_state_with_rate_limit_ioa(); + state + .authz + .reload_tenant_policies("beta", "permit(principal, action, resource);") + .expect("beta functional test policy should parse"); state.enable_commons_guardrails("default"); state.enable_commons_guardrails("beta"); - let app = build_router(state.clone()); + let app = authenticated_router(state.clone()); let alice_bucket = ServerState::commons_rate_limit_entity_id("alice", "write"); let response = app @@ -771,6 +1043,25 @@ async fn commons_rate_limit_returns_429_per_owner_bucket() { .unwrap(); assert_eq!(exhausted.status(), StatusCode::TOO_MANY_REQUESTS); + let mut claimed_admin_request = Request::post("/tdata/Widgets") + .header("Content-Type", "application/json") + .body(Body::from( + r#"{"Id":"wd-alice-admin","OwnerId":"alice","Name":"claimed admin"}"#, + )) + .expect("claimed-admin request should build"); + claimed_admin_request + .extensions_mut() + .insert(temper_authz::AuthenticatedRequestContext::new( + TenantId::default(), + claimed_admin_security_context(), + )); + let claimed_admin = app + .clone() + .oneshot(claimed_admin_request) + .await + .expect("claimed-admin request should run"); + assert_eq!(claimed_admin.status(), StatusCode::TOO_MANY_REQUESTS); + let bob_bucket = ServerState::commons_rate_limit_entity_id("bob", "write"); let response = app .clone() @@ -907,12 +1198,13 @@ async fn commons_rate_limit_returns_429_per_owner_bucket() { #[tokio::test] async fn test_blob_ingest_raw_route_streams_body_without_path_param() { - let app = build_router(test_state_with_blob_ioa()); + let app = authenticated_router(test_state_with_blob_ioa()); let response = app .oneshot( Request::post("/tdata/Blobs/Temper.IngestRaw") .header("Content-Type", "application/octet-stream") .header("Content-Length", "3") + .header("X-Expected-Object-Id", git_blob_id(b"abc")) .header("X-Repository-Id", "rp-acme-demo") .body(Body::from("abc")) .unwrap(), @@ -945,11 +1237,12 @@ async fn test_blob_ingest_raw_applies_cedar_create_policy() { ) .expect("install Cedar policy"); - let response = build_router(state.clone()) + let response = authenticated_router(state.clone()) .oneshot( Request::post("/tdata/Blobs/Temper.IngestRaw") .header("Content-Type", "application/octet-stream") .header("Content-Length", "3") + .header("X-Expected-Object-Id", git_blob_id(b"abc")) .header("X-Repository-Id", "rp-acme-demo") .header("X-Temper-Principal-Id", "customer-1") .header("X-Temper-Principal-Kind", "customer") @@ -967,7 +1260,7 @@ async fn test_blob_ingest_raw_applies_cedar_create_policy() { async fn commons_storage_cap_blocks_raw_blob_ingest_per_owner() { let state = test_state_with_storage_cap_ioa(); state.enable_commons_guardrails("default"); - let app = build_router(state.clone()); + let app = authenticated_router(state.clone()); let alice_owner = app .clone() @@ -1007,6 +1300,7 @@ async fn commons_storage_cap_blocks_raw_blob_ingest_per_owner() { Request::post("/tdata/Blobs/Temper.IngestRaw") .header("Content-Type", "application/octet-stream") .header("Content-Length", "3") + .header("X-Expected-Object-Id", git_blob_id(b"abc")) .header("X-Repository-Id", "rp-alice") .header("X-Temper-Principal-Id", "alice") .header("X-Temper-Principal-Kind", "customer") @@ -1017,21 +1311,28 @@ async fn commons_storage_cap_blocks_raw_blob_ingest_per_owner() { .unwrap(); assert_eq!(alice_first.status(), StatusCode::CREATED); + let exceeded_body_polls = Arc::new(AtomicUsize::new(0)); let alice_exceeded = app .clone() .oneshot( Request::post("/tdata/Blobs/Temper.IngestRaw") .header("Content-Type", "application/octet-stream") .header("Content-Length", "2") + .header("X-Expected-Object-Id", git_blob_id(b"de")) .header("X-Repository-Id", "rp-alice") .header("X-Temper-Principal-Id", "alice") .header("X-Temper-Principal-Kind", "customer") - .body(Body::from("de")) + .body(counted_body(b"de", exceeded_body_polls.clone())) .unwrap(), ) .await .unwrap(); assert_eq!(alice_exceeded.status(), StatusCode::PAYLOAD_TOO_LARGE); + assert_eq!( + exceeded_body_polls.load(Ordering::SeqCst), + 0, + "over-quota raw bodies must not be polled" + ); let bob_owner = app .clone() @@ -1071,6 +1372,7 @@ async fn commons_storage_cap_blocks_raw_blob_ingest_per_owner() { Request::post("/tdata/Blobs/Temper.IngestRaw") .header("Content-Type", "application/octet-stream") .header("Content-Length", "2") + .header("X-Expected-Object-Id", git_blob_id(b"xy")) .header("X-Repository-Id", "rp-bob") .header("X-Temper-Principal-Id", "bob") .header("X-Temper-Principal-Kind", "customer") @@ -1104,7 +1406,7 @@ async fn commons_storage_cap_blocks_raw_blob_ingest_per_owner() { async fn commons_storage_projection_cache_invalidates_after_blob_write() { let state = test_state_with_storage_cap_ioa(); state.enable_commons_guardrails("default"); - let app = build_router(state.clone()); + let app = authenticated_router(state.clone()); let owner = app .clone() @@ -1145,6 +1447,7 @@ async fn commons_storage_projection_cache_invalidates_after_blob_write() { Request::post("/tdata/Blobs/Temper.IngestRaw") .header("Content-Type", "application/octet-stream") .header("Content-Length", "2") + .header("X-Expected-Object-Id", git_blob_id(b"aa")) .header("X-Repository-Id", "rp-carol") .header("X-Temper-Principal-Id", "carol") .header("X-Temper-Principal-Kind", "customer") @@ -1168,6 +1471,7 @@ async fn commons_storage_projection_cache_invalidates_after_blob_write() { Request::post("/tdata/Blobs/Temper.IngestRaw") .header("Content-Type", "application/octet-stream") .header("Content-Length", "2") + .header("X-Expected-Object-Id", git_blob_id(b"bb")) .header("X-Repository-Id", "rp-carol") .header("X-Temper-Principal-Id", "carol") .header("X-Temper-Principal-Kind", "customer") @@ -1188,10 +1492,23 @@ async fn commons_storage_projection_cache_invalidates_after_blob_write() { } #[tokio::test] -async fn commons_storage_cap_serializes_concurrent_blob_writes_per_owner() { - let state = test_state_with_storage_cap_ioa(); +async fn commons_storage_reservation_allows_other_writers_and_prevents_overreservation() { + let mut state = test_state_with_storage_cap_ioa(); + state.raw_blob_ingest_budget = crate::blob_store::BlobIngestBudget::with_limits( + 16, + 1, + 4, + 2, + crate::blob_store::BlobIngestProgressPolicy::new( + std::time::Duration::from_secs(5), + std::time::Duration::from_secs(10), + std::time::Duration::from_secs(5), + std::time::Duration::from_secs(1), + 1, + ), + ); state.enable_commons_guardrails("default"); - let app = build_router(state.clone()); + let app = authenticated_router(state.clone()); let owner = app .clone() @@ -1226,46 +1543,80 @@ async fn commons_storage_cap_serializes_concurrent_blob_writes_per_owner() { assert_eq!(repo.status(), StatusCode::CREATED); let first_app = app.clone(); - let second_app = app.clone(); - let (first, second) = tokio::join!( - async move { - first_app - .oneshot( - Request::post("/tdata/Blobs/Temper.IngestRaw") - .header("Content-Type", "application/octet-stream") - .header("Content-Length", "4") - .header("X-Repository-Id", "rp-dana") - .header("X-Temper-Principal-Id", "dana") - .header("X-Temper-Principal-Kind", "customer") - .body(Body::from("abcd")) - .unwrap(), - ) - .await - .unwrap() - }, - async move { - second_app - .oneshot( - Request::post("/tdata/Blobs/Temper.IngestRaw") - .header("Content-Type", "application/octet-stream") - .header("Content-Length", "4") - .header("X-Repository-Id", "rp-dana") - .header("X-Temper-Principal-Id", "dana") - .header("X-Temper-Principal-Kind", "customer") - .body(Body::from("wxyz")) - .unwrap(), - ) - .await - .unwrap() - } + let first_body_polls = Arc::new(AtomicUsize::new(0)); + let second_body_polls = Arc::new(AtomicUsize::new(0)); + let first_body_polls_for_request = first_body_polls.clone(); + let slow_body = Body::from_stream(async_stream::stream! { + first_body_polls_for_request.fetch_add(1, Ordering::SeqCst); + yield Ok::<_, std::io::Error>(bytes::Bytes::from_static(b"abcd")); + std::future::pending::<()>().await; + }); + let first = tokio::spawn( + first_app.oneshot( + Request::post("/tdata/Blobs/Temper.IngestRaw") + .header("Content-Type", "application/octet-stream") + .header("Content-Length", "4") + .header("X-Expected-Object-Id", git_blob_id(b"abcd")) + .header("X-Repository-Id", "rp-dana") + .body(slow_body) + .unwrap(), + ), ); + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while first_body_polls.load(Ordering::SeqCst) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("slow upload should reach body staging"); - let mut statuses = vec![first.status(), second.status()]; - statuses.sort(); - assert_eq!( - statuses, - vec![StatusCode::CREATED, StatusCode::PAYLOAD_TOO_LARGE] - ); + let unrelated_owner = tokio::time::timeout( + std::time::Duration::from_secs(2), + app.clone().oneshot( + Request::post("/tdata/Owners") + .header("Content-Type", "application/json") + .body(Body::from( + r#"{"Id":"erin","AccountId":"erin","DisplayName":"Erin","Contact":"erin@example.test","StorageCapBytes":4,"RateLimitTier":"free","PublicKey":""}"#, + )) + .unwrap(), + ), + ) + .await + .expect("slow Blob body must not hold the coarse commons lock") + .unwrap(); + assert_eq!(unrelated_owner.status(), StatusCode::CREATED); + + let second = app + .clone() + .oneshot( + Request::post("/tdata/Blobs/Temper.IngestRaw") + .header("Content-Type", "application/octet-stream") + .header("Content-Length", "4") + .header("X-Expected-Object-Id", git_blob_id(b"wxyz")) + .header("X-Repository-Id", "rp-dana") + .body(counted_body(b"wxyz", second_body_polls.clone())) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(second.status(), StatusCode::PAYLOAD_TOO_LARGE); + assert_eq!(second_body_polls.load(Ordering::SeqCst), 0); + + first.abort(); + let _ = first.await; + let committed = app + .oneshot( + Request::post("/tdata/Blobs/Temper.IngestRaw") + .header("Content-Type", "application/octet-stream") + .header("Content-Length", "4") + .header("X-Expected-Object-Id", git_blob_id(b"abcd")) + .header("X-Repository-Id", "rp-dana") + .body(Body::from("abcd")) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(committed.status(), StatusCode::CREATED); let tenant = temper_runtime::tenant::TenantId::default(); let projection = state @@ -1282,7 +1633,7 @@ async fn commons_storage_cap_serializes_concurrent_blob_writes_per_owner() { async fn commons_account_verification_blocks_owner_scoped_writes_until_verified() { let state = test_state_with_account_verification_ioa(); state.enable_commons_guardrails("default"); - let app = build_router(state.clone()); + let app = authenticated_router(state.clone()); let owner = app .clone() @@ -1382,7 +1733,7 @@ async fn commons_account_verification_blocks_owner_scoped_writes_until_verified( async fn commons_app_name_unique_per_owner_on_create_and_patch() { let state = test_state_with_owner_app_ioa(); state.enable_commons_guardrails("default"); - let app = build_router(state); + let app = authenticated_router(state); let owner = app .clone() @@ -1502,7 +1853,7 @@ async fn commons_app_name_unique_per_owner_on_create_and_patch() { #[tokio::test] async fn test_post_bound_action() { - let app = build_router(test_state_with_ioa()); + let app = authenticated_router(test_state_with_ioa()); let response = app .oneshot( Request::post("/tdata/Orders('abc-123')/Temper.Example.CancelOrder") @@ -1524,7 +1875,7 @@ async fn test_post_bound_action() { #[tokio::test] async fn test_odata_version_header() { - let app = build_router(test_state()); + let app = authenticated_router(test_state()); let response = app .oneshot(Request::get("/tdata/Orders").body(Body::empty()).unwrap()) .await @@ -1536,7 +1887,7 @@ async fn test_odata_version_header() { #[tokio::test] async fn test_old_odata_path_returns_404() { - let app = build_router(test_state()); + let app = authenticated_router(test_state()); let response = app .oneshot(Request::get("/odata").body(Body::empty()).unwrap()) .await @@ -1547,7 +1898,7 @@ async fn test_old_odata_path_returns_404() { #[tokio::test] async fn test_post_body_used_for_entity_creation() { - let app = build_router(test_state_with_ioa()); + let app = authenticated_router(test_state_with_ioa()); // Create with specific ID and fields let response = app @@ -1573,7 +1924,7 @@ async fn test_post_body_used_for_entity_creation() { #[tokio::test] async fn test_entity_set_returns_created_entities() { - let app = build_router(test_state_with_ioa()); + let app = authenticated_router(test_state_with_ioa()); // Create two entities let _ = app @@ -1614,7 +1965,7 @@ async fn test_entity_set_returns_created_entities() { #[tokio::test] async fn test_patch_updates_entity() { - let app = build_router(test_state_with_ioa()); + let app = authenticated_router(test_state_with_ioa()); // Create entity let _ = app @@ -1650,7 +2001,7 @@ async fn test_patch_updates_entity() { #[tokio::test] async fn test_delete_removes_entity() { - let app = build_router(test_state_with_ioa()); + let app = authenticated_router(test_state_with_ioa()); // Create entity let _ = app @@ -1690,7 +2041,7 @@ async fn test_delete_removes_entity() { #[tokio::test] async fn test_patch_nonexistent_returns_404() { - let app = build_router(test_state_with_ioa()); + let app = authenticated_router(test_state_with_ioa()); let response = app .oneshot( Request::patch("/tdata/Orders('nope')") @@ -1705,7 +2056,7 @@ async fn test_patch_nonexistent_returns_404() { #[tokio::test] async fn test_delete_nonexistent_returns_404() { - let app = build_router(test_state_with_ioa()); + let app = authenticated_router(test_state_with_ioa()); let response = app .oneshot( Request::delete("/tdata/Orders('nope')") @@ -1719,7 +2070,7 @@ async fn test_delete_nonexistent_returns_404() { #[tokio::test] async fn test_navigation_property_single_entity() { - let app = build_router(test_state_with_order_and_payment_ioa()); + let app = authenticated_router(test_state_with_order_and_payment_ioa()); // Create parent order. let order_create = app @@ -1793,7 +2144,7 @@ async fn test_collection_navigation_requires_cedar_list_policy() { ) .expect("install Cedar policy"); - let response = build_router(state) + let response = authenticated_router(state) .oneshot( Request::get("/tdata/Customers('cust-nav')?$expand=Orders") .header("X-Temper-Principal-Id", "customer-1") @@ -1809,7 +2160,7 @@ async fn test_collection_navigation_requires_cedar_list_policy() { #[tokio::test] async fn test_navigation_property_not_found_returns_404() { - let app = build_router(test_state_with_ioa()); + let app = authenticated_router(test_state_with_ioa()); let _ = app .clone() .oneshot( @@ -1834,7 +2185,7 @@ async fn test_navigation_property_not_found_returns_404() { #[tokio::test] async fn test_temper_client_script_served() { - let app = build_router(test_state()); + let app = authenticated_router(test_state()); let response = app .oneshot( Request::get("/temper-client.js") @@ -1862,7 +2213,7 @@ async fn test_temper_client_script_served() { #[tokio::test] async fn test_temper_client_script_alias_served() { - let app = build_router(test_state()); + let app = authenticated_router(test_state()); let response = app .oneshot( Request::get("/static/temper-client.js") @@ -1881,7 +2232,7 @@ async fn test_temper_client_script_alias_served() { #[tokio::test] async fn test_cors_header_present() { - let app = build_router(test_state()); + let app = authenticated_router(test_state()); let response = app .oneshot( Request::get("/tdata/Orders") @@ -1931,7 +2282,7 @@ async fn collect_sse_frames_until( async fn test_sse_events_endpoint_delivers_state_changes() { let state = test_state_with_ioa(); let event_tx = state.event_tx.clone(); - let app = build_router(state); + let app = authenticated_router(state); // Connect to SSE endpoint — response should be 200 with text/event-stream. let response = app @@ -2004,7 +2355,7 @@ async fn test_sse_events_lagged_receiver_continues() { }); } - let app = build_router(state); + let app = authenticated_router(state); let response = app .oneshot( Request::get("/tdata/$events") @@ -2040,80 +2391,6 @@ async fn test_sse_events_lagged_receiver_continues() { ); } -#[test] -fn bridge_resolved_principal_builds_security_context_with_scopes() { - let callback = serde_json::json!({ - "action_params": {}, - "bridge_principal": { - "kind": "customer", - "id": "user-rita", - "scopes": ["repo:push", "force", " ", ""] - } - }); - - let ctx = bridge_resolved_principal(&callback).expect("principal should resolve"); - - assert_eq!(ctx.principal.id, "user-rita"); - assert!(matches!( - ctx.principal.kind, - temper_authz::PrincipalKind::Customer - )); - let scopes = ctx - .principal - .attributes - .get("scopes") - .and_then(|v| v.as_array()) - .cloned() - .unwrap_or_default(); - assert!(scopes.contains(&serde_json::Value::String("repo:push".to_string()))); - assert!(scopes.contains(&serde_json::Value::String("force".to_string()))); - assert_eq!(scopes.len(), 2); -} - -#[test] -fn bridge_resolved_principal_rejects_missing_or_empty_identity() { - assert!(bridge_resolved_principal(&serde_json::json!({ "action_params": {} })).is_none()); - assert!( - bridge_resolved_principal(&serde_json::json!({ - "action_params": {}, - "bridge_principal": { "kind": "customer", "id": " " } - })) - .is_none() - ); - assert!( - bridge_resolved_principal(&serde_json::json!({ - "action_params": {}, - "bridge_principal": { "kind": "", "id": "user-1" } - })) - .is_none() - ); -} - -#[test] -fn bridge_resolved_principal_requires_structured_action_params() { - // A passthrough adapter (top-level params, no action_params key) - // must never hand the caller an identity (ADR-0138). - assert!( - bridge_resolved_principal(&serde_json::json!({ - "bridge_principal": { "kind": "customer", "id": "user-1" } - })) - .is_none() - ); -} - -#[test] -fn bridge_resolved_principal_cannot_smuggle_system_kind() { - let ctx = bridge_resolved_principal(&serde_json::json!({ - "action_params": {}, - "bridge_principal": { "kind": "system", "id": "evil" } - })) - .expect("principal should resolve"); - assert!(matches!( - ctx.principal.kind, - temper_authz::PrincipalKind::Customer - )); -} - #[test] fn bridge_action_params_fallback_strips_control_keys() { let params = bridge_action_params(&serde_json::json!({ @@ -2231,26 +2508,115 @@ fn bridge_short_circuit_response_absent_is_none() { } #[test] -fn http_endpoint_fallback_tenant_prefers_default_over_sort_order() { - // Regression: tenants that sort before "default" (e.g. Directed - // Evolution control tenants on production) must not capture - // header-less protocol requests. - let de = TenantId::new("de-control-agent-answers"); - let default = TenantId::new("default"); - let other = TenantId::new("acme"); - let ids = vec![&other, &de, &default]; - assert_eq!(http_endpoint_fallback_tenant(&ids), Some(&default)); -} +fn credential_headers_are_not_forwarded_to_wasm_guests() { + // ARN-208: a WASM guest reads everything in its invocation context's headers. + // This asserts the invariant against the shared extraction point + // (`guest_visible_headers`) rather than the classifier alone, so deleting the + // filter inside that function fails here. It does NOT assert the dispatch call + // site: re-inlining the filter_map in `dispatch_matched_route` would still pass. + // A true wire-level assertion needs a guest fixture that echoes its invocation + // context; that is deferred with the outbound-header work in ARN-346. + let mut headers = HeaderMap::new(); + for (name, value) in [ + ("authorization", "Bearer caller-token"), + ("proxy-authorization", "Basic proxy"), + ("cookie", "session=abc"), + ("x-api-key", "sk-live-123"), + ("x-forwarded-authorization", "Bearer upstream"), + ("x-forwarded-access-token", "at-456"), + ("x-goog-iap-jwt-assertion", "iap-jwt"), + ("cf-access-jwt-assertion", "cf-jwt"), + ("x-amzn-oidc-accesstoken", "alb-access"), + ("x-amzn-oidc-data", "alb-data"), + ("x-amzn-oidc-identity", "alb-identity"), + ("x-ms-token-aad-access-token", "aad-access"), + ("x-ms-token-aad-id-token", "aad-id"), + ("x-ms-token-aad-refresh-token", "aad-refresh"), + ("x-auth-request-access-token", "oauth2p-access"), + ("x-amz-security-token", "aws-token"), + // Ordinary headers the guest legitimately needs. + ("content-type", "application/json"), + ("accept", "*/*"), + ("user-agent", "curl/8"), + ("x-request-id", "req-1"), + ] { + headers.insert( + axum::http::HeaderName::from_static(name), + axum::http::HeaderValue::from_static(value), + ); + } -#[test] -fn http_endpoint_fallback_tenant_skips_system_then_takes_first() { - let system = TenantId::new("temper-system"); - let acme = TenantId::new("acme"); + // Forces the next author who grows the const to also plant a header/value here: + // the name loop below iterates the const and would otherwise pass trivially for + // an unplanted entry. assert_eq!( - http_endpoint_fallback_tenant(&[&system, &acme]), - Some(&acme) + GUEST_FORBIDDEN_CREDENTIAL_HEADERS.len(), + 16, + "a credential header was added or removed — plant it in this test too" ); - // Only the system tenant registered: still resolves rather than 404. - assert_eq!(http_endpoint_fallback_tenant(&[&system]), Some(&system)); - assert_eq!(http_endpoint_fallback_tenant(&[]), None); + + let visible = guest_visible_headers(&headers); + let names: Vec = visible + .iter() + .map(|(k, _)| k.to_ascii_lowercase()) + .collect(); + + for forbidden in GUEST_FORBIDDEN_CREDENTIAL_HEADERS { + assert!( + !names.iter().any(|n| n == forbidden), + "{forbidden} carries a caller credential and must not reach a guest; got {names:?}" + ); + } + // No credential VALUE survives either. This catches a renamed-but-same-value + // leak, and — unlike the name loop, which iterates the const itself and so + // shrinks with it — every planted secret is asserted independently, so removing + // any single entry from GUEST_FORBIDDEN_CREDENTIAL_HEADERS fails here. + let values: Vec<&str> = visible.iter().map(|(_, v)| v.as_str()).collect(); + for secret in [ + "Bearer caller-token", + "Basic proxy", + "session=abc", + "sk-live-123", + "Bearer upstream", + "at-456", + "iap-jwt", + "cf-jwt", + "alb-access", + "alb-data", + "alb-identity", + "aad-access", + "aad-id", + "aad-refresh", + "oauth2p-access", + "aws-token", + ] { + assert!( + !values.contains(&secret), + "credential value {secret:?} leaked to the guest: {values:?}" + ); + } + // Ordinary request context still reaches the guest. + for expected in ["content-type", "accept", "user-agent", "x-request-id"] { + assert!( + names.iter().any(|n| n == expected), + "{expected} is not a credential and must still be forwarded; got {names:?}" + ); + } +} + +#[test] +fn credential_header_classifier_is_case_insensitive() { + for name in [ + "Authorization", + "COOKIE", + "X-Api-Key", + "Cf-Access-Jwt-Assertion", + ] { + assert!( + is_credential_header(name), + "{name} must be classified as a credential" + ); + } + assert!(!is_credential_header("x-temper-observe-session-id")); + assert!(!is_credential_header("content-type")); } diff --git a/crates/temper-server/src/sentinel.rs b/crates/temper-server/src/sentinel.rs index cd37bb990..7e571f2f2 100644 --- a/crates/temper-server/src/sentinel.rs +++ b/crates/temper-server/src/sentinel.rs @@ -352,6 +352,7 @@ mod tests { request_body: None, intent: None, matched_policy_ids: None, + capture_seq: None, }) .collect(); @@ -401,6 +402,7 @@ mod tests { request_body: None, intent: None, matched_policy_ids: None, + capture_seq: None, }) .collect(); diff --git a/crates/temper-server/src/state/admission.rs b/crates/temper-server/src/state/admission.rs index b82c59c99..63c9dcd34 100644 --- a/crates/temper-server/src/state/admission.rs +++ b/crates/temper-server/src/state/admission.rs @@ -377,6 +377,7 @@ impl crate::state::ServerState { mod tests { use super::*; use std::sync::atomic::{AtomicUsize, Ordering}; + use tokio::spawn as spawn_admission_test_task; // determinism-ok: test-only concurrency harness fn tenant(name: &str) -> TenantId { TenantId::from(name.to_string()) @@ -424,6 +425,38 @@ mod tests { } } + /// Red regression for the separately tracked runtime-override redesign. + /// + /// The admin endpoint currently records an override here, but dispatch + /// immediately supplies spec caps to `try_acquire_with_caps`, which + /// replaces the registered value. Unignore when overrides are keyed by + /// tenant and are resolved ahead of spec defaults at dispatch. + #[tokio::test] + #[ignore = "runtime admission overrides are currently overwritten by inline spec caps"] + async fn runtime_override_should_precede_inline_spec_caps() { + let ac = AdmissionController::new(); + ac.override_caps( + "Session", + Some(make_admission(None, &[("Submit", 1)], Some(0))), + ) + .await; + let spec_caps = make_admission(None, &[("Submit", 2)], Some(1)); + let tenant = tenant("acme"); + + let _first = match ac + .try_acquire_with_caps(&tenant, "Session", "Submit", Some(&spec_caps)) + .await + { + AdmissionOutcome::Granted(permit) => permit, + _ => panic!("first request should acquire the runtime cap"), + }; + assert!(matches!( + ac.try_acquire_with_caps(&tenant, "Session", "Submit", Some(&spec_caps)) + .await, + AdmissionOutcome::Deferred { .. } + )); + } + #[tokio::test] async fn grants_up_to_cap_and_defers_beyond() { let ac = Arc::new(AdmissionController::new()); @@ -517,8 +550,7 @@ mod tests { let t_c = t.clone(); let grant_order = grant_order.clone(); let arrived = arrived.clone(); - let handle = tokio::spawn(async move { - // determinism-ok: test-only concurrency harness + let handle = spawn_admission_test_task(async move { // Stagger arrivals deterministically so arrival order // matches spawn index, exercising the FIFO contract. while arrived.load(Ordering::Acquire) < i { diff --git a/crates/temper-server/src/state/dispatch/actions.rs b/crates/temper-server/src/state/dispatch/actions.rs index 20954e510..18769a72f 100644 --- a/crates/temper-server/src/state/dispatch/actions.rs +++ b/crates/temper-server/src/state/dispatch/actions.rs @@ -36,6 +36,25 @@ fn background_reaction_semaphore() -> Arc { ) } +/// Maximum request-body length, in bytes, echoed into the dispatch-failure log. +const LOG_REQUEST_BODY_MAX_BYTES: usize = 4096; + +/// Bound a serialized request body for the failure log line. +/// +/// Slicing at a fixed byte offset panics when the cap lands inside a multi-byte +/// character, which turns a logged dispatch failure into a second failure. The +/// cut walks back to the nearest character boundary instead. +fn truncate_request_body_for_log(serialized: &str) -> String { + if serialized.len() <= LOG_REQUEST_BODY_MAX_BYTES { + return serialized.to_string(); + } + let mut end = LOG_REQUEST_BODY_MAX_BYTES; + while end > 0 && !serialized.is_char_boundary(end) { + end -= 1; + } + format!("{}[truncated]", &serialized[..end]) +} + impl crate::state::ServerState { /// Dispatch an action using the unified command object. /// @@ -148,9 +167,35 @@ impl crate::state::ServerState { .await } + /// Dispatch an externally authorized action only if the target actor still + /// matches the exact local state used for the Cedar decision. + #[instrument(skip_all, fields( + otel.name = %format_args!("{}.{}", cmd.entity_type, cmd.action), + tenant = %cmd.tenant, + entity_type = cmd.entity_type, + entity_id = cmd.entity_id, + action_name = cmd.action, + ))] + pub(crate) async fn dispatch_tenant_action_ext_typed_if_current( + &self, + cmd: DispatchCommand<'_>, + expected_authorization_precondition: String, + ) -> Result { + self.dispatch_typed_checked(cmd, Some(expected_authorization_precondition)) + .await + } + async fn dispatch_typed( &self, cmd: DispatchCommand<'_>, + ) -> Result { + self.dispatch_typed_checked(cmd, None).await + } + + async fn dispatch_typed_checked( + &self, + cmd: DispatchCommand<'_>, + expected_authorization_precondition: Option, ) -> Result { let DispatchCommand { tenant, @@ -179,6 +224,7 @@ impl crate::state::ServerState { params, agent_ctx, await_integration, + expected_authorization_precondition, ) .await?; @@ -338,6 +384,7 @@ impl crate::state::ServerState { params: serde_json::Value, agent_ctx: &AgentContext, await_integration: bool, + expected_authorization_precondition: Option, ) -> Result { let explicit_workflow_context = agent_ctx.workflow_run_id.is_some() || agent_ctx.workflow_root_entity_type.is_some() @@ -516,6 +563,7 @@ impl crate::state::ServerState { let action_name = action.to_string(); let params_for_retry = params; let cross_for_retry = cross_entity_booleans; + let authorization_precondition_for_retry = expected_authorization_precondition; let idempotency_key = Some(agent_ctx.idempotency_key.clone().unwrap_or_else(|| { format!( "dispatch:{tenant}:{entity_type}:{entity_id}:{action}:{}", @@ -539,6 +587,8 @@ impl crate::state::ServerState { params: params_for_retry.clone(), cross_entity_booleans: cross_for_retry.clone(), idempotency_key: idempotency_key.clone(), + expected_authorization_precondition: authorization_precondition_for_retry + .clone(), }, &policy, ) @@ -627,15 +677,9 @@ impl crate::state::ServerState { request_body: Some(action_params.clone()), intent: agent_ctx.intent.clone(), matched_policy_ids: None, + capture_seq: None, }; - let request_body_str = { - let s = action_params.to_string(); - if s.len() > 4096 { - format!("{}[truncated]", &s[..4096]) - } else { - s - } - }; + let request_body_str = truncate_request_body_for_log(&action_params.to_string()); let from_status = entry.from_status.as_deref().unwrap_or("unknown"); let to_status = entry.to_status.as_deref().unwrap_or("unknown"); let observation_metadata = @@ -725,3 +769,35 @@ impl crate::state::ServerState { Ok(response) } } + +#[cfg(test)] +mod log_truncation_tests { + use super::{LOG_REQUEST_BODY_MAX_BYTES, truncate_request_body_for_log}; + + #[test] + fn short_body_is_logged_verbatim() { + let body = r#"{"ProductId":"p-1"}"#; + assert_eq!(truncate_request_body_for_log(body), body); + } + + #[test] + fn oversized_ascii_body_is_marked_truncated() { + let body = "a".repeat(LOG_REQUEST_BODY_MAX_BYTES + 100); + let truncated = truncate_request_body_for_log(&body); + assert!(truncated.ends_with("[truncated]")); + assert_eq!( + truncated.len(), + LOG_REQUEST_BODY_MAX_BYTES + "[truncated]".len() + ); + } + + #[test] + fn oversized_multibyte_body_does_not_panic_at_the_cap() { + // 3-byte characters: the cap at 4096 falls mid-character (4096 % 3 != 0), + // which a raw byte slice would panic on. + let body = "\u{4e16}".repeat(LOG_REQUEST_BODY_MAX_BYTES); + let truncated = truncate_request_body_for_log(&body); + assert!(truncated.ends_with("[truncated]")); + assert!(truncated.len() <= LOG_REQUEST_BODY_MAX_BYTES + "[truncated]".len()); + } +} diff --git a/crates/temper-server/src/state/dispatch/adapter.rs b/crates/temper-server/src/state/dispatch/adapter.rs index e26b7dd40..b14a8941e 100644 --- a/crates/temper-server/src/state/dispatch/adapter.rs +++ b/crates/temper-server/src/state/dispatch/adapter.rs @@ -1,14 +1,151 @@ +use std::collections::BTreeMap; +use std::panic::AssertUnwindSafe; +use std::sync::Arc; +use std::time::Duration; + +use futures_util::FutureExt; +use sha2::{Digest, Sha256}; +use tokio::spawn as spawn_external_adapter_task; // determinism-ok: external adapter side effects use tracing::{Instrument, instrument}; -use crate::adapters::{AdapterAgentContext, AdapterContext, AdapterResult}; -use crate::entity_actor::{EntityResponse, EntityState}; +use crate::adapters::{ + AdapterAgentContext, AdapterContext, AdapterError, AdapterResult, AgentAdapter, +}; +use crate::entity_actor::{EntityMsg, EntityResponse, EntityState}; use crate::identity::hash_token; use crate::request_context::AgentContext; use crate::secrets::template::resolve_secret_templates; -use temper_runtime::scheduler::sim_uuid; +use temper_runtime::scheduler::{sim_now, sim_uuid}; use temper_runtime::tenant::TenantId; -use super::{WasmDispatchMode, WasmDispatchRequest, WasmEntityRef, record_workflow_span_attrs}; +use super::{ + WasmDispatchMode, WasmDispatchRequest, WasmEntityRef, record_workflow_span_attrs, retry, +}; + +const ADAPTER_INVOCATION_BUDGET_SECS: u64 = 60 * 60; +const ADAPTER_CREDENTIAL_TTL_SECS: i64 = 61 * 60; +const ADAPTER_CREDENTIAL_REVOKE_ATTEMPTS: usize = 3; + +struct MintedAdapterCredential { + plaintext: String, + key_hash: String, +} + +const REDACTED_ADAPTER_CREDENTIAL: &str = "[REDACTED_ADAPTER_CREDENTIAL]"; +const REDACTED_ADAPTER_SECRET: &str = "[REDACTED_ADAPTER_SECRET]"; + +fn derive_adapter_credential_plaintext(first: uuid::Uuid, second: uuid::Uuid) -> String { + let mut digest = Sha256::new(); + digest.update(b"temper-adapter-credential-v1\0"); + digest.update(first.as_bytes()); + digest.update(second.as_bytes()); + format!("tmpr_{:x}", digest.finalize()) +} + +fn adapter_redactions(ctx: &AdapterContext) -> Vec<(String, &'static str)> { + let mut by_value = BTreeMap::new(); + for secret in ctx.secrets.values().filter(|secret| !secret.is_empty()) { + by_value.insert(secret.clone(), REDACTED_ADAPTER_SECRET); + } + if let Some(credential) = ctx + .agent_ctx + .agent_api_key + .as_ref() + .filter(|credential| !credential.is_empty()) + { + by_value.insert(credential.clone(), REDACTED_ADAPTER_CREDENTIAL); + } + let mut redactions = by_value.into_iter().collect::>(); + redactions.sort_by(|left, right| { + right + .0 + .len() + .cmp(&left.0.len()) + .then_with(|| left.0.cmp(&right.0)) + }); + redactions +} + +fn redact_adapter_text(mut text: String, redactions: &[(String, &'static str)]) -> String { + for (secret, replacement) in redactions { + text = text.replace(secret, replacement); + } + text +} + +fn redact_adapter_json(value: &mut serde_json::Value, redactions: &[(String, &'static str)]) { + match value { + serde_json::Value::String(text) => { + *text = redact_adapter_text(std::mem::take(text), redactions); + } + serde_json::Value::Array(values) => { + for value in values { + redact_adapter_json(value, redactions); + } + } + serde_json::Value::Object(fields) => { + let prior = std::mem::take(fields); + for (key, mut value) in prior { + redact_adapter_json(&mut value, redactions); + fields.insert(redact_adapter_text(key, redactions), value); + } + } + serde_json::Value::Null | serde_json::Value::Bool(_) | serde_json::Value::Number(_) => {} + } +} + +fn redact_adapter_execution( + execution: Result, + redactions: &[(String, &'static str)], +) -> Result { + if redactions.is_empty() { + return execution; + } + match execution { + Ok(mut result) => { + if let Some(action) = result.callback_action.take() { + result.callback_action = Some(redact_adapter_text(action, redactions)); + } + if let Some(error) = result.error.take() { + result.error = Some(redact_adapter_text(error, redactions)); + } + redact_adapter_json(&mut result.callback_params, redactions); + Ok(result) + } + Err(AdapterError::Invocation(error)) => Err(AdapterError::Invocation(redact_adapter_text( + error, redactions, + ))), + Err(AdapterError::Execution(error)) => Err(AdapterError::Execution(redact_adapter_text( + error, redactions, + ))), + Err(AdapterError::Parse(error)) => { + Err(AdapterError::Parse(redact_adapter_text(error, redactions))) + } + } +} + +async fn execute_adapter_with_budget( + adapter: Arc, + adapter_ctx: AdapterContext, +) -> Result { + let redactions = adapter_redactions(&adapter_ctx); + let execution = AssertUnwindSafe(tokio::time::timeout( + Duration::from_secs(ADAPTER_INVOCATION_BUDGET_SECS), + adapter.execute(adapter_ctx), + )) + .catch_unwind() + .await; + let execution = match execution { + Ok(Ok(result)) => result, + Ok(Err(_elapsed)) => Err(AdapterError::Execution(format!( + "adapter invocation exceeded its {ADAPTER_INVOCATION_BUDGET_SECS}-second budget" + ))), + Err(_) => Err(AdapterError::Execution( + "adapter invocation panicked".to_string(), + )), + }; + redact_adapter_execution(execution, &redactions) +} struct AdapterDispatchCtx<'a> { entity_ref: WasmEntityRef<'a>, @@ -62,7 +199,7 @@ impl crate::state::ServerState { entity_id = %entity_id, ); - tokio::spawn( + spawn_external_adapter_task( async move { // determinism-ok: async integration side-effects run outside simulation core let req = WasmDispatchRequest { @@ -215,9 +352,16 @@ impl crate::state::ServerState { // Mint a platform credential if the entity references an AgentType (ADR-0033). // The plaintext key is passed to the adapter and never persisted. - let agent_api_key = self - .mint_agent_credential_if_needed(ctx.entity_ref.tenant, entity_state, ctx.agent_ctx) - .await; + let minted_credential = if adapter.requires_platform_credential() { + self.mint_agent_credential_if_needed(ctx.entity_ref.tenant, entity_state, ctx.agent_ctx) + .await? + } else { + None + }; + let (agent_api_key, credential_key_hash) = match minted_credential { + Some(credential) => (Some(credential.plaintext), Some(credential.key_hash)), + None => (None, None), + }; let adapter_ctx = AdapterContext { tenant, @@ -236,7 +380,15 @@ impl crate::state::ServerState { secrets, }; - let result = match adapter.execute(adapter_ctx).await { + let result = match self + .execute_adapter_with_credential_cleanup( + adapter, + adapter_ctx, + ctx.entity_ref.tenant, + credential_key_hash, + ) + .await + { Ok(result) => result, Err(e) => { return self @@ -273,6 +425,99 @@ impl crate::state::ServerState { .await } + async fn execute_adapter_with_credential_cleanup( + &self, + adapter: Arc, + adapter_ctx: AdapterContext, + tenant: &TenantId, + credential_key_hash: Option, + ) -> Result { + let Some(credential_key_hash) = credential_key_hash else { + return execute_adapter_with_budget(adapter, adapter_ctx).await; + }; + let state = self.clone(); + let tenant = tenant.clone(); + let span = tracing::Span::current(); + + // This task owns both execution and cleanup. Dropping the caller's + // JoinHandle detaches rather than cancels it, so request cancellation + // cannot skip durable credential revocation. + let cleanup_task = spawn_external_adapter_task( + async move { + // determinism-ok: native adapter execution is an external side effect + let execution = execute_adapter_with_budget(adapter, adapter_ctx).await; + if let Err(error) = state + .revoke_minted_adapter_credential(&tenant, &credential_key_hash) + .await + { + tracing::error!( + tenant = %tenant, + adapter_execution_completed = execution.is_ok(), + credential_ttl_secs = ADAPTER_CREDENTIAL_TTL_SECS, + error = %error, + "adapter credential cleanup exhausted its retry budget; preserving the adapter execution outcome" + ); + } + execution + } + .instrument(span), + ); + + cleanup_task.await.map_err(|error| { + AdapterError::Execution(format!("adapter cleanup task failed: {error}")) + })? + } + + async fn revoke_minted_adapter_credential( + &self, + tenant: &TenantId, + key_hash: &str, + ) -> Result<(), String> { + let actor = self + .get_or_spawn_tenant_actor(tenant, "AgentCredential", key_hash) + .ok_or_else(|| "AgentCredential transition table is unavailable".to_string())?; + let policy = self.dispatch_retry_policy(); + let idempotency_key = format!("adapter-credential-revoke:{key_hash}"); + let mut last_error = "credential remained Active".to_string(); + + for attempt in 1..=ADAPTER_CREDENTIAL_REVOKE_ATTEMPTS { + let outcome = retry::ask_with_backoff::<_, EntityResponse, _>( + &actor, + || EntityMsg::Action { + name: "Revoke".to_string(), + params: serde_json::json!({}), + cross_entity_booleans: BTreeMap::new(), + idempotency_key: Some(idempotency_key.clone()), + expected_authorization_precondition: None, + }, + &policy, + ) + .await; + match outcome.result { + Ok(response) if response.success || response.state.status != "Active" => { + return Ok(()); + } + Ok(response) => { + last_error = response + .error + .unwrap_or_else(|| "credential remained Active".to_string()); + } + Err(error) => { + last_error = error.to_string(); + } + } + tracing::warn!( + tenant = %tenant, + attempt, + max_attempts = ADAPTER_CREDENTIAL_REVOKE_ATTEMPTS, + error = %last_error, + "adapter credential revocation attempt failed" + ); + } + + Err(last_error) + } + #[instrument(skip_all, fields(otel.name = "dispatch.handle_adapter_failure", integration = %integration.name))] async fn handle_adapter_failure( &self, @@ -334,6 +579,7 @@ impl crate::state::ServerState { callback_params, agent_ctx, false, + None, ) .await .map_err(|e| e.to_string())?; @@ -365,8 +611,9 @@ impl crate::state::ServerState { /// Mint a platform credential for adapter execution if the entity has an `agent_type_id`. /// /// Generates a random API key, hashes it, creates an `AgentCredential` entity - /// via the `Issue` action, and returns the plaintext key. The key is never - /// persisted — it exists only for the lifetime of this adapter invocation. + /// via the `Issue` action, and returns the plaintext key. The full key is + /// never persisted or logged; only its hash and prefix are durable. The + /// credential has a bounded expiry and is revoked after execution. /// /// See ADR-0033: Platform-Assigned Agent Identity. async fn mint_agent_credential_if_needed( @@ -374,31 +621,73 @@ impl crate::state::ServerState { tenant: &TenantId, entity_state: &EntityState, agent_ctx: &AgentContext, - ) -> Option { + ) -> Result, String> { let agent_type_id = entity_state .fields .get("agent_type_id") .and_then(|v| v.as_str()) .map(str::trim) - .filter(|v| !v.is_empty())?; + .filter(|v| !v.is_empty()); + let Some(agent_type_id) = agent_type_id else { + return Ok(None); + }; - // Generate a random API key from a UUIDv7 (deterministic in simulation). - let key_uuid = sim_uuid(); - let plaintext_key = format!("tmpr_{key_uuid}"); + // UUIDv7 contains only 74 random bits. Derive the opaque credential + // from two independent scheduler-provided UUIDs so production exceeds + // the 128-bit entropy bar while DST retains deterministic sources. + let plaintext_key = derive_adapter_credential_plaintext(sim_uuid(), sim_uuid()); let key_hash = hash_token(&plaintext_key); - let key_prefix = &plaintext_key[..9]; // "tmpr_" + first 4 chars of UUID + let key_prefix = &plaintext_key[..9]; // "tmpr_" + first four digest characters let agent_instance_id = sim_uuid().to_string(); + let expires_at = sim_now() + .checked_add_signed(chrono::Duration::seconds(ADAPTER_CREDENTIAL_TTL_SECS)) + .ok_or_else(|| "adapter credential expiry overflowed".to_string())? + .to_rfc3339(); let issue_params = serde_json::json!({ "agent_type_id": agent_type_id, "agent_instance_id": agent_instance_id, "key_hash": key_hash, "key_prefix": key_prefix, - "description": format!("Auto-minted for adapter invocation"), + "description": "Auto-minted for adapter invocation", "created_by": "platform", - "expires_at": "", + "expires_at": expires_at, }); + // Credential issuance is a separate authority from permission to run + // the source entity action. Re-evaluate the exact invoking principal + // against the credential resource so a low-privilege action cannot use + // the platform as a deputy to mint a more privileged AgentType token. + let security_ctx = agent_ctx.security_ctx.as_ref().ok_or_else(|| { + "adapter credential mint requires an explicit security context".to_string() + })?; + let credential_attrs = BTreeMap::from([ + ( + "id".to_string(), + serde_json::Value::String(key_hash.clone()), + ), + ( + "agent_type_id".to_string(), + serde_json::Value::String(agent_type_id.to_string()), + ), + ( + "agent_instance_id".to_string(), + serde_json::Value::String(agent_instance_id.clone()), + ), + ( + "expires_at".to_string(), + serde_json::Value::String(expires_at.clone()), + ), + ]); + self.authorize_with_context( + security_ctx, + "Issue", + "AgentCredential", + &credential_attrs, + tenant.as_str(), + ) + .map_err(|denial| format!("adapter credential delegation denied: {denial}"))?; + // Create the AgentCredential entity using key_hash as entity ID for O(1) lookup. let dispatch_ctx = AgentContext::for_service_inheriting("platform-dispatch", agent_ctx); let result = self @@ -421,24 +710,19 @@ impl crate::state::ServerState { key_prefix = key_prefix, "minted agent credential for adapter execution" ); - Some(plaintext_key) - } - Ok(resp) => { - tracing::warn!( - tenant = %tenant, - error = ?resp.error, - "failed to mint agent credential — adapter will run without credential" - ); - None - } - Err(e) => { - tracing::warn!( - tenant = %tenant, - error = %e, - "failed to mint agent credential — adapter will run without credential" - ); - None + Ok(Some(MintedAdapterCredential { + plaintext: plaintext_key, + key_hash, + })) } + Ok(resp) => Err(format!( + "failed to mint required adapter credential: {}", + resp.error + .unwrap_or_else(|| "Issue action was rejected".to_string()) + )), + Err(error) => Err(format!( + "failed to mint required adapter credential: {error}" + )), } } } @@ -461,3 +745,7 @@ fn normalize_success_params(result: AdapterResult) -> serde_json::Value { }), } } + +#[cfg(test)] +#[path = "adapter_credential_test.rs"] +mod credential_tests; diff --git a/crates/temper-server/src/state/dispatch/adapter_credential_test.rs b/crates/temper-server/src/state/dispatch/adapter_credential_test.rs new file mode 100644 index 000000000..2b9af7da3 --- /dev/null +++ b/crates/temper-server/src/state/dispatch/adapter_credential_test.rs @@ -0,0 +1,544 @@ +use super::*; + +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use async_trait::async_trait; +use temper_runtime::ActorSystem; +use temper_runtime::persistence::EventStore; +use temper_runtime::scheduler::{DeterministicIdGen, LogicalClock, install_sim_context}; +use temper_spec::csdl::parse_csdl; +use temper_store_turso::TursoEventStore; +use tokio::sync::{Notify, oneshot}; + +use crate::storage::StorageStack; + +const AGENT_CSDL: &str = include_str!("../../../../temper-platform/src/specs/agent_model.csdl.xml"); +const AGENT_TYPE_IOA: &str = + include_str!("../../../../temper-platform/src/specs/agent_type.ioa.toml"); +const AGENT_CREDENTIAL_IOA: &str = + include_str!("../../../../temper-platform/src/specs/agent_credential.ioa.toml"); +const PROVIDER_SECRET: &str = "provider-super-secret"; + +struct PersistedIdentityFixture { + first: crate::state::ServerState, + second: crate::state::ServerState, + store: TursoEventStore, + _directory: tempfile::TempDir, +} + +async fn persisted_identity_fixture() -> PersistedIdentityFixture { + let csdl = parse_csdl(AGENT_CSDL).expect("agent CSDL should parse"); + let mut registry = crate::registry::SpecRegistry::new(); + registry.register_tenant( + "default", + csdl, + AGENT_CSDL.to_string(), + &[ + ("AgentType", AGENT_TYPE_IOA), + ("AgentCredential", AGENT_CREDENTIAL_IOA), + ], + ); + let second_registry = registry.clone(); + let mut first = crate::state::ServerState::from_registry( + ActorSystem::new("adapter-credential-first"), + registry, + ); + let mut second = crate::state::ServerState::from_registry( + ActorSystem::new("adapter-credential-second"), + second_registry, + ); + let directory = tempfile::tempdir().expect("create adapter credential test directory"); + let database_url = format!("file:{}", directory.path().join("identity.db").display()); + let store = TursoEventStore::new(&database_url, None) + .await + .expect("create adapter credential store"); + first.set_storage_stack(StorageStack::from_turso(store.clone())); + second.set_storage_stack(StorageStack::from_turso(store.clone())); + + let response = first + .dispatch_tenant_action( + &TenantId::default(), + "AgentType", + "adapter-agent-type", + "Define", + serde_json::json!({ + "name": "adapter-agent", + "system_prompt": "test", + "tool_set": "local", + "model": "test", + "max_turns": "1", + "adapter_config": "{}", + "default_budget_cents": "0" + }), + &AgentContext::system(), + ) + .await + .expect("define adapter AgentType"); + assert!(response.success, "Define failed: {:?}", response.error); + + PersistedIdentityFixture { + first, + second, + store, + _directory: directory, + } +} + +fn credential_entity_state() -> EntityState { + EntityState { + entity_type: "AdapterRun".to_string(), + entity_id: "run-1".to_string(), + status: "Running".to_string(), + item_count: 0, + counters: BTreeMap::new(), + booleans: BTreeMap::new(), + lists: BTreeMap::new(), + fields: serde_json::json!({"agent_type_id": "adapter-agent-type"}), + events: std::collections::VecDeque::new(), + total_event_count: 0, + events_since_snapshot: 0, + last_snapshot_sequence_nr: 0, + sequence_nr: 0, + processed_idempotency_keys: BTreeMap::new(), + } +} + +#[test] +fn adapter_credential_uses_both_uuid_sources_and_has_opaque_shape() { + let first = uuid::Uuid::from_u128(1); + let second = uuid::Uuid::from_u128(2); + let token = derive_adapter_credential_plaintext(first, second); + + assert_eq!(token.len(), "tmpr_".len() + 64); + assert!( + token + .strip_prefix("tmpr_") + .is_some_and(|material| material.bytes().all(|byte| byte.is_ascii_hexdigit())) + ); + assert_ne!( + token, + derive_adapter_credential_plaintext(uuid::Uuid::from_u128(3), second), + "the first independent UUID must contribute to the token" + ); + assert_ne!( + token, + derive_adapter_credential_plaintext(first, uuid::Uuid::from_u128(3)), + "the second independent UUID must contribute to the token" + ); + assert!(!token.contains(&first.to_string())); + assert!(!token.contains(&second.to_string())); +} + +fn adapter_context(token: String) -> AdapterContext { + AdapterContext { + tenant: "default".to_string(), + entity_type: "AdapterRun".to_string(), + entity_id: "run-1".to_string(), + trigger_action: "Run".to_string(), + trigger_params: serde_json::json!({}), + entity_state: serde_json::json!({}), + integration_config: BTreeMap::new(), + agent_ctx: AdapterAgentContext { + agent_api_key: Some(token), + ..AdapterAgentContext::default() + }, + secrets: BTreeMap::from([("OPENAI_API_KEY".to_string(), PROVIDER_SECRET.to_string())]), + } +} + +async fn mint(fixture: &PersistedIdentityFixture) -> MintedAdapterCredential { + fixture + .first + .mint_agent_credential_if_needed( + &TenantId::default(), + &credential_entity_state(), + &AgentContext::system(), + ) + .await + .expect("credential Issue should succeed") + .expect("agent_type_id should request a credential") +} + +async fn assert_token_resolves(state: &crate::state::ServerState, token: &str, expected: bool) { + let resolved = crate::identity::IdentityResolver::new() + .resolve(state, &TenantId::default(), token) + .await; + assert_eq!(resolved.is_some(), expected); +} + +struct ResultAdapter { + captured: Mutex>>, + fails: bool, +} + +#[async_trait] +impl AgentAdapter for ResultAdapter { + fn adapter_type(&self) -> &str { + "credential-result-test" + } + + async fn execute(&self, ctx: AdapterContext) -> Result { + let token = ctx + .agent_ctx + .agent_api_key + .expect("test adapter should receive credential"); + let provider_secret = ctx + .secrets + .get("OPENAI_API_KEY") + .expect("test adapter should receive provider secret"); + let captured = token.clone(); + if let Some(sender) = self + .captured + .lock() + .expect("capture lock should be healthy") + .take() + { + let _ = sender.send(captured); + } + if self.fails { + Err(AdapterError::Execution(format!( + "injected adapter error containing {token} and {provider_secret}" + ))) + } else { + Ok(AdapterResult::success( + serde_json::json!({ + "echo": token, + format!("key-{token}"): [ + format!("nested-{token}"), + format!("provider-{provider_secret}") + ] + }), + 1, + )) + } + } +} + +struct BlockingAdapter { + started: Arc, + finish: Arc, +} + +struct PanicAdapter; + +#[async_trait] +impl AgentAdapter for PanicAdapter { + fn adapter_type(&self) -> &str { + "credential-panic-test" + } + + async fn execute(&self, _ctx: AdapterContext) -> Result { + panic!("injected adapter panic") + } +} + +struct NeverCompletesAdapter { + started: Arc, +} + +#[async_trait] +impl AgentAdapter for NeverCompletesAdapter { + fn adapter_type(&self) -> &str { + "credential-timeout-test" + } + + async fn execute(&self, _ctx: AdapterContext) -> Result { + self.started.notify_one(); + std::future::pending().await + } +} + +#[async_trait] +impl AgentAdapter for BlockingAdapter { + fn adapter_type(&self) -> &str { + "credential-cancellation-test" + } + + async fn execute(&self, _ctx: AdapterContext) -> Result { + self.started.notify_one(); + self.finish.notified().await; + Ok(AdapterResult::success(serde_json::json!({}), 1)) + } +} + +#[tokio::test] +async fn adapter_success_revokes_captured_token_and_never_persists_plaintext() { + let fixture = persisted_identity_fixture().await; + let credential = mint(&fixture).await; + let plaintext = credential.plaintext.clone(); + let key_hash = credential.key_hash.clone(); + assert_token_resolves(&fixture.second, &plaintext, true).await; + let (sender, receiver) = oneshot::channel(); + + let result = fixture + .first + .execute_adapter_with_credential_cleanup( + Arc::new(ResultAdapter { + captured: Mutex::new(Some(sender)), + fails: false, + }), + adapter_context(credential.plaintext), + &TenantId::default(), + Some(credential.key_hash), + ) + .await + .expect("adapter should succeed"); + assert!(result.success); + let serialized_result = serde_json::to_string(&result).expect("serialize adapter result"); + assert!(!serialized_result.contains(&plaintext)); + assert!(!serialized_result.contains(PROVIDER_SECRET)); + assert!(serialized_result.contains(REDACTED_ADAPTER_CREDENTIAL)); + assert!(serialized_result.contains(REDACTED_ADAPTER_SECRET)); + assert_eq!( + receiver.await.expect("adapter should capture token"), + plaintext + ); + assert_token_resolves(&fixture.second, &plaintext, false).await; + + let events = fixture + .store + .read_events(&format!("default:AgentCredential:{key_hash}"), 0) + .await + .expect("read credential journal"); + let durable_json = serde_json::to_string(&events).expect("serialize credential events"); + assert!(!durable_json.contains(&plaintext)); +} + +#[tokio::test] +async fn adapter_error_still_revokes_captured_token() { + let fixture = persisted_identity_fixture().await; + let credential = mint(&fixture).await; + let plaintext = credential.plaintext.clone(); + let (sender, receiver) = oneshot::channel(); + + let error = fixture + .first + .execute_adapter_with_credential_cleanup( + Arc::new(ResultAdapter { + captured: Mutex::new(Some(sender)), + fails: true, + }), + adapter_context(credential.plaintext), + &TenantId::default(), + Some(credential.key_hash), + ) + .await + .expect_err("adapter error should propagate after cleanup"); + assert!(error.to_string().contains("injected adapter error")); + assert!(!error.to_string().contains(&plaintext)); + assert!(!error.to_string().contains(PROVIDER_SECRET)); + assert!(error.to_string().contains(REDACTED_ADAPTER_CREDENTIAL)); + assert!(error.to_string().contains(REDACTED_ADAPTER_SECRET)); + assert_eq!( + receiver.await.expect("adapter should capture token"), + plaintext + ); + assert_token_resolves(&fixture.second, &plaintext, false).await; +} + +#[tokio::test] +async fn adapter_success_is_preserved_when_credential_cleanup_fails() { + let fixture = persisted_identity_fixture().await; + + let result = fixture + .first + .execute_adapter_with_credential_cleanup( + Arc::new(ResultAdapter { + captured: Mutex::new(None), + fails: false, + }), + adapter_context("ephemeral-token".to_string()), + &TenantId::default(), + Some("missing-credential-key-hash".to_string()), + ) + .await + .expect("successful adapter result must survive cleanup failure"); + + assert!(result.success); +} + +#[tokio::test] +async fn adapter_error_is_preserved_and_redacted_when_credential_cleanup_fails() { + let fixture = persisted_identity_fixture().await; + let plaintext = "ephemeral-token".to_string(); + + let error = fixture + .first + .execute_adapter_with_credential_cleanup( + Arc::new(ResultAdapter { + captured: Mutex::new(None), + fails: true, + }), + adapter_context(plaintext.clone()), + &TenantId::default(), + Some("missing-credential-key-hash".to_string()), + ) + .await + .expect_err("adapter error must remain primary after cleanup failure"); + + let error = error.to_string(); + assert!(error.contains("injected adapter error")); + assert!(!error.contains(&plaintext)); + assert!(!error.contains(PROVIDER_SECRET)); + assert!(error.contains(REDACTED_ADAPTER_CREDENTIAL)); + assert!(error.contains(REDACTED_ADAPTER_SECRET)); +} + +#[tokio::test] +async fn adapter_panic_is_contained_and_still_revokes_token() { + let fixture = persisted_identity_fixture().await; + let credential = mint(&fixture).await; + let plaintext = credential.plaintext.clone(); + + let error = fixture + .first + .execute_adapter_with_credential_cleanup( + Arc::new(PanicAdapter), + adapter_context(credential.plaintext), + &TenantId::default(), + Some(credential.key_hash), + ) + .await + .expect_err("adapter panic should become a typed error after cleanup"); + assert!(error.to_string().contains("adapter invocation panicked")); + assert_token_resolves(&fixture.second, &plaintext, false).await; +} + +#[tokio::test(start_paused = true)] +async fn adapter_timeout_revokes_token_at_the_execution_budget() { + let fixture = persisted_identity_fixture().await; + let credential = mint(&fixture).await; + let plaintext = credential.plaintext.clone(); + let state = fixture.first.clone(); + let started = Arc::new(Notify::new()); + let started_for_adapter = started.clone(); + let execution = tokio::spawn(async move { + state + .execute_adapter_with_credential_cleanup( + Arc::new(NeverCompletesAdapter { + started: started_for_adapter, + }), + adapter_context(credential.plaintext), + &TenantId::default(), + Some(credential.key_hash), + ) + .await + }); + + started.notified().await; + tokio::time::advance(Duration::from_secs(ADAPTER_INVOCATION_BUDGET_SECS + 1)).await; + let error = execution + .await + .expect("execution task should stay healthy") + .expect_err("adapter should exceed its execution budget"); + assert!( + error + .to_string() + .contains("exceeded its 3600-second budget") + ); + assert_token_resolves(&fixture.second, &plaintext, false).await; +} + +#[tokio::test] +async fn caller_cancellation_detaches_cleanup_and_revokes_after_adapter_finishes() { + let fixture = persisted_identity_fixture().await; + let credential = mint(&fixture).await; + let plaintext = credential.plaintext.clone(); + let started = Arc::new(Notify::new()); + let finish = Arc::new(Notify::new()); + let state = fixture.first.clone(); + let started_for_adapter = started.clone(); + let finish_for_adapter = finish.clone(); + let caller = tokio::spawn(async move { + state + .execute_adapter_with_credential_cleanup( + Arc::new(BlockingAdapter { + started: started_for_adapter, + finish: finish_for_adapter, + }), + adapter_context(credential.plaintext), + &TenantId::default(), + Some(credential.key_hash), + ) + .await + }); + + started.notified().await; + caller.abort(); + finish.notify_one(); + assert!( + caller + .await + .expect_err("caller task should be cancelled") + .is_cancelled() + ); + + for _ in 0..100 { + if crate::identity::IdentityResolver::new() + .resolve(&fixture.second, &TenantId::default(), &plaintext) + .await + .is_none() + { + return; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + panic!("detached cleanup did not revoke the captured credential"); +} + +#[tokio::test(flavor = "current_thread")] +async fn bounded_expiry_denies_token_from_second_state_without_cleanup() { + let clock = Arc::new(LogicalClock::with_delta_ms(1_000)); + let id_gen = Arc::new(DeterministicIdGen::new(42)); + let _clock_guard = install_sim_context(clock.clone(), id_gen); + let fixture = persisted_identity_fixture().await; + let credential = mint(&fixture).await; + let plaintext = credential.plaintext.clone(); + assert_token_resolves(&fixture.second, &plaintext, true).await; + + clock.advance_by(ADAPTER_CREDENTIAL_TTL_SECS as u64 + 1); + assert_token_resolves(&fixture.second, &plaintext, false).await; + + fixture + .first + .revoke_minted_adapter_credential(&TenantId::default(), &credential.key_hash) + .await + .expect("expired credential should still revoke durably"); +} + +#[tokio::test] +async fn adapter_credential_mint_requires_caller_delegation_authority() { + let fixture = persisted_identity_fixture().await; + let unprivileged = AgentContext { + security_ctx: Some(temper_authz::SecurityContext::from_resolved_identity( + "unprivileged-agent", + "worker", + None, + )), + ..AgentContext::default() + }; + let before = fixture + .first + .list_entity_ids(&TenantId::default(), "AgentCredential"); + + let result = fixture + .first + .mint_agent_credential_if_needed( + &TenantId::default(), + &credential_entity_state(), + &unprivileged, + ) + .await; + let Err(error) = result else { + panic!("default-deny caller minted an adapter credential"); + }; + + assert!(error.contains("delegation denied"), "{error}"); + assert_eq!( + fixture + .first + .list_entity_ids(&TenantId::default(), "AgentCredential"), + before + ); +} diff --git a/crates/temper-server/src/state/dispatch/composite.rs b/crates/temper-server/src/state/dispatch/composite.rs index 554a26a59..40d1763ff 100644 --- a/crates/temper-server/src/state/dispatch/composite.rs +++ b/crates/temper-server/src/state/dispatch/composite.rs @@ -146,7 +146,7 @@ impl crate::state::ServerState { agent_ctx .security_ctx .clone() - .unwrap_or_else(|| SecurityContext::from_headers(&[])) + .unwrap_or_else(SecurityContext::anonymous) .with_action_context(composite_action_context), ); @@ -193,6 +193,7 @@ impl crate::state::ServerState { prepared.params, &sub_agent_ctx, false, + None, ) .await?; diff --git a/crates/temper-server/src/state/dispatch/composite/helpers.rs b/crates/temper-server/src/state/dispatch/composite/helpers.rs index 99ec9525f..820890259 100644 --- a/crates/temper-server/src/state/dispatch/composite/helpers.rs +++ b/crates/temper-server/src/state/dispatch/composite/helpers.rs @@ -370,7 +370,10 @@ pub(super) fn composite_batch_persistence_error(error: PersistenceError) -> Disp pub(super) fn composite_storage_cap_error(error: CommonsStorageCapError) -> DispatchError { match error { - CommonsStorageCapError::Exceeded(_) => DispatchError::QuotaExceeded(error.to_string()), + CommonsStorageCapError::Exceeded(_) + | CommonsStorageCapError::ReservationCapacityExhausted => { + DispatchError::QuotaExceeded(error.to_string()) + } CommonsStorageCapError::OwnerSuspended(_) => DispatchError::AuthzDenied(error.to_string()), CommonsStorageCapError::MissingAttribution(_) | CommonsStorageCapError::Internal(_) => { DispatchError::Internal(error.to_string()) diff --git a/crates/temper-server/src/state/dispatch/composite_test.rs b/crates/temper-server/src/state/dispatch/composite_test.rs index 24e6c9842..e8d7b2236 100644 --- a/crates/temper-server/src/state/dispatch/composite_test.rs +++ b/crates/temper-server/src/state/dispatch/composite_test.rs @@ -318,13 +318,18 @@ fn composite_test_state() -> ServerState { specs.insert("App".to_string(), APP_IOA.to_string()); specs.insert("Blob".to_string(), BLOB_IOA.to_string()); specs.insert("Ref".to_string(), REF_IOA.to_string()); - ServerState::with_specs( + let state = ServerState::with_specs( ActorSystem::new("composite-dispatch-test"), csdl, COMPOSITE_CSDL.to_string(), specs, ) - .expect("test state should build") + .expect("test state should build"); + state + .authz + .reload_tenant_policies("default", "permit(principal, action, resource);") + .expect("composite functional test policy should parse"); + state } #[cfg(feature = "sim")] @@ -336,14 +341,19 @@ fn composite_test_state_with_store(store: SimEventStore) -> ServerState { specs.insert("App".to_string(), APP_IOA.to_string()); specs.insert("Blob".to_string(), BLOB_IOA.to_string()); specs.insert("Ref".to_string(), REF_IOA.to_string()); - ServerState::with_storage_stack( + let state = ServerState::with_storage_stack( ActorSystem::new("composite-dispatch-test"), csdl, COMPOSITE_CSDL.to_string(), specs, StorageStack::from_sim(store, None), ) - .expect("test state should build") + .expect("test state should build"); + state + .authz + .reload_tenant_policies("default", "permit(principal, action, resource);") + .expect("composite functional test policy should parse"); + state } #[tokio::test] @@ -440,7 +450,8 @@ async fn composite_sub_write_authorization_receives_action_context() { state .authz - .reload_policies( + .reload_tenant_policies( + tenant.as_str(), r#" permit( principal is Agent, @@ -476,7 +487,8 @@ async fn composite_sub_write_authorization_receives_action_context() { state .authz - .reload_policies( + .reload_tenant_policies( + tenant.as_str(), r#" permit( principal is Agent, @@ -581,19 +593,25 @@ async fn composite_app_create_sub_write_authorization_can_enforce_owner_scope() let state = composite_test_state(); let tenant = TenantId::default(); let agent = AgentContext { - security_ctx: Some(SecurityContext::from_headers(&[ - ("X-Temper-Principal-Id".to_string(), "alice".to_string()), - ( - "X-Temper-Principal-Kind".to_string(), - "customer".to_string(), - ), - ])), + security_ctx: Some(SecurityContext { + principal: temper_authz::Principal { + id: "alice".to_string(), + kind: temper_authz::PrincipalKind::Customer, + role: None, + acting_for: None, + agent_type: None, + attributes: std::collections::HashMap::new(), + }, + context_attrs: std::collections::HashMap::new(), + correlation_id: "composite-owner-scope".to_string(), + }), ..Default::default() }; state .authz - .reload_policies( + .reload_tenant_policies( + tenant.as_str(), r#" permit( principal, @@ -673,7 +691,8 @@ async fn composite_preflights_sub_write_auth_before_persisting_any_write() { state .authz - .reload_policies( + .reload_tenant_policies( + tenant.as_str(), r#" permit( principal is Agent, diff --git a/crates/temper-server/src/state/dispatch/effects.rs b/crates/temper-server/src/state/dispatch/effects.rs index 49a612641..0b587e6bd 100644 --- a/crates/temper-server/src/state/dispatch/effects.rs +++ b/crates/temper-server/src/state/dispatch/effects.rs @@ -52,6 +52,28 @@ fn dispatch_trajectory_persistence_mode() -> DispatchTrajectoryPersistenceMode { DispatchTrajectoryPersistenceMode::Background } +/// The state the action was attempted from. +/// +/// A successful dispatch appends an event whose `from_status` is exactly that +/// state, so the newest event answers the question. A rejected one appends no +/// event (`entity_actor::effects::process_action` returns `event: None` on a +/// guard or state mismatch), so the newest event belongs to the *previous* +/// successful transition and names a state the entity has already left. +/// +/// Reading it anyway hides the fault the row exists to record: a second +/// `SubmitOrder` refused in `Submitted` would be stored as attempted from +/// `Draft`, where it is legal, and every downstream reader — the conformance +/// checker first among them — would see a legal source state for an illegal +/// retry. A rejected action changes nothing, so the entity's current status is +/// where the attempt was made. +fn source_status(response: &EntityResponse) -> Option { + if response.success { + response.state.events.back().map(|e| e.from_status.clone()) + } else { + Some(response.state.status.clone()) + } +} + impl crate::state::ServerState { pub(super) fn persist_trajectory_entry_background(&self, entry: TrajectoryEntry) { debug_assert_eq!( @@ -269,7 +291,7 @@ impl crate::state::ServerState { entity_id: ctx.entity_id.to_string(), action: ctx.action.to_string(), success: response.success, - from_status: response.state.events.back().map(|e| e.from_status.clone()), + from_status: source_status(response), to_status: Some(response.state.status.clone()), error: if response.success { None @@ -289,13 +311,15 @@ impl crate::state::ServerState { source: Some(TrajectorySource::Entity), spec_governed: None, agent_type: ctx.agent_ctx.agent_type.clone(), - request_body: if response.success { - None - } else { - Some(ctx.action_params.clone()) - }, + // Captured on success as well as failure. A trajectory that + // records only what failed cannot be replayed or learned from: + // the successful action and the arguments that produced it are + // exactly the signal RL and replay consumers need. Size is bounded + // when the entry is enqueued (see `enqueue_trajectory_entry`). + request_body: Some(ctx.action_params.clone()), intent: ctx.agent_ctx.intent.clone(), matched_policy_ids: None, + capture_seq: None, }; let from_status = entry.from_status.as_deref().unwrap_or("unknown"); let to_status = entry.to_status.as_deref().unwrap_or("unknown"); @@ -437,7 +461,7 @@ impl crate::state::ServerState { entity_id: ctx.entity_id.to_string(), action: ctx.action.to_string(), success: response.success, - from_status: response.state.events.back().map(|e| e.from_status.clone()), + from_status: source_status(response), to_status: Some(response.state.status.clone()), error: response.error.clone(), agent_id: ctx.agent_ctx.agent_id.clone(), @@ -448,9 +472,13 @@ impl crate::state::ServerState { source: Some(TrajectorySource::Entity), spec_governed: None, agent_type: ctx.agent_ctx.agent_type.clone(), + // Deliberately absent, unlike the persisted trajectory entry: + // webhook payloads leave the deployment, so action arguments + // are not egressed to third-party endpoints. request_body: None, intent: ctx.agent_ctx.intent.clone(), matched_policy_ids: None, + capture_seq: None, }; let from_status = entry.from_status.as_deref().unwrap_or("unknown"); let to_status = entry.to_status.as_deref().unwrap_or("unknown"); @@ -615,7 +643,7 @@ impl crate::state::ServerState { action_params: ctx.action_params, mode: super::WasmDispatchMode::Inline, }; - match Box::pin(self.dispatch_wasm_integrations_internal(&req)).await { + match super::wasm::dispatch_wasm_integrations_boxed(self, &req).await { Ok(Some(final_response)) => { // Silent-exit regression guard: trigger integration // returned but state didn't advance. Under healthy diff --git a/crates/temper-server/src/state/dispatch/mod.rs b/crates/temper-server/src/state/dispatch/mod.rs index 42026cfde..b782aa3bd 100644 --- a/crates/temper-server/src/state/dispatch/mod.rs +++ b/crates/temper-server/src/state/dispatch/mod.rs @@ -6,6 +6,7 @@ use crate::entity_actor::EntityState; use crate::request_context::AgentContext; use temper_runtime::tenant::TenantId; use temper_wasm::{WasmAuthzContext, WasmAuthzDecision, WasmAuthzGate}; +use tokio::spawn as spawn_background_wasm_integration; // determinism-ok: external WASM side effects use tracing::Instrument; mod actions; @@ -17,6 +18,9 @@ mod effects; pub(crate) mod retry; pub(crate) mod state_timeouts; mod wasm; +pub(crate) use wasm::authorized_http_endpoint_host; +#[cfg(feature = "observe")] +pub(crate) use wasm::internal_http_capability_issuer; mod wasm_secrets; pub use state_timeouts::StateTimeoutTracker; @@ -385,7 +389,7 @@ impl crate::state::ServerState { entity_type = %entity_type, entity_id = %entity_id, ); - tokio::spawn( + spawn_background_wasm_integration( async move { // determinism-ok: async integration side-effects run outside simulation core let req = WasmDispatchRequest { @@ -443,7 +447,7 @@ mod tests { let state = test_state(); state .authz - .reload_policies("") + .reload_tenant_policies("test-tenant", "") .expect("empty policy set should parse"); let gate = state.wasm_authz_gate(); @@ -465,7 +469,8 @@ mod tests { let state = test_state(); state .authz - .reload_policies( + .reload_tenant_policies( + "test-tenant", r#" permit( principal is Agent, diff --git a/crates/temper-server/src/state/dispatch/wasm.rs b/crates/temper-server/src/state/dispatch/wasm.rs index 708869e1f..c99ed70e8 100644 --- a/crates/temper-server/src/state/dispatch/wasm.rs +++ b/crates/temper-server/src/state/dispatch/wasm.rs @@ -11,11 +11,14 @@ use crate::entity_actor::{EntityResponse, EntityState}; use crate::request_context::AgentContext; use crate::secrets::template::resolve_secret_templates; use crate::state::sim_now; +use temper_authz::{AuthenticatedRequestContext, PrincipalKind, SecurityContext}; use temper_runtime::tenant::TenantId; +use temper_wasm::host_trait::clamp_redacted_metadata_value; use temper_wasm::{ - AuthorizedWasmHost, BinaryHttpInterceptorFn, ProductionWasmHost, ProgressEmitterFn, - StreamRegistry, TextHttpInterceptorFn, WasmAuthzContext, WasmAuthzGate, WasmHost, - WasmInvocationContext, WasmResourceLimits, + AuthorizedWasmHost, BinaryHttpInterceptorFn, InternalHttpCapability, + InternalHttpCapabilityIssuerFn, ProductionWasmHost, ProgressEmitterFn, StreamRegistry, + TextHttpInterceptorFn, WasmAuthzContext, WasmAuthzGate, WasmHost, WasmInvocationContext, + WasmResourceLimits, }; use super::{ @@ -24,12 +27,116 @@ use super::{ }; use replay_inputs::{extract_trajectory_actions_from_ots, has_replay_trajectory_input}; +mod boxed; mod invocation_artifacts; mod local_tdata_host; mod replay_inputs; +pub(super) use boxed::{ + dispatch_tenant_action_core_boxed, dispatch_wasm_callback_boxed, + dispatch_wasm_integrations_boxed, +}; +use boxed::{handle_wasm_failure_boxed, invoke_and_handle_result_boxed}; use local_tdata_host::LocalTDataWasmHost; +/// Build a request-bound internal HTTP capability issuer for a non-System caller. +pub(crate) fn internal_http_capability_issuer( + state: &crate::state::ServerState, + tenant: &TenantId, + security_context: Option<&SecurityContext>, +) -> Option { + let security_context = security_context?; + if security_context.principal.kind == PrincipalKind::System { + return None; + } + let authenticated = AuthenticatedRequestContext::new(tenant.clone(), security_context.clone()); + let tenant = tenant.clone(); + let store = state.internal_invocation_credentials.clone(); + Some(Arc::new(move |method, url| { + let bearer = store + .issue_for_url(authenticated.clone(), method, url) + .map_err(|error| error.to_string())?; + InternalHttpCapability::new(bearer, tenant.to_string()) + })) +} + +/// Build the same Cedar-gated host chain for an inbound `HttpEndpoint` guest +/// that ordinary action-triggered WASM integrations receive. +/// +/// The shared HTTP stream registry is the only endpoint-specific transport +/// detail. Secret access, outbound HTTP, local TData calls, and internal HTTP +/// re-entry all use the canonical authorization components. +pub(crate) fn authorized_http_endpoint_host( + state: &crate::state::ServerState, + tenant: &TenantId, + module_name: &str, + invocation_context: &WasmInvocationContext, + http_streams: Arc, + security_context: &SecurityContext, +) -> Result, String> { + let gate = state.wasm_authz_gate(); + let authz_context = WasmAuthzContext { + tenant: tenant.to_string(), + module_name: module_name.to_string(), + agent_id: invocation_context.agent_id.clone(), + session_id: invocation_context.session_id.clone(), + entity_type: invocation_context.entity_type.clone(), + trigger_action: invocation_context.trigger_action.clone(), + }; + let bootstrap_secrets = + state.get_authorized_wasm_host_bootstrap_secrets(tenant, &*gate, &authz_context); + let gate = crate::authz::wasm_gate::bind_local_blob_endpoint( + gate, + bootstrap_secrets.get("blob_endpoint").map(String::as_str), + ); + let secret_resolver = + state.authorized_wasm_secret_resolver(tenant, Arc::clone(&gate), authz_context.clone()); + let capability_issuer = internal_http_capability_issuer(state, tenant, Some(security_context)) + .ok_or_else(|| "HttpEndpoint caller authority cannot be delegated".to_string())?; + let internal_api_url = internal_api_base_url(state); + let local_blob_interceptor = local_blob_binary_interceptor( + state.clone(), + tenant.clone(), + bootstrap_secrets.get("blob_endpoint").cloned(), + ); + let progress_emitter = progress_emitter_fn( + state.clone(), + tenant.to_string(), + invocation_context.entity_type.clone(), + invocation_context.entity_id.clone(), + module_name.to_string(), + ); + + let mut base_host = ProductionWasmHost::with_shared_streams(bootstrap_secrets, http_streams) + .with_spec_evaluator(spec_evaluator_fn()) + .with_progress_emitter(progress_emitter) + .with_internal_api_base_url(internal_api_url) + .with_internal_capability_issuer(capability_issuer) + .with_invocation_context(invocation_context.clone()) + // ARN-243: the HttpEndpoint path honours the same per-tenant LLM content + // export decision as the integration path above. + .with_llm_content_export(state.export_llm_content(tenant.as_str())); + if let Some(resolver) = secret_resolver { + base_host = base_host.with_secret_resolver(resolver); + } + if let Some(interceptor) = local_blob_interceptor { + base_host = base_host.with_binary_http_interceptor(interceptor); + } + + let production_host: Arc = Arc::new(base_host); + let local_host: Arc = Arc::new(LocalTDataWasmHost::new( + state.clone(), + tenant.clone(), + Some(security_context), + production_host, + )); + Ok(Arc::new(AuthorizedWasmHost::new( + local_host, + gate, + authz_context, + ))) +} + /// Shared context threaded through the WASM dispatch call chain. /// /// Bundles the entity reference, trigger action, agent identity, and dispatch @@ -188,20 +295,13 @@ fn local_blob_binary_interceptor( tenant: TenantId, blob_endpoint: Option, ) -> Option { - let endpoint = blob_endpoint?; - if !crate::blob_store::is_local_internal_blob_endpoint(&endpoint) { - return None; - } - - let endpoint = endpoint.trim_end_matches('/').to_string(); + let endpoint = crate::blob_store::LocalInternalBlobEndpoint::parse(&blob_endpoint?)?; Some(Arc::new(move |method, url, _headers, body| { let state = state.clone(); let tenant = tenant.clone(); let endpoint = endpoint.clone(); Box::pin(async move { - let prefix = format!("{endpoint}/"); - let blob_key = url.strip_prefix(&prefix)?; - let blob_key = blob_key.to_string(); + let blob_key = endpoint.object_key(&url)?; crate::runtime_metrics::record_blob_local_fast_path_request(&method); tracing::info!( method = %method, @@ -209,7 +309,7 @@ fn local_blob_binary_interceptor( "handling local blob request without loopback HTTP" ); - let result = match method.as_str() { + let result = match method.to_ascii_uppercase().as_str() { "PUT" => state .put_blob_object(&tenant, &blob_key, &body, None) .await @@ -229,7 +329,7 @@ fn local_blob_binary_interceptor( })) } -fn internal_api_base_url(state: &crate::state::ServerState) -> Option { +pub(crate) fn internal_api_base_url(state: &crate::state::ServerState) -> Option { std::env::var("TEMPER_API_URL") // determinism-ok: production host loopback config .ok() .map(|value| value.trim_end_matches('/').to_string()) @@ -551,16 +651,17 @@ impl crate::state::ServerState { // ADR-0046: inline-hydrate blob refs below the 128KB ceiling; defer // oversize refs into a blob_cache the WASM guest can read via // host_read_field_stream. No-op on tenants without a Turso store. + let blob_hydration_budget = crate::blobs::BlobHydrationBudget::wasm_dispatch(); let blob_cache = instrument_wasm_dispatch_phase( active_parent_span.clone(), ctx, &module_name, WASM_DISPATCH_PHASE_BLOB_REF_HYDRATION, - crate::blobs::hydrate_blob_refs_for_tenant_with_ceiling( + crate::blobs::hydrate_blob_refs_for_tenant_with_budget( self, ctx.entity_ref.tenant, &mut inv_ctx.entity_state, - crate::entity_actor::effects::DEFAULT_FIELD_INLINE_MAX, + &blob_hydration_budget, ), ) .await; @@ -582,6 +683,10 @@ impl crate::state::ServerState { ) }, ); + let gate = crate::authz::wasm_gate::bind_local_blob_endpoint( + gate, + tenant_secrets.get("blob_endpoint").map(String::as_str), + ); let secret_resolver = self.authorized_wasm_secret_resolver( ctx.entity_ref.tenant, Arc::clone(&gate), @@ -593,6 +698,7 @@ impl crate::state::ServerState { &module_name, WASM_DISPATCH_PHASE_HOST_CHAIN_BUILD, || { + let internal_api_url = internal_api_base_url(self); let local_blob_interceptor = local_blob_binary_interceptor( self.clone(), ctx.entity_ref.tenant.clone(), @@ -602,7 +708,7 @@ impl crate::state::ServerState { self.clone(), ctx.entity_ref.tenant.clone(), ctx.agent_ctx.clone(), - tenant_secrets.get("temper_api_url").cloned(), + internal_api_url.clone(), ); // Use integration config timeout for both WASM execution and HTTP client. // @@ -652,8 +758,11 @@ impl crate::state::ServerState { module_name.clone(), ); let host_invocation_context = inv_ctx.clone(); - let internal_api_key = std::env::var("TEMPER_API_KEY").ok(); // determinism-ok: production host loopback config - let internal_api_url = internal_api_base_url(self); + let internal_capability_issuer = internal_http_capability_issuer( + self, + ctx.entity_ref.tenant, + ctx.agent_ctx.security_ctx.as_ref(), + ); let mut production_host_builder = ProductionWasmHost::with_timeout(tenant_secrets, http_timeout) .with_binary_http_interceptor( @@ -663,8 +772,10 @@ impl crate::state::ServerState { .with_spec_evaluator(spec_evaluator_fn()) .with_progress_emitter(progress_emitter) .with_internal_api_base_url(internal_api_url) - .with_internal_api_key(internal_api_key) .with_invocation_context(host_invocation_context) + .with_llm_content_export( + self.export_llm_content(ctx.entity_ref.tenant.as_str()), + ) .with_text_http_interceptor( local_file_interceptor .unwrap_or_else(|| Arc::new(|_, _, _, _| Box::pin(async { None }))), @@ -673,6 +784,10 @@ impl crate::state::ServerState { current_otel_trace_id(active_span) .or_else(|| ctx.agent_ctx.trace_id.clone()), ); + if let Some(issuer) = internal_capability_issuer { + production_host_builder = + production_host_builder.with_internal_capability_issuer(issuer); + } if let Some(resolver) = secret_resolver.clone() { production_host_builder = production_host_builder.with_secret_resolver(resolver); @@ -753,7 +868,8 @@ impl crate::state::ServerState { ctx, &module_name, WASM_DISPATCH_PHASE_ENGINE_INVOKE_AND_HANDLE, - self.invoke_and_handle_result( + invoke_and_handle_result_boxed( + self, ctx, integration, &module_name, @@ -868,12 +984,12 @@ impl crate::state::ServerState { for row in rows { let data = match store - .get_ots_trajectory(&row.trajectory_id) + .get_ots_trajectory(&row.tenant, &row.trajectory_id) .await .ok() .flatten() { - Some(d) => d, + Some(document) => document.data, None => continue, }; let trajectory = match serde_json::from_str::(&data) { @@ -985,6 +1101,17 @@ impl crate::state::ServerState { ); } + // ARN-243: redact LLM content (prompt/completion/system/tool) + // from the callback params unless this tenant opted into content + // export. Stripping here covers every downstream telemetry sink + // — the span record below, `llm_call_wide_event`, + // `submit_llmobs_llm_span`, and `submit_llmobs_tool_spans` — all + // of which read from these params. Metadata (tokens, model, + // provider, finish reason, trace ids) is preserved. See ADR-0166. + redact_llm_content_params( + &mut result.callback_params, + self.export_llm_content(ctx.entity_ref.tenant.as_str()), + ); let callback_params = &result.callback_params; if should_record_gen_ai_span_attrs(integration.llm, callback_params) { @@ -1067,16 +1194,16 @@ impl crate::state::ServerState { if let Some(reason) = denial_tracker.take_denial() { let error_str = http_call_authz_denied_error(&reason); record_wasm_error_on_current_span(&error_str); - return self - .handle_wasm_failure( - ctx, - &integration.name, - module_name, - &integration.on_failure, - error_str, - result.duration_ms, - ) - .await; + return handle_wasm_failure_boxed( + self, + ctx, + &integration.name, + module_name, + &integration.on_failure, + error_str, + result.duration_ms, + ) + .await; } if integration.llm { @@ -1235,7 +1362,8 @@ impl crate::state::ServerState { // `on_failure` recovery or — when none is declared — returns // `Err` so the failure is never silently treated as success // (ADR-0152). - self.handle_wasm_failure( + handle_wasm_failure_boxed( + self, ctx, &integration.name, module_name, @@ -1286,7 +1414,8 @@ impl crate::state::ServerState { // exhaustion, or panic also leaves the integration's effect // unrealized. `handle_wasm_failure` records it and propagates // `Err` when no `on_failure` is declared (ADR-0152). - self.handle_wasm_failure( + handle_wasm_failure_boxed( + self, ctx, &integration.name, module_name, @@ -1299,11 +1428,27 @@ impl crate::state::ServerState { } } - /// Invoke a WASM module directly (not triggered by an entity action). + /// In-process `/tdata` host for `$value` / `blob_adapter`. /// - /// Used by `$value` handlers for blob operations. The WASM module controls - /// the entire blob lifecycle (auth, hashing, caching, upload/download) via - /// streaming host functions. Bytes never enter WASM memory. + /// Uses the HTTP caller. System is dropped so the guest cannot inherit + /// `system-platform:broad-permit`. + pub(crate) fn local_tdata_direct_host( + &self, + tenant: &TenantId, + production_host: Arc, + security_ctx: &SecurityContext, + ) -> Arc { + let loopback_ctx = + (security_ctx.principal.kind != PrincipalKind::System).then_some(security_ctx); + Arc::new(LocalTDataWasmHost::new( + self.clone(), + tenant.clone(), + loopback_ctx, + production_host, + )) + } + + /// Invoke a WASM module directly (not triggered by an entity action). #[allow(clippy::too_many_arguments)] pub(crate) async fn invoke_wasm_direct( &self, @@ -1311,6 +1456,7 @@ impl crate::state::ServerState { module_name: &str, mut context: WasmInvocationContext, streams: Arc>, + security_ctx: &SecurityContext, ) -> Result { if context.wasm_module.is_none() { context.wasm_module = Some(module_name.to_string()); @@ -1340,6 +1486,10 @@ impl crate::state::ServerState { }; let tenant_secrets = self.get_authorized_wasm_host_bootstrap_secrets(tenant, &*base_gate, &authz_ctx); + let base_gate = crate::authz::wasm_gate::bind_local_blob_endpoint( + base_gate, + tenant_secrets.get("blob_endpoint").map(String::as_str), + ); let secret_resolver = self.authorized_wasm_secret_resolver(tenant, Arc::clone(&base_gate), authz_ctx.clone()); let local_blob_interceptor = local_blob_binary_interceptor( @@ -1358,8 +1508,8 @@ impl crate::state::ServerState { .with_spec_evaluator(spec_evaluator_fn()) .with_progress_emitter(progress_emitter) .with_internal_api_base_url(internal_api_base_url(self)) - .with_internal_api_key(std::env::var("TEMPER_API_KEY").ok()) // determinism-ok: production host loopback config - .with_invocation_context(context.clone()); + .with_invocation_context(context.clone()) + .with_llm_content_export(self.export_llm_content(tenant.as_str())); if let Some(resolver) = secret_resolver { base_host = base_host.with_secret_resolver(resolver); } @@ -1367,12 +1517,7 @@ impl crate::state::ServerState { base_host = base_host.with_binary_http_interceptor(interceptor); } let production_host: Arc = Arc::new(base_host); - let inner: Arc = Arc::new(LocalTDataWasmHost::new( - self.clone(), - tenant.clone(), - None, - production_host, - )); + let inner = self.local_tdata_direct_host(tenant, production_host, security_ctx); let host: Arc = Arc::new(AuthorizedWasmHost::new(inner, base_gate, authz_ctx)); let limits = WasmResourceLimits::default(); @@ -1929,6 +2074,83 @@ fn strip_private_observability_params(mut params: Value) -> Value { params } +/// Callback-param keys that carry LLM *content* (prompt, completion, system +/// prompt, and tool arguments/results) rather than safe metadata. These are the +/// keys the telemetry sinks read — the span record, [`llm_call_wide_event`], +/// [`submit_llmobs_llm_span`], and [`submit_llmobs_tool_spans`] — so stripping +/// them from `callback_params` redacts content across every sink at once. +/// The gate is an allowlist over [`is_private_llm_observability_param`], so these +/// are not what enforces redaction — they are the explicit statement of which +/// params are content, used by the test that proves the allowlist drops each one. +/// See ADR-0166. +#[cfg(test)] +const LLM_CONTENT_PARAM_KEYS: [&str; 4] = [ + "_gen_ai_input_messages", + "_gen_ai_output_messages", + "_gen_ai_system_instructions", + "_dd_llmobs_tool_spans", +]; + +/// Callback-param keys the sinks record under `gen_ai.*` semantic-convention +/// names. Their values come from the guest, so a key name cannot establish that +/// the value is metadata: a module for a non-opted-in tenant that returns +/// `{"_gen_ai_model": ""}` would otherwise reach LLM +/// Observability as `gen_ai.request.model`. They are kept, but bounded — the same +/// rule the other three channels apply. See ADR-0166. +const LLM_METADATA_PARAM_KEYS: [&str; 8] = [ + "_gen_ai_provider", + "_gen_ai_model", + "_gen_ai_finish_reason", + "_gen_ai_parent_trace_id", + "_gen_ai_parent_span_id", + "_gen_ai_llm_parent_span_id", + "_gen_ai_llmobs_agent_span_id", + "_gen_ai_llmobs_workflow_span_id", +]; + +/// Redact LLM content params from `callback_params` unless the tenant has opted +/// into LLM content export. Removes only the content keys in +/// [`LLM_CONTENT_PARAM_KEYS`]; metadata is preserved. No-op when +/// `export_content` is true. See ADR-0166. +/// Whether a callback param is a private LLM-observability channel (the `_gen_ai_` +/// and `_dd_llmobs_` prefixes the telemetry sinks read) rather than ordinary +/// action output. Prefix-based so a param added later is governed by default +/// instead of silently exempt. +fn is_private_llm_observability_param(key: &str) -> bool { + // Case-insensitive, matching the normalization the other channels apply. + // Today's sinks look these up with exact lowercase names, so `_GEN_AI_prompt` + // is not exported — but it would sit in the map looking governed, waiting for + // the first sink that folds case. Cheaper to normalize than to rely on that. + let key = key.to_ascii_lowercase(); + key.starts_with("_gen_ai_") || key.starts_with("_dd_llmobs_") +} + +fn redact_llm_content_params(callback_params: &mut Value, export_content: bool) { + if export_content { + return; + } + let Some(object) = callback_params.as_object_mut() else { + return; + }; + // Allowlist, not denylist. Removing four known content keys leaves any *other* + // observability param — `_gen_ai_completion`, say — in the map for a sink + // added later to pick up. The other channels were switched to an allowlist for + // exactly this reason; this one is now consistent with them: a private + // observability param survives only if it is recognised metadata. + object.retain(|key, _| { + !is_private_llm_observability_param(key) + || LLM_METADATA_PARAM_KEYS.contains(&key.to_ascii_lowercase().as_str()) + }); + for key in LLM_METADATA_PARAM_KEYS { + let Some(Value::String(text)) = object.get_mut(key) else { + continue; + }; + if let Some(clamped) = clamp_redacted_metadata_value(text) { + *text = clamped; + } + } +} + fn integration_error_type(error: &str) -> String { let normalized = error.to_ascii_lowercase(); if normalized.contains("rate limit") { @@ -2034,431 +2256,8 @@ fn progress_emitter_fn( } #[cfg(test)] -mod tests { - use super::*; - - #[test] - fn composite_wasm_result_inherits_generated_dispatch_idempotency() { - let agent = AgentContext::for_service("version-publisher"); - - let composite_agent = agent_ctx_for_composite_wasm_result( - &agent, - Some("dispatch:default:App:app:PublishNewVersion:one"), - ); - - assert_eq!( - composite_agent.idempotency_key.as_deref(), - Some("dispatch:default:App:app:PublishNewVersion:one"), - "composite sub-writes need the parent dispatch idempotency so repeated app version updates get distinct sub-write keys" - ); - } - - #[test] - fn composite_wasm_result_preserves_caller_supplied_idempotency() { - let mut agent = AgentContext::for_service("version-publisher"); - agent.idempotency_key = Some("caller-key".to_string()); - - let composite_agent = agent_ctx_for_composite_wasm_result(&agent, Some("generated-key")); - - assert_eq!( - composite_agent.idempotency_key.as_deref(), - Some("caller-key"), - "caller idempotency remains authoritative for retries" - ); - } - - #[test] - fn strips_private_llm_observability_params_before_callback_dispatch() { - let params = json!({ - "provider_response_file_id": "file-123", - "input_tokens": 10, - "_gen_ai_input_messages": "[{\"role\":\"user\"}]", - "_gen_ai_output_messages": "[{\"role\":\"assistant\"}]", - "_gen_ai_system_instructions": "system", - "_gen_ai_provider": "anthropic", - "_gen_ai_model": "claude-sonnet-4-6", - "_gen_ai_finish_reason": "end_turn", - "_gen_ai_llm_parent_span_id": "parent-span-private", - "_dd_llmobs_tool_spans": "[]", - "gen_ai_parent_trace_id": "trace-public", - "gen_ai_llm_parent_span_id": "parent-span-public", - }); - - let stripped = strip_private_observability_params(params); - - assert_eq!(stripped["provider_response_file_id"], "file-123"); - assert_eq!(stripped["input_tokens"], 10); - assert_eq!(stripped["gen_ai_parent_trace_id"], "trace-public"); - assert_eq!(stripped["gen_ai_llm_parent_span_id"], "parent-span-public"); - assert!(stripped.get("_gen_ai_input_messages").is_none()); - assert!(stripped.get("_gen_ai_output_messages").is_none()); - assert!(stripped.get("_gen_ai_system_instructions").is_none()); - assert!(stripped.get("_gen_ai_provider").is_none()); - assert!(stripped.get("_gen_ai_model").is_none()); - assert!(stripped.get("_gen_ai_finish_reason").is_none()); - assert!(stripped.get("_gen_ai_llm_parent_span_id").is_none()); - assert!(stripped.get("_dd_llmobs_tool_spans").is_none()); - } - - #[test] - fn gen_ai_span_attrs_are_recorded_only_for_llm_integrations() { - let params = json!({ - "input_tokens": 10, - "output_tokens": 20, - "_gen_ai_input_messages": "[{\"role\":\"user\"}]", - "_gen_ai_output_messages": "[{\"role\":\"assistant\"}]", - "_gen_ai_provider": "openai", - "_gen_ai_model": "gpt-5.4", - }); - - assert!(should_record_gen_ai_span_attrs(true, ¶ms)); - assert!(!should_record_gen_ai_span_attrs(false, ¶ms)); - } - - #[test] - fn llmobs_service_name_prefers_runtime_service_identity() { - unsafe { - std::env::set_var("DD_SERVICE", "temperpaw"); - std::env::remove_var("OTEL_SERVICE_NAME"); - } - assert_eq!(llmobs_service_name(), "temperpaw"); - - unsafe { - std::env::remove_var("DD_SERVICE"); - std::env::set_var("OTEL_SERVICE_NAME", "temper-agent"); - } - assert_eq!(llmobs_service_name(), "temper-agent"); - - unsafe { - std::env::remove_var("DD_SERVICE"); - std::env::remove_var("OTEL_SERVICE_NAME"); - } - assert_eq!(llmobs_service_name(), "temper-platform"); - } - - #[test] - fn llm_model_for_observability_prefers_callback_model() { - let entity_state = EntityState { - entity_type: "Session".to_string(), - entity_id: "session-1".to_string(), - status: "CallingProvider".to_string(), - item_count: 0, - counters: std::collections::BTreeMap::new(), - booleans: std::collections::BTreeMap::new(), - lists: std::collections::BTreeMap::new(), - fields: json!({"model": "claude-sonnet-4-6"}), - events: std::collections::VecDeque::new(), - total_event_count: 0, - events_since_snapshot: 0, - last_snapshot_sequence_nr: 0, - sequence_nr: 0, - processed_idempotency_keys: std::collections::BTreeMap::new(), - }; - let callback_params = json!({ - "_gen_ai_model": "gpt-5.4", - }); - - assert_eq!( - llm_model_for_observability(&entity_state, &callback_params), - "gpt-5.4" - ); - } - - #[test] - fn parse_internal_file_value_request_matches_only_value_paths() { - assert_eq!( - parse_internal_file_value_request( - "http://127.0.0.1:3467", - "http://127.0.0.1:3467/tdata/Files('fl-123')/$value" - ) - .as_deref(), - Some("fl-123") - ); - assert!( - parse_internal_file_value_request( - "http://127.0.0.1:3467", - "http://127.0.0.1:3467/tdata/Files('fl-123')" - ) - .is_none() - ); - } - - #[test] - fn llm_root_span_stays_on_active_trace() { - use opentelemetry::trace::TracerProvider as _; - use opentelemetry_sdk::trace::SdkTracerProvider; - use tracing_subscriber::prelude::*; - - let tracer_provider = SdkTracerProvider::builder().build(); - let subscriber = tracing_subscriber::registry().with( - tracing_opentelemetry::layer() - .with_tracer(tracer_provider.tracer("temper-server-llm-root-test")), - ); - let _subscriber_guard = tracing::subscriber::set_default(subscriber); - - let tenant = TenantId::default(); - let entity_state = EntityState { - entity_type: "Session".to_string(), - entity_id: "ss-1".to_string(), - status: "CallingProvider".to_string(), - item_count: 0, - counters: std::collections::BTreeMap::new(), - booleans: std::collections::BTreeMap::new(), - lists: std::collections::BTreeMap::new(), - fields: json!({"provider": "openai", "model": "gpt-5.4"}), - events: std::collections::VecDeque::new(), - total_event_count: 0, - events_since_snapshot: 0, - last_snapshot_sequence_nr: 0, - sequence_nr: 0, - processed_idempotency_keys: std::collections::BTreeMap::new(), - }; - let integration = temper_spec::automaton::Integration { - name: "provider_caller".to_string(), - trigger: "call_provider".to_string(), - integration_type: "wasm".to_string(), - module: Some("provider_caller".to_string()), - config: std::collections::BTreeMap::new(), - on_success: None, - on_failure: None, - llm: true, - }; - let agent_ctx = AgentContext { - session_id: Some("ss-1".to_string()), - ..AgentContext::default() - }; - - let parent = tracing::info_span!("dispatch.dispatch_tenant_action_core"); - let expected_trace_id = parent.in_scope(|| { - tracing::Span::current() - .context() - .span() - .span_context() - .trace_id() - .to_string() - }); - let (llm_trace_id, has_llmobs_auto_conversion_opt_out) = parent.in_scope(|| { - let ctx = WasmDispatchCtx { - entity_ref: WasmEntityRef { - tenant: &tenant, - entity_type: "Session", - entity_id: "ss-1", - }, - action: "ContextReady", - agent_ctx: &agent_ctx, - dispatch_idempotency_key: None, - mode: WasmDispatchMode::Inline, - }; - let span = build_llm_root_span(&ctx, &integration, &entity_state, "provider_caller"); - let has_opt_out = span - .metadata() - .map(|metadata| { - metadata - .fields() - .iter() - .any(|field| field.name() == "dd_llmobs_enabled") - }) - .unwrap_or(false); - ( - span.context().span().span_context().trace_id().to_string(), - has_opt_out, - ) - }); - - assert_eq!(llm_trace_id, expected_trace_id); - assert!( - has_llmobs_auto_conversion_opt_out, - "root LLM OTel span must opt out of Datadog auto LLMObs conversion" - ); - } - - #[test] - fn llm_parent_context_records_llm_span_and_dispatch_parent_ids() { - use opentelemetry::trace::TracerProvider as _; - use opentelemetry_sdk::trace::SdkTracerProvider; - use tracing_subscriber::prelude::*; - - let tracer_provider = SdkTracerProvider::builder().build(); - let subscriber = tracing_subscriber::registry().with( - tracing_opentelemetry::layer() - .with_tracer(tracer_provider.tracer("temper-server-llm-parent-test")), - ); - let _subscriber_guard = tracing::subscriber::set_default(subscriber); - - let dispatch_parent = tracing::info_span!("dispatch.dispatch_tenant_action_core"); - let (expected_trace_id, expected_parent_span_id) = dispatch_parent.in_scope(|| { - let span_context = tracing::Span::current() - .context() - .span() - .span_context() - .clone(); - ( - span_context.trace_id().to_string(), - span_context.span_id().to_string(), - ) - }); - - let mut callback_params = json!({}); - let entity_state = EntityState { - entity_type: "Session".to_string(), - entity_id: "session-1".to_string(), - status: "CallingProvider".to_string(), - item_count: 0, - counters: std::collections::BTreeMap::new(), - booleans: std::collections::BTreeMap::new(), - lists: std::collections::BTreeMap::new(), - fields: json!({}), - events: std::collections::VecDeque::new(), - total_event_count: 0, - events_since_snapshot: 0, - last_snapshot_sequence_nr: 0, - sequence_nr: 0, - processed_idempotency_keys: std::collections::BTreeMap::new(), - }; - let (llm_trace_id, llm_span_id) = dispatch_parent.in_scope(|| { - let llm_span = tracing::info_span!("llm_caller.trace"); - let span_context = llm_span.context().span().span_context().clone(); - attach_llm_parent_context( - &llm_span, - Some(&expected_parent_span_id), - &entity_state, - "session-1", - 1_234, - &mut callback_params, - ); - ( - span_context.trace_id().to_string(), - span_context.span_id().to_string(), - ) - }); - - assert_eq!(llm_trace_id, expected_trace_id); - assert_ne!(llm_span_id, expected_parent_span_id); - assert_eq!( - callback_params["_gen_ai_parent_trace_id"], - expected_trace_id - ); - assert_eq!(callback_params["_gen_ai_parent_span_id"], llm_span_id); - assert_eq!( - callback_params["_gen_ai_llm_parent_span_id"], - expected_parent_span_id - ); - assert_eq!( - callback_params["gen_ai_llm_parent_span_id"], - expected_parent_span_id - ); - let expected_agent_span_id = temper_observe::llmobs_api::derive_span_id(&format!( - "{expected_trace_id}:session-1:agent" - )); - assert_eq!( - callback_params["_gen_ai_llmobs_agent_span_id"], - expected_agent_span_id - ); - assert_eq!( - callback_params["llmobs_agent_span_id"], - expected_agent_span_id - ); - assert_ne!( - callback_params["_gen_ai_llmobs_agent_span_id"], - expected_parent_span_id - ); - assert!( - callback_params["_gen_ai_llmobs_workflow_span_id"] - .as_str() - .is_some_and(|workflow_span_id| !workflow_span_id.is_empty() - && workflow_span_id != expected_parent_span_id - && workflow_span_id != llm_span_id) - ); - assert_eq!( - callback_params["llmobs_workflow_span_id"], - callback_params["_gen_ai_llmobs_workflow_span_id"] - ); - assert!( - callback_params["llmobs_agent_start_ns"] - .as_u64() - .is_some_and(|start_ns| start_ns > 0) - ); - } - - #[test] - fn llm_parent_context_reuses_existing_llmobs_agent_root() { - use opentelemetry::trace::TracerProvider as _; - use opentelemetry_sdk::trace::SdkTracerProvider; - use tracing_subscriber::prelude::*; - - let tracer_provider = SdkTracerProvider::builder().build(); - let subscriber = tracing_subscriber::registry().with( - tracing_opentelemetry::layer() - .with_tracer(tracer_provider.tracer("temper-server-llm-parent-reuse-test")), - ); - let _subscriber_guard = tracing::subscriber::set_default(subscriber); - - let entity_state = EntityState { - entity_type: "Session".to_string(), - entity_id: "session-1".to_string(), - status: "CallingProvider".to_string(), - item_count: 0, - counters: std::collections::BTreeMap::new(), - booleans: std::collections::BTreeMap::new(), - lists: std::collections::BTreeMap::new(), - fields: json!({ - "llmobs_agent_span_id": "stable-agent-root", - "llmobs_agent_start_ns": 12345_u64, - }), - events: std::collections::VecDeque::new(), - total_event_count: 0, - events_since_snapshot: 0, - last_snapshot_sequence_nr: 0, - sequence_nr: 0, - processed_idempotency_keys: std::collections::BTreeMap::new(), - }; - - let mut callback_params = json!({}); - let llm_span = tracing::info_span!("llm_caller.trace"); - attach_llm_parent_context( - &llm_span, - Some("turn-parent-span"), - &entity_state, - "session-1", - 99, - &mut callback_params, - ); - - assert_eq!( - callback_params["_gen_ai_llmobs_agent_span_id"], - "stable-agent-root" - ); - assert_eq!(callback_params["llmobs_agent_span_id"], "stable-agent-root"); - assert_eq!(callback_params["_gen_ai_llmobs_agent_start_ns"], 12345_u64); - assert_eq!(callback_params["llmobs_agent_start_ns"], 12345_u64); - } - - #[test] - fn llmobs_tool_parent_prefers_workflow_span_id() { - let entity_state = EntityState { - entity_type: "Session".to_string(), - entity_id: "ss-1".to_string(), - status: "CallingTools".to_string(), - item_count: 0, - counters: std::collections::BTreeMap::new(), - booleans: std::collections::BTreeMap::new(), - lists: std::collections::BTreeMap::new(), - fields: json!({ - "gen_ai_parent_trace_id": "trace-1", - "gen_ai_parent_span_id": "legacy-llm-parent", - "llmobs_workflow_span_id": "workflow-parent", - }), - events: std::collections::VecDeque::new(), - total_event_count: 0, - events_since_snapshot: 0, - last_snapshot_sequence_nr: 0, - sequence_nr: 0, - processed_idempotency_keys: std::collections::BTreeMap::new(), - }; - - assert_eq!( - llmobs_tool_trace_and_parent(&entity_state, &json!({})), - Some(("trace-1".to_string(), "workflow-parent".to_string())) - ); - } -} +#[path = "wasm/llm_redaction_test.rs"] +mod llm_redaction_test; +#[cfg(test)] +#[path = "wasm/wasm_test.rs"] +mod tests; diff --git a/crates/temper-server/src/state/dispatch/wasm/boxed.rs b/crates/temper-server/src/state/dispatch/wasm/boxed.rs new file mode 100644 index 000000000..82acc8c3f --- /dev/null +++ b/crates/temper-server/src/state/dispatch/wasm/boxed.rs @@ -0,0 +1,134 @@ +use futures_util::FutureExt; +use futures_util::future::BoxFuture; + +use std::sync::Arc; + +use super::{ + HttpCallAuthzDenialTracker, WasmDispatchCtx, WasmDispatchMode, WasmDispatchRequest, + WasmEntityRef, +}; +use crate::entity_actor::{EntityResponse, EntityState}; +use crate::request_context::AgentContext; +use crate::state::dispatch::DispatchError; +use temper_runtime::tenant::TenantId; +use temper_wasm::{WasmHost, WasmInvocationContext, WasmResourceLimits}; + +/// Heap-allocate the complete inline WASM integration state machine. +/// +/// Keeping this adapter in a child module gives Rust an explicit opaque-type +/// boundary for the callback path that can recursively enter action dispatch. +/// The underlying method retains its tracing instrumentation and semantics. +pub(in crate::state::dispatch) fn dispatch_wasm_integrations_boxed<'a>( + state: &'a crate::state::ServerState, + request: &'a WasmDispatchRequest<'a>, +) -> BoxFuture<'a, Result, String>> { + state.dispatch_wasm_integrations_internal(request).boxed() +} + +/// Heap-allocate WASM invocation and result handling behind an opaque boundary. +#[expect( + clippy::too_many_arguments, + reason = "opaque boundary mirrors the existing instrumented async signature" +)] +pub(in crate::state::dispatch::wasm) fn invoke_and_handle_result_boxed<'a>( + state: &'a crate::state::ServerState, + ctx: &'a WasmDispatchCtx<'a>, + integration: &'a temper_spec::automaton::Integration, + module_name: &'a str, + hash: &'a str, + entity_state: &'a EntityState, + invocation_context: WasmInvocationContext, + host: Arc, + limits: &'a WasmResourceLimits, + denial_tracker: &'a HttpCallAuthzDenialTracker, + blob_cache: std::collections::BTreeMap>, + llm_parent_span_id: Option<&'a str>, +) -> BoxFuture<'a, Result, String>> { + state + .invoke_and_handle_result( + ctx, + integration, + module_name, + hash, + entity_state, + invocation_context, + host, + limits, + denial_tracker, + blob_cache, + llm_parent_span_id, + ) + .boxed() +} + +/// Heap-allocate failure recording and its optional recovery callback. +pub(in crate::state::dispatch::wasm) fn handle_wasm_failure_boxed<'a>( + state: &'a crate::state::ServerState, + ctx: &'a WasmDispatchCtx<'a>, + integration_name: &'a str, + module_name: &'a str, + on_failure: &'a Option, + error: String, + duration_ms: u64, +) -> BoxFuture<'a, Result, String>> { + state + .handle_wasm_failure( + ctx, + integration_name, + module_name, + on_failure, + error, + duration_ms, + ) + .boxed() +} + +/// Heap-allocate an inline or background WASM callback dispatch. +pub(in crate::state::dispatch) fn dispatch_wasm_callback_boxed<'a>( + state: &'a crate::state::ServerState, + entity_ref: WasmEntityRef<'a>, + callback_action: &'a str, + callback_params: serde_json::Value, + agent_context: &'a AgentContext, + mode: WasmDispatchMode, +) -> BoxFuture<'a, Result, String>> { + state + .dispatch_wasm_callback( + entity_ref, + callback_action, + callback_params, + agent_context, + mode, + ) + .boxed() +} + +/// Heap-allocate recursive core action dispatch from a WASM callback. +#[expect( + clippy::too_many_arguments, + reason = "opaque boundary mirrors the existing instrumented async signature" +)] +pub(in crate::state::dispatch) fn dispatch_tenant_action_core_boxed<'a>( + state: &'a crate::state::ServerState, + tenant: &'a TenantId, + entity_type: &'a str, + entity_id: &'a str, + action: &'a str, + params: serde_json::Value, + agent_context: &'a AgentContext, + await_integration: bool, + expected_authorization_precondition: Option, +) -> BoxFuture<'a, Result> { + state + .dispatch_tenant_action_core( + tenant, + entity_type, + entity_id, + action, + params, + agent_context, + await_integration, + expected_authorization_precondition, + ) + .boxed() +} diff --git a/crates/temper-server/src/state/dispatch/wasm/invocation_artifacts.rs b/crates/temper-server/src/state/dispatch/wasm/invocation_artifacts.rs index 5fef0ebac..18ca25c8c 100644 --- a/crates/temper-server/src/state/dispatch/wasm/invocation_artifacts.rs +++ b/crates/temper-server/src/state/dispatch/wasm/invocation_artifacts.rs @@ -130,9 +130,15 @@ impl crate::state::ServerState { params["decision_id"] = serde_json::json!(did); params["authz_denied"] = serde_json::json!(true); } - return self - .dispatch_wasm_callback(ctx.entity_ref, cb, params, ctx.agent_ctx, ctx.mode) - .await; + return super::dispatch_wasm_callback_boxed( + self, + ctx.entity_ref, + cb, + params, + ctx.agent_ctx, + ctx.mode, + ) + .await; } // No declared recovery: propagate the failure instead of swallowing it @@ -158,18 +164,19 @@ impl crate::state::ServerState { // its own WASM trigger; returning before that nested trigger // commits lets concurrent requests observe stale detailed // fields while counters advance. - let resp = self - .dispatch_tenant_action_core( - entity_ref.tenant, - entity_ref.entity_type, - entity_ref.entity_id, - callback_action, - callback_params, - agent_ctx, - true, - ) - .await - .map_err(|e| e.to_string())?; + let resp = super::dispatch_tenant_action_core_boxed( + self, + entity_ref.tenant, + entity_ref.entity_type, + entity_ref.entity_id, + callback_action, + callback_params, + agent_ctx, + true, + None, + ) + .await + .map_err(|e| e.to_string())?; Ok(Some(resp)) } WasmDispatchMode::Background => { @@ -263,17 +270,25 @@ impl crate::state::ServerState { from_status: None, to_status: None, error: Some(error_str.to_string()), - agent_id: None, - session_id: None, + // The denial belongs to the agent whose dispatch triggered the + // WASM call. Dropping that identity left every WASM denial + // unattributable in the trajectory stream. + agent_id: agent_ctx.agent_id.clone(), + session_id: agent_ctx.session_id.clone(), authz_denied: Some(true), denied_resource: Some(integration_name.to_string()), denied_module: Some(module_name.to_string()), source: Some(TrajectorySource::Authz), spec_governed: None, - agent_type: None, - request_body: None, - intent: None, + agent_type: agent_ctx.agent_type.clone(), + request_body: Some(serde_json::json!({ + "integration": integration_name, + "module": module_name, + "trigger_action": trigger_action, + })), + intent: agent_ctx.intent.clone(), matched_policy_ids: None, + capture_seq: None, }; tracing::info!( tenant = %traj.tenant, @@ -286,6 +301,10 @@ impl crate::state::ServerState { error = ?traj.error, source = ?traj.source, authz_denied = ?traj.authz_denied, + agent_id = traj.agent_id.as_deref().unwrap_or(""), + session_id = traj.session_id.as_deref().unwrap_or(""), + agent_type = traj.agent_type.as_deref().unwrap_or(""), + intent = traj.intent.as_deref().unwrap_or(""), "trajectory.entry" ); if !traj.success { diff --git a/crates/temper-server/src/state/dispatch/wasm/llm_redaction_test.rs b/crates/temper-server/src/state/dispatch/wasm/llm_redaction_test.rs new file mode 100644 index 000000000..bee96639b --- /dev/null +++ b/crates/temper-server/src/state/dispatch/wasm/llm_redaction_test.rs @@ -0,0 +1,186 @@ +//! ADR-0166 tests for the dispatch callback-param gate. +//! +//! Split from `wasm_test.rs`: these cover one thing — which private LLM +//! observability params survive for a tenant that has not opted into content +//! export, and how their values are bounded. + +use super::*; + +#[test] +fn strips_private_llm_observability_params_before_callback_dispatch() { + let params = json!({ + "provider_response_file_id": "file-123", + "input_tokens": 10, + "_gen_ai_input_messages": "[{\"role\":\"user\"}]", + "_gen_ai_output_messages": "[{\"role\":\"assistant\"}]", + "_gen_ai_system_instructions": "system", + "_gen_ai_provider": "anthropic", + "_gen_ai_model": "claude-sonnet-4-6", + "_gen_ai_finish_reason": "end_turn", + "_gen_ai_llm_parent_span_id": "parent-span-private", + "_dd_llmobs_tool_spans": "[]", + "gen_ai_parent_trace_id": "trace-public", + "gen_ai_llm_parent_span_id": "parent-span-public", + }); + + let stripped = strip_private_observability_params(params); + + assert_eq!(stripped["provider_response_file_id"], "file-123"); + assert_eq!(stripped["input_tokens"], 10); + assert_eq!(stripped["gen_ai_parent_trace_id"], "trace-public"); + assert_eq!(stripped["gen_ai_llm_parent_span_id"], "parent-span-public"); + assert!(stripped.get("_gen_ai_input_messages").is_none()); + assert!(stripped.get("_gen_ai_output_messages").is_none()); + assert!(stripped.get("_gen_ai_system_instructions").is_none()); + assert!(stripped.get("_gen_ai_provider").is_none()); + assert!(stripped.get("_gen_ai_model").is_none()); + assert!(stripped.get("_gen_ai_finish_reason").is_none()); + assert!(stripped.get("_gen_ai_llm_parent_span_id").is_none()); + assert!(stripped.get("_dd_llmobs_tool_spans").is_none()); +} + +/// A key name cannot make an untrusted value into metadata. The sinks record +/// `_gen_ai_model` as `gen_ai.request.model`, which LLM Observability reads as +/// LLM data — so a module that returns its prompt under that key would export +/// content for a non-opted-in tenant under a semantic-convention name. The +/// other three channels bound these values; so must this one. +/// The dispatch gate is an allowlist: an unrecognised private observability +/// param must not survive just because it is not on a content list. A denylist +/// here would leave `_gen_ai_completion` in the map for any sink added later. +#[test] +fn unrecognized_private_observability_params_do_not_survive() { + let mut params = json!({ + "_gen_ai_completion": "SECRET COMPLETION", + "_gen_ai_prompt": "SECRET PROMPT", + "_dd_llmobs_something_new": "SECRET", + "_gen_ai_model": "claude-opus-4-8", + "_gen_ai_llm_parent_span_id": "parent-span", + "output": "ordinary action output", + }); + + redact_llm_content_params(&mut params, false); + + for dropped in [ + "_gen_ai_completion", + "_gen_ai_prompt", + "_dd_llmobs_something_new", + ] { + assert!( + params.get(dropped).is_none(), + "`{dropped}` must not survive an allowlist; got {params:?}" + ); + } + // Every key the content list names must also be gone. + for content_key in LLM_CONTENT_PARAM_KEYS { + assert!( + params.get(content_key).is_none(), + "{content_key} must be dropped" + ); + } + // Recognised metadata and ordinary output are untouched. + assert_eq!( + params.get("_gen_ai_model").and_then(Value::as_str), + Some("claude-opus-4-8") + ); + assert_eq!( + params + .get("_gen_ai_llm_parent_span_id") + .and_then(Value::as_str), + Some("parent-span"), + "trace-correlation ids must survive or LLMObs spans lose their parent" + ); + assert_eq!( + params.get("output").and_then(Value::as_str), + Some("ordinary action output"), + "non-observability params are not this gate's business" + ); +} + +#[test] +fn clamps_guest_supplied_llm_metadata_values_for_non_opted_in_tenant() { + use temper_wasm::host_trait::MAX_REDACTED_LLM_METADATA_VALUE_BYTES; + let prompt = "P".repeat(4096); + let mut params = json!({ + "_gen_ai_model": prompt, + "_gen_ai_provider": prompt, + "_gen_ai_finish_reason": prompt, + "input_tokens": 10, + }); + + redact_llm_content_params(&mut params, false); + + for key in ["_gen_ai_model", "_gen_ai_provider", "_gen_ai_finish_reason"] { + let value = params + .get(key) + .and_then(Value::as_str) + .unwrap_or_else(|| panic!("{key} should survive, bounded")); + assert!( + value.len() <= MAX_REDACTED_LLM_METADATA_VALUE_BYTES, + "{key} must be clamped, got {} bytes", + value.len() + ); + } + assert_eq!(params.get("input_tokens").and_then(Value::as_u64), Some(10)); + + // An opted-in tenant is not clamped. + let mut exported = json!({ "_gen_ai_model": prompt }); + redact_llm_content_params(&mut exported, true); + assert_eq!( + exported.get("_gen_ai_model").and_then(Value::as_str), + Some(prompt.as_str()) + ); +} + +#[test] +fn redacts_llm_content_params_for_non_opted_in_tenant() { + let base = json!({ + "input_tokens": 10, + "output_tokens": 20, + "_gen_ai_provider": "anthropic", + "_gen_ai_model": "claude-sonnet-4-6", + "_gen_ai_finish_reason": "end_turn", + "_gen_ai_llm_parent_span_id": "parent-span", + "_gen_ai_input_messages": "[{\"role\":\"user\",\"content\":\"SECRET PROMPT\"}]", + "_gen_ai_output_messages": "[{\"role\":\"assistant\",\"content\":\"SECRET REPLY\"}]", + "_gen_ai_system_instructions": "SECRET SYSTEM", + "_dd_llmobs_tool_spans": "[{\"arguments\":\"SECRET ARGS\",\"result\":\"SECRET RESULT\"}]", + }); + + // Non-opted-in tenant: content stripped, metadata preserved. + let mut redacted = base.clone(); + redact_llm_content_params(&mut redacted, false); + assert!( + redacted.get("_gen_ai_input_messages").is_none(), + "prompt must be redacted" + ); + assert!( + redacted.get("_gen_ai_output_messages").is_none(), + "completion must be redacted" + ); + assert!( + redacted.get("_gen_ai_system_instructions").is_none(), + "system prompt must be redacted" + ); + assert!( + redacted.get("_dd_llmobs_tool_spans").is_none(), + "tool content must be redacted" + ); + assert_eq!(redacted["input_tokens"], 10); + assert_eq!(redacted["output_tokens"], 20); + assert_eq!(redacted["_gen_ai_provider"], "anthropic"); + assert_eq!(redacted["_gen_ai_model"], "claude-sonnet-4-6"); + assert_eq!(redacted["_gen_ai_finish_reason"], "end_turn"); + assert_eq!(redacted["_gen_ai_llm_parent_span_id"], "parent-span"); + + // Opted-in tenant: content preserved. + let mut exported = base.clone(); + redact_llm_content_params(&mut exported, true); + assert_eq!( + exported["_gen_ai_input_messages"], + "[{\"role\":\"user\",\"content\":\"SECRET PROMPT\"}]" + ); + assert_eq!( + exported["_dd_llmobs_tool_spans"], + "[{\"arguments\":\"SECRET ARGS\",\"result\":\"SECRET RESULT\"}]" + ); +} diff --git a/crates/temper-server/src/state/dispatch/wasm/local_tdata_host.rs b/crates/temper-server/src/state/dispatch/wasm/local_tdata_host.rs index b73165f45..8d6e599e3 100644 --- a/crates/temper-server/src/state/dispatch/wasm/local_tdata_host.rs +++ b/crates/temper-server/src/state/dispatch/wasm/local_tdata_host.rs @@ -3,11 +3,11 @@ use std::sync::Arc; use async_trait::async_trait; use axum::body::{Bytes, to_bytes}; -use axum::extract::{Path, Query, State}; +use axum::extract::{Extension, Path, Query, State}; use axum::http::{HeaderMap, HeaderName, HeaderValue}; use axum::response::IntoResponse; use reqwest::Url; -use temper_authz::{PrincipalKind, SecurityContext}; +use temper_authz::{AuthenticatedRequestContext, SecurityContext}; use temper_runtime::tenant::TenantId; use temper_wasm::WasmHost; use temper_wasm::http_stream::{ @@ -25,8 +25,7 @@ const LOCAL_TDATA_RESPONSE_LIMIT_BYTES: usize = 64 * 1024 * 1024; /// through the same OData handlers as external HTTP traffic. pub(super) struct LocalTDataWasmHost { state: ServerState, - tenant: TenantId, - inherited_headers: Vec<(String, String)>, + authenticated: Option, delegate: Arc, } @@ -40,10 +39,9 @@ impl LocalTDataWasmHost { ) -> Self { Self { state, - tenant, - inherited_headers: security_ctx - .map(security_context_headers) - .unwrap_or_default(), + authenticated: security_ctx + .cloned() + .map(|security_ctx| AuthenticatedRequestContext::new(tenant, security_ctx)), delegate, } } @@ -64,7 +62,10 @@ impl LocalTDataWasmHost { if !matches!(method_upper.as_str(), "GET" | "POST") { return Ok(None); } - let headers = header_map(headers, &self.tenant, &self.inherited_headers); + let Some(authenticated) = self.authenticated.clone() else { + return Ok(None); + }; + let headers = header_map(headers); let path_for_span = request.path.clone(); let span = tracing::info_span!( "wasm.local_tdata_http_call", @@ -78,7 +79,7 @@ impl LocalTDataWasmHost { match method_upper.as_str() { "GET" => crate::odata::handle_odata_get( State(self.state.clone()), - None, + Some(Extension(authenticated.clone())), headers, Path(request.path), Query(request.query), @@ -87,7 +88,7 @@ impl LocalTDataWasmHost { .into_response(), "POST" => crate::odata::handle_odata_post( State(self.state.clone()), - None, + Some(Extension(authenticated)), headers, Path(request.path), Query(request.query), @@ -111,6 +112,15 @@ impl LocalTDataWasmHost { #[async_trait] impl WasmHost for LocalTDataWasmHost { + /// Forward the delegate's per-tenant LLM content decision (ADR-0166). The + /// production stack is `AuthorizedWasmHost(LocalTDataWasmHost(ProductionWasmHost))`, + /// and only the innermost host holds the flag, so a wrapper that does not + /// forward makes the engine read the trait default and redact even for a + /// tenant that opted in. + fn exports_llm_content(&self) -> bool { + self.delegate.exports_llm_content() + } + async fn http_call( &self, method: &str, @@ -281,98 +291,22 @@ fn is_file_value_path(path: &str) -> bool { path.starts_with("Files('") && path.ends_with("')/$value") } -fn header_map( - headers: &[(String, String)], - tenant: &TenantId, - inherited_headers: &[(String, String)], -) -> HeaderMap { +fn header_map(headers: &[(String, String)]) -> HeaderMap { let mut map = HeaderMap::new(); - for (name, value) in inherited_headers { - insert_header_if_absent(&mut map, name, value); - } for (name, value) in headers { - insert_header(&mut map, name, value); - } - if !map.contains_key("x-tenant-id") { - let value = - HeaderValue::from_str(tenant.as_str()).expect("TenantId is a valid HTTP header value"); - map.insert(HeaderName::from_static("x-tenant-id"), value); - } - map -} - -fn insert_header(map: &mut HeaderMap, name: &str, value: &str) { - let Ok(name) = HeaderName::from_bytes(name.as_bytes()) else { - return; - }; - let Ok(value) = HeaderValue::from_str(value) else { - return; - }; - map.insert(name, value); -} - -fn insert_header_if_absent(map: &mut HeaderMap, name: &str, value: &str) { - let Ok(name) = HeaderName::from_bytes(name.as_bytes()) else { - return; - }; - if map.contains_key(&name) { - return; - } - let Ok(value) = HeaderValue::from_str(value) else { - return; - }; - map.insert(name, value); -} - -fn security_context_headers(ctx: &SecurityContext) -> Vec<(String, String)> { - let mut headers = Vec::new(); - headers.push(( - "x-temper-principal-id".to_string(), - ctx.principal.id.clone(), - )); - let kind = match ctx.principal.kind { - PrincipalKind::Customer => "customer", - PrincipalKind::Agent => "agent", - PrincipalKind::Admin => "admin", - // `SecurityContext::from_headers` intentionally rejects system from - // external headers. Local TData inheritance keeps that invariant and - // relies on explicit policies/service principals for system paths. - PrincipalKind::System => "customer", - }; - headers.push(("x-temper-principal-kind".to_string(), kind.to_string())); - if let Some(role) = &ctx.principal.role { - headers.push(("x-temper-agent-role".to_string(), role.clone())); - } - if let Some(agent_type) = &ctx.principal.agent_type { - headers.push(("x-temper-agent-type".to_string(), agent_type.clone())); - } - if let Some(scopes) = ctx - .principal - .attributes - .get("scopes") - .and_then(|value| value.as_array()) - { - let scopes = scopes - .iter() - .filter_map(|value| value.as_str()) - .collect::>() - .join(","); - if !scopes.is_empty() { - headers.push(("x-temper-principal-scopes".to_string(), scopes)); + let Ok(parsed_name) = HeaderName::from_bytes(name.as_bytes()) else { + continue; + }; + if parsed_name == HeaderName::from_static("x-tenant-id") + || crate::authz::edge::is_caller_authority_header(&parsed_name) + { + continue; + } + if let Ok(value) = HeaderValue::from_str(value) { + map.insert(parsed_name, value); } } - if let Some(action_context) = ctx - .principal - .attributes - .get("action_context") - .and_then(|value| value.as_str()) - { - headers.push(( - "x-temper-action-context".to_string(), - action_context.to_string(), - )); - } - headers + map } #[cfg(test)] diff --git a/crates/temper-server/src/state/dispatch/wasm/local_tdata_host_test.rs b/crates/temper-server/src/state/dispatch/wasm/local_tdata_host_test.rs index 471589f09..bb33891db 100644 --- a/crates/temper-server/src/state/dispatch/wasm/local_tdata_host_test.rs +++ b/crates/temper-server/src/state/dispatch/wasm/local_tdata_host_test.rs @@ -171,6 +171,35 @@ fn test_state() -> ServerState { .expect("test state should build") } +fn permit_agents(state: &ServerState) { + state + .authz + .reload_tenant_policies( + TenantId::default().as_str(), + "permit(principal is Agent, action, resource);", + ) + .expect("agent-only policy should parse"); +} + +fn test_agent() -> SecurityContext { + SecurityContext::from_resolved_identity("agent-1", "operator", None) +} + +fn customer_security_context(id: &str) -> SecurityContext { + SecurityContext { + principal: temper_authz::Principal { + id: id.to_string(), + kind: temper_authz::PrincipalKind::Customer, + role: None, + acting_for: None, + agent_type: None, + attributes: Default::default(), + }, + context_attrs: Default::default(), + correlation_id: "local-tdata-test".to_string(), + } +} + #[test] fn parses_loopback_tdata_request() { let request = LocalTDataRequest::parse( @@ -211,45 +240,45 @@ fn parses_allowlisted_public_tdata_request() { } #[test] -fn local_tdata_headers_inherit_invoking_security_context() { - let inherited = SecurityContext::from_headers(&[ - ("x-temper-principal-id".to_string(), "admin-1".to_string()), +fn local_tdata_headers_discard_guest_authority_and_tenant() { + let map = header_map(&[ + ("accept".to_string(), "application/json".to_string()), + ("x-tenant-id".to_string(), "victim".to_string()), + ("x-temper-principal-id".to_string(), "attacker".to_string()), ("x-temper-principal-kind".to_string(), "admin".to_string()), + ("x-temper-agent-role".to_string(), "supervisor".to_string()), + ("x-temper-principal-scopes".to_string(), "root".to_string()), + ("x-temper-attr-region".to_string(), "all".to_string()), + ("x-temper-action-context".to_string(), "forged".to_string()), ( - "x-temper-principal-scopes".to_string(), - "admin:repos,repo:write".to_string(), + "x-temper-workflow-run-id".to_string(), + "workflow-1".to_string(), ), ]); - let inherited_headers = security_context_headers(&inherited); - let map = header_map( - &[("accept".to_string(), "application/json".to_string())], - &temper_runtime::tenant::TenantId::default(), - &inherited_headers, - ); + assert!(map.get("x-tenant-id").is_none()); + assert!(map.get("x-temper-principal-id").is_none()); + assert!(map.get("x-temper-principal-kind").is_none()); + assert!(map.get("x-temper-agent-role").is_none()); + assert!(map.get("x-temper-principal-scopes").is_none()); + assert!(map.get("x-temper-attr-region").is_none()); + assert!(map.get("x-temper-action-context").is_none()); assert_eq!( - map.get("x-temper-principal-id") - .and_then(|v| v.to_str().ok()), - Some("admin-1") - ); - assert_eq!( - map.get("x-temper-principal-kind") - .and_then(|v| v.to_str().ok()), - Some("admin") - ); - assert_eq!( - map.get("x-temper-principal-scopes") - .and_then(|v| v.to_str().ok()), - Some("admin:repos,repo:write") + map.get("x-temper-workflow-run-id") + .and_then(|value| value.to_str().ok()), + Some("workflow-1") ); } #[tokio::test] async fn local_tdata_calls_use_odata_handlers() { + let state = test_state(); + permit_agents(&state); + let agent = test_agent(); let host = LocalTDataWasmHost::new( - test_state(), + state, temper_runtime::tenant::TenantId::default(), - None, + Some(&agent), Arc::new(FailingHost), ); let headers = vec![ @@ -298,16 +327,47 @@ async fn local_tdata_calls_use_odata_handlers() { assert_eq!(submitted["status"], "Submitted"); } +/// ARN-170 regression guard for the direct-invocation (blob_adapter) loopback. +/// +/// This drives the real production helper `ServerState::local_tdata_direct_host` +/// that `invoke_wasm_direct` uses, so it guards the actual authority decision (not +/// just the `LocalTDataWasmHost` contract): the helper must build the loopback +/// WITH the caller's typed authority. The delegate is `FailingHost`, so if the +/// helper regresses to no authority the `/tdata` call falls through to it and +/// the test fails — the silent-401 blob regression ARN-170 introduced. #[tokio::test] -async fn local_tdata_synthesizes_invocation_tenant_header() { - let mut state = test_state(); - state.single_tenant_mode = false; - let host = LocalTDataWasmHost::new( - state, - temper_runtime::tenant::TenantId::default(), - None, - Arc::new(FailingHost), - ); +async fn direct_invocation_loopback_dispatches_in_process_with_caller_authority() { + let state = test_state(); + state + .authz + .reload_tenant_policies( + TenantId::default().as_str(), + "permit(principal is Agent, action, resource);", + ) + .expect("agent-only policy should parse"); + let caller = SecurityContext::from_resolved_identity("agent-1", "operator", None); + let host = state.local_tdata_direct_host(&TenantId::default(), Arc::new(FailingHost), &caller); + let headers = vec![ + ("x-tenant-id".to_string(), "default".to_string()), + ("accept".to_string(), "application/json".to_string()), + ]; + + let (status, _body) = host + .http_call("GET", "http://127.0.0.1:8787/tdata/Orders", &headers, "") + .await + .expect("direct-invocation loopback must dispatch in-process, not delegate"); + assert_eq!(status, StatusCode::OK.as_u16()); +} + +/// A customer with no permit must not create via the blob_adapter loopback. +/// +/// `test_state()` installs `system-platform:broad-permit`, so a System +/// loopback would return 201. The helper must carry the caller instead. +#[tokio::test] +async fn direct_invocation_loopback_does_not_run_as_system() { + let state = test_state(); + let caller = customer_security_context("customer-1"); + let host = state.local_tdata_direct_host(&TenantId::default(), Arc::new(FailingHost), &caller); let headers = vec![ ("content-type".to_string(), "application/json".to_string()), ("accept".to_string(), "application/json".to_string()), @@ -318,137 +378,148 @@ async fn local_tdata_synthesizes_invocation_tenant_header() { "POST", "http://127.0.0.1:8787/tdata/Orders", &headers, - r#"{"id":"order-local-no-header","Customer":"Lin"}"#, + r#"{"id":"system-elevated-order","Customer":"Eve"}"#, ) .await - .expect("local create should synthesize tenant header"); - assert_eq!(status, StatusCode::CREATED.as_u16(), "{body}"); + .expect("loopback must stay in-process under the caller principal"); + assert_eq!( + status, + StatusCode::FORBIDDEN.as_u16(), + "customer loopback must not inherit System, got {status}: {body}" + ); + assert!(!state.entity_exists(&TenantId::default(), "Order", "system-elevated-order")); +} + +#[tokio::test] +async fn local_tdata_forged_admin_headers_cannot_upgrade_customer() { + let state = test_state(); + state + .authz + .reload_tenant_policies( + TenantId::default().as_str(), + "permit(principal is Admin, action, resource);", + ) + .expect("admin-only policy should parse"); + let customer = customer_security_context("customer-1"); + let host = LocalTDataWasmHost::new( + state, + TenantId::default(), + Some(&customer), + Arc::new(FailingHost), + ); + let headers = vec![ + ("content-type".to_string(), "application/json".to_string()), + ("x-temper-principal-kind".to_string(), "admin".to_string()), + ("x-temper-principal-id".to_string(), "attacker".to_string()), + ("x-temper-principal-scopes".to_string(), "root".to_string()), + ("x-temper-attr-owner".to_string(), "*".to_string()), + ("x-temper-action-context".to_string(), "forged".to_string()), + ]; let (status, body) = host .http_call( - "GET", - "http://127.0.0.1:8787/tdata/Orders('order-local-no-header')", + "POST", + "http://127.0.0.1:8787/tdata/Orders", &headers, - "", + r#"{"id":"forged-admin-order"}"#, ) .await - .expect("local read should synthesize tenant header"); - assert_eq!(status, StatusCode::OK.as_u16(), "{body}"); - let fetched: serde_json::Value = serde_json::from_str(&body).expect("fetched JSON"); - assert_eq!(fetched["fields"]["Customer"], "Lin"); + .expect("local OData response should be returned"); + + assert_eq!(status, StatusCode::FORBIDDEN.as_u16(), "{body}"); } #[tokio::test] -async fn boundary_paths_delegate_to_production_host() { - let calls = Arc::new(AtomicUsize::new(0)); +async fn local_tdata_uses_exact_agent_and_ignores_guest_tenant() { + let state = test_state(); + state + .authz + .reload_tenant_policies( + TenantId::default().as_str(), + "permit(principal is Agent, action, resource);", + ) + .expect("agent-only policy should parse"); + let agent = SecurityContext::from_resolved_identity("agent-1", "operator", None); let host = LocalTDataWasmHost::new( - test_state(), - temper_runtime::tenant::TenantId::default(), - None, - Arc::new(CountingHost { - calls: calls.clone(), - stream_calls: Arc::new(AtomicUsize::new(0)), - }), + state.clone(), + TenantId::default(), + Some(&agent), + Arc::new(FailingHost), ); - let headers = vec![("x-tenant-id".to_string(), "default".to_string())]; - - let delegated = [ - ( - "DELETE", - "http://127.0.0.1:8787/tdata/Orders('order-local-1')", - ), - ( - "GET", - "http://127.0.0.1:8787/tdata/Files('file-local-1')/$value", - ), - ("GET", "https://api.example.com/tdata/Orders"), + let headers = vec![ + ("content-type".to_string(), "application/json".to_string()), + ("x-tenant-id".to_string(), "victim".to_string()), + ("x-temper-principal-kind".to_string(), "admin".to_string()), + ("x-temper-principal-id".to_string(), "attacker".to_string()), ]; - for (method, url) in delegated { - let (status, body) = host - .http_call(method, url, &headers, "") - .await - .expect("boundary path should delegate"); - assert_eq!(status, 299); - assert_eq!(body, "delegated"); - } + let (status, body) = host + .http_call( + "POST", + "http://localhost:8787/tdata/Orders", + &headers, + r#"{"id":"exact-agent-order"}"#, + ) + .await + .expect("local OData response should be returned"); - assert_eq!(calls.load(Ordering::SeqCst), delegated.len()); + assert_eq!(status, StatusCode::CREATED.as_u16(), "{body}"); + assert!(state.entity_exists(&TenantId::default(), "Order", "exact-agent-order")); + assert!(!state.entity_exists(&TenantId::new("victim"), "Order", "exact-agent-order")); } #[tokio::test] -async fn outbound_streaming_delegates_to_production_host() { - let stream_calls = Arc::new(AtomicUsize::new(0)); +async fn local_tdata_uses_invocation_tenant_without_a_tenant_header() { + let mut state = test_state(); + state.single_tenant_mode = false; + permit_agents(&state); + let agent = test_agent(); let host = LocalTDataWasmHost::new( - test_state(), + state, temper_runtime::tenant::TenantId::default(), - None, - Arc::new(CountingHost { - calls: Arc::new(AtomicUsize::new(0)), - stream_calls: stream_calls.clone(), - }), + Some(&agent), + Arc::new(FailingHost), ); + let headers = vec![ + ("content-type".to_string(), "application/json".to_string()), + ("accept".to_string(), "application/json".to_string()), + ]; - let handles = host - .http_stream_begin_outbound(HttpRequestHead { - method: "POST".to_string(), - url: "https://chatgpt.com/backend-api/codex/responses".to_string(), - headers: vec![("accept".to_string(), "text/event-stream".to_string())], - }) + let (status, body) = host + .http_call( + "POST", + "http://127.0.0.1:8787/tdata/Orders", + &headers, + r#"{"id":"order-local-no-header","Customer":"Lin"}"#, + ) .await - .expect("local TData wrapper must preserve outbound streaming support"); + .expect("local create should use typed tenant context"); + assert_eq!(status, StatusCode::CREATED.as_u16(), "{body}"); - assert_eq!(handles.request_body, StreamHandle(11)); - assert_eq!(handles.response_body, StreamHandle(12)); - assert_eq!( - host.http_stream_try_write(handles.request_body, b"hello".to_vec()) - .await - .expect("stream writes must delegate"), - 5 - ); - let head = host - .http_stream_response_head(handles.response_body) - .await - .expect("stream response head must delegate"); - assert_eq!(head.status, 299); - assert_eq!( - head.headers, - vec![("x-test-stream".to_string(), "delegated".to_string())] - ); - let bounded_chunk = host - .http_stream_read_bounded(handles.response_body, 1024) - .await - .expect("bounded stream reads must delegate"); - assert_eq!(bounded_chunk, b"delegated-bounded-read"); - let direct_chunk = host - .http_stream_read(handles.response_body) - .await - .expect("direct stream reads must delegate"); - assert_eq!(direct_chunk, b"delegated-direct-read"); - host.http_stream_send_response_head( - handles.response_body, - HttpResponseHead { - status: 204, - headers: Vec::new(), - }, - ) - .await - .expect("inbound stream response heads must delegate"); - host.http_stream_close(handles.request_body) + let (status, body) = host + .http_call( + "GET", + "http://127.0.0.1:8787/tdata/Orders('order-local-no-header')", + &headers, + "", + ) .await - .expect("stream close must delegate"); - - assert_eq!(stream_calls.load(Ordering::SeqCst), 7); + .expect("local read should use typed tenant context"); + assert_eq!(status, StatusCode::OK.as_u16(), "{body}"); + let fetched: serde_json::Value = serde_json::from_str(&body).expect("fetched JSON"); + assert_eq!(fetched["fields"]["Customer"], "Lin"); } #[tokio::test] async fn allowlisted_public_tdata_calls_use_odata_handlers() { let mut state = test_state(); state.local_tdata_hosts = Arc::new(BTreeSet::from(["temper.example".to_string()])); + permit_agents(&state); + let agent = test_agent(); let host = LocalTDataWasmHost::new( state, temper_runtime::tenant::TenantId::default(), - None, + Some(&agent), Arc::new(FailingHost), ); let headers = vec![ @@ -483,3 +554,59 @@ async fn allowlisted_public_tdata_calls_use_odata_handlers() { let fetched: serde_json::Value = serde_json::from_str(&body).expect("fetched JSON"); assert_eq!(fetched["fields"]["Customer"], "Grace"); } + +#[path = "local_tdata_host_test/delegation_tests.rs"] +mod delegation_tests; + +/// ARN-243 / ADR-0166. The engine reads the tenant's content decision off the +/// host it is handed, and production hands it a three-layer stack: +/// `AuthorizedWasmHost(LocalTDataWasmHost(ProductionWasmHost))`. Only the +/// innermost host holds the flag, so every wrapper in between has to forward it. +/// A wrapper that does not silently disables the opt-in for every tenant — +/// fail-safe, but inert. Asserted on the real composition, because a test that +/// wraps `ProductionWasmHost` directly builds a stack production never uses and +/// passes while the real one drops the decision. +#[tokio::test] +async fn production_host_stack_forwards_the_llm_content_export_decision() { + use temper_wasm::authorized_host::AuthorizedWasmHost; + use temper_wasm::host_trait::ProductionWasmHost; + + for opted_in in [true, false] { + let inner: Arc = Arc::new( + ProductionWasmHost::new(std::collections::BTreeMap::new()) + .with_llm_content_export(opted_in), + ); + let state = test_state(); + let agent = test_agent(); + let local_tdata: Arc = Arc::new(LocalTDataWasmHost::new( + state, + temper_runtime::tenant::TenantId::default(), + Some(&agent), + inner, + )); + assert_eq!( + local_tdata.exports_llm_content(), + opted_in, + "LocalTDataWasmHost must forward the decision (opted_in={opted_in})" + ); + + let full_stack = AuthorizedWasmHost::new( + local_tdata, + test_state().wasm_authz_gate(), + temper_wasm::WasmAuthzContext { + tenant: TenantId::default().to_string(), + module_name: "llm_caller".to_string(), + agent_id: None, + session_id: None, + entity_type: "Order".to_string(), + trigger_action: "SubmitOrder".to_string(), + }, + ); + assert_eq!( + full_stack.exports_llm_content(), + opted_in, + "the production host stack must carry the tenant's decision to the \ + engine (opted_in={opted_in})" + ); + } +} diff --git a/crates/temper-server/src/state/dispatch/wasm/local_tdata_host_test/delegation_tests.rs b/crates/temper-server/src/state/dispatch/wasm/local_tdata_host_test/delegation_tests.rs new file mode 100644 index 000000000..77e12fb69 --- /dev/null +++ b/crates/temper-server/src/state/dispatch/wasm/local_tdata_host_test/delegation_tests.rs @@ -0,0 +1,127 @@ +use super::*; + +#[tokio::test] +async fn boundary_paths_delegate_to_production_host() { + let calls = Arc::new(AtomicUsize::new(0)); + let host = LocalTDataWasmHost::new( + test_state(), + temper_runtime::tenant::TenantId::default(), + Some(&SecurityContext::system()), + Arc::new(CountingHost { + calls: calls.clone(), + stream_calls: Arc::new(AtomicUsize::new(0)), + }), + ); + let headers = vec![("x-tenant-id".to_string(), "default".to_string())]; + + let delegated = [ + ( + "DELETE", + "http://127.0.0.1:8787/tdata/Orders('order-local-1')", + ), + ( + "GET", + "http://127.0.0.1:8787/tdata/Files('file-local-1')/$value", + ), + ("GET", "https://api.example.com/tdata/Orders"), + ]; + + for (method, url) in delegated { + let (status, body) = host + .http_call(method, url, &headers, "") + .await + .expect("boundary path should delegate"); + assert_eq!(status, 299); + assert_eq!(body, "delegated"); + } + + assert_eq!(calls.load(Ordering::SeqCst), delegated.len()); +} + +#[tokio::test] +async fn local_tdata_without_invocation_authority_delegates() { + let calls = Arc::new(AtomicUsize::new(0)); + let host = LocalTDataWasmHost::new( + test_state(), + TenantId::default(), + None, + Arc::new(CountingHost { + calls: calls.clone(), + stream_calls: Arc::new(AtomicUsize::new(0)), + }), + ); + + let (status, body) = host + .http_call("GET", "http://127.0.0.1:8787/tdata/Orders", &[], "") + .await + .expect("missing typed authority should use authenticated fallthrough"); + + assert_eq!(status, 299); + assert_eq!(body, "delegated"); + assert_eq!(calls.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn outbound_streaming_delegates_to_production_host() { + let stream_calls = Arc::new(AtomicUsize::new(0)); + let host = LocalTDataWasmHost::new( + test_state(), + temper_runtime::tenant::TenantId::default(), + Some(&SecurityContext::system()), + Arc::new(CountingHost { + calls: Arc::new(AtomicUsize::new(0)), + stream_calls: stream_calls.clone(), + }), + ); + + let handles = host + .http_stream_begin_outbound(HttpRequestHead { + method: "POST".to_string(), + url: "https://chatgpt.com/backend-api/codex/responses".to_string(), + headers: vec![("accept".to_string(), "text/event-stream".to_string())], + }) + .await + .expect("local TData wrapper must preserve outbound streaming support"); + + assert_eq!(handles.request_body, StreamHandle(11)); + assert_eq!(handles.response_body, StreamHandle(12)); + assert_eq!( + host.http_stream_try_write(handles.request_body, b"hello".to_vec()) + .await + .expect("stream writes must delegate"), + 5 + ); + let head = host + .http_stream_response_head(handles.response_body) + .await + .expect("stream response head must delegate"); + assert_eq!(head.status, 299); + assert_eq!( + head.headers, + vec![("x-test-stream".to_string(), "delegated".to_string())] + ); + let bounded_chunk = host + .http_stream_read_bounded(handles.response_body, 1024) + .await + .expect("bounded stream reads must delegate"); + assert_eq!(bounded_chunk, b"delegated-bounded-read"); + let direct_chunk = host + .http_stream_read(handles.response_body) + .await + .expect("direct stream reads must delegate"); + assert_eq!(direct_chunk, b"delegated-direct-read"); + host.http_stream_send_response_head( + handles.response_body, + HttpResponseHead { + status: 204, + headers: Vec::new(), + }, + ) + .await + .expect("inbound stream response heads must delegate"); + host.http_stream_close(handles.request_body) + .await + .expect("stream close must delegate"); + + assert_eq!(stream_calls.load(Ordering::SeqCst), 7); +} diff --git a/crates/temper-server/src/state/dispatch/wasm/wasm_test.rs b/crates/temper-server/src/state/dispatch/wasm/wasm_test.rs new file mode 100644 index 000000000..3c2480d3a --- /dev/null +++ b/crates/temper-server/src/state/dispatch/wasm/wasm_test.rs @@ -0,0 +1,409 @@ +//! Unit tests for WASM dispatch, including the ADR-0166 callback-param gate. +use super::*; + +#[test] +fn internal_http_issuer_refuses_system_and_accepts_resolved_agents() { + let state = crate::state::ServerState::from_registry( + temper_runtime::ActorSystem::new("internal-capability-issuer-test"), + crate::registry::SpecRegistry::new(), + ); + let tenant = TenantId::new("tenant-a"); + + assert!( + internal_http_capability_issuer(&state, &tenant, Some(&SecurityContext::system())) + .is_none() + ); + let agent = SecurityContext::from_resolved_identity("agent-1", "worker", None); + assert!(internal_http_capability_issuer(&state, &tenant, Some(&agent)).is_some()); +} + +#[test] +fn composite_wasm_result_inherits_generated_dispatch_idempotency() { + let agent = AgentContext::for_service("version-publisher"); + + let composite_agent = agent_ctx_for_composite_wasm_result( + &agent, + Some("dispatch:default:App:app:PublishNewVersion:one"), + ); + + assert_eq!( + composite_agent.idempotency_key.as_deref(), + Some("dispatch:default:App:app:PublishNewVersion:one"), + "composite sub-writes need the parent dispatch idempotency so repeated app version updates get distinct sub-write keys" + ); +} + +#[test] +fn composite_wasm_result_preserves_caller_supplied_idempotency() { + let mut agent = AgentContext::for_service("version-publisher"); + agent.idempotency_key = Some("caller-key".to_string()); + + let composite_agent = agent_ctx_for_composite_wasm_result(&agent, Some("generated-key")); + + assert_eq!( + composite_agent.idempotency_key.as_deref(), + Some("caller-key"), + "caller idempotency remains authoritative for retries" + ); +} + +#[test] +fn gen_ai_span_attrs_are_recorded_only_for_llm_integrations() { + let params = json!({ + "input_tokens": 10, + "output_tokens": 20, + "_gen_ai_input_messages": "[{\"role\":\"user\"}]", + "_gen_ai_output_messages": "[{\"role\":\"assistant\"}]", + "_gen_ai_provider": "openai", + "_gen_ai_model": "gpt-5.4", + }); + + assert!(should_record_gen_ai_span_attrs(true, ¶ms)); + assert!(!should_record_gen_ai_span_attrs(false, ¶ms)); +} + +#[test] +fn llmobs_service_name_prefers_runtime_service_identity() { + unsafe { + std::env::set_var("DD_SERVICE", "temperpaw"); + std::env::remove_var("OTEL_SERVICE_NAME"); + } + assert_eq!(llmobs_service_name(), "temperpaw"); + + unsafe { + std::env::remove_var("DD_SERVICE"); + std::env::set_var("OTEL_SERVICE_NAME", "temper-agent"); + } + assert_eq!(llmobs_service_name(), "temper-agent"); + + unsafe { + std::env::remove_var("DD_SERVICE"); + std::env::remove_var("OTEL_SERVICE_NAME"); + } + assert_eq!(llmobs_service_name(), "temper-platform"); +} + +#[test] +fn llm_model_for_observability_prefers_callback_model() { + let entity_state = EntityState { + entity_type: "Session".to_string(), + entity_id: "session-1".to_string(), + status: "CallingProvider".to_string(), + item_count: 0, + counters: std::collections::BTreeMap::new(), + booleans: std::collections::BTreeMap::new(), + lists: std::collections::BTreeMap::new(), + fields: json!({"model": "claude-sonnet-4-6"}), + events: std::collections::VecDeque::new(), + total_event_count: 0, + events_since_snapshot: 0, + last_snapshot_sequence_nr: 0, + sequence_nr: 0, + processed_idempotency_keys: std::collections::BTreeMap::new(), + }; + let callback_params = json!({ + "_gen_ai_model": "gpt-5.4", + }); + + assert_eq!( + llm_model_for_observability(&entity_state, &callback_params), + "gpt-5.4" + ); +} + +#[test] +fn parse_internal_file_value_request_matches_only_value_paths() { + assert_eq!( + parse_internal_file_value_request( + "http://127.0.0.1:3467", + "http://127.0.0.1:3467/tdata/Files('fl-123')/$value" + ) + .as_deref(), + Some("fl-123") + ); + assert!( + parse_internal_file_value_request( + "http://127.0.0.1:3467", + "http://127.0.0.1:3467/tdata/Files('fl-123')" + ) + .is_none() + ); +} + +#[test] +fn llm_root_span_stays_on_active_trace() { + use opentelemetry::trace::TracerProvider as _; + use opentelemetry_sdk::trace::SdkTracerProvider; + use tracing_subscriber::prelude::*; + + let tracer_provider = SdkTracerProvider::builder().build(); + let subscriber = tracing_subscriber::registry().with( + tracing_opentelemetry::layer() + .with_tracer(tracer_provider.tracer("temper-server-llm-root-test")), + ); + let _subscriber_guard = tracing::subscriber::set_default(subscriber); + + let tenant = TenantId::default(); + let entity_state = EntityState { + entity_type: "Session".to_string(), + entity_id: "ss-1".to_string(), + status: "CallingProvider".to_string(), + item_count: 0, + counters: std::collections::BTreeMap::new(), + booleans: std::collections::BTreeMap::new(), + lists: std::collections::BTreeMap::new(), + fields: json!({"provider": "openai", "model": "gpt-5.4"}), + events: std::collections::VecDeque::new(), + total_event_count: 0, + events_since_snapshot: 0, + last_snapshot_sequence_nr: 0, + sequence_nr: 0, + processed_idempotency_keys: std::collections::BTreeMap::new(), + }; + let integration = temper_spec::automaton::Integration { + name: "provider_caller".to_string(), + trigger: "call_provider".to_string(), + integration_type: "wasm".to_string(), + module: Some("provider_caller".to_string()), + config: std::collections::BTreeMap::new(), + on_success: None, + on_failure: None, + llm: true, + }; + let agent_ctx = AgentContext { + session_id: Some("ss-1".to_string()), + ..AgentContext::default() + }; + + let parent = tracing::info_span!("dispatch.dispatch_tenant_action_core"); + let expected_trace_id = parent.in_scope(|| { + tracing::Span::current() + .context() + .span() + .span_context() + .trace_id() + .to_string() + }); + let (llm_trace_id, has_llmobs_auto_conversion_opt_out) = parent.in_scope(|| { + let ctx = WasmDispatchCtx { + entity_ref: WasmEntityRef { + tenant: &tenant, + entity_type: "Session", + entity_id: "ss-1", + }, + action: "ContextReady", + agent_ctx: &agent_ctx, + dispatch_idempotency_key: None, + mode: WasmDispatchMode::Inline, + }; + let span = build_llm_root_span(&ctx, &integration, &entity_state, "provider_caller"); + let has_opt_out = span + .metadata() + .map(|metadata| { + metadata + .fields() + .iter() + .any(|field| field.name() == "dd_llmobs_enabled") + }) + .unwrap_or(false); + ( + span.context().span().span_context().trace_id().to_string(), + has_opt_out, + ) + }); + + assert_eq!(llm_trace_id, expected_trace_id); + assert!( + has_llmobs_auto_conversion_opt_out, + "root LLM OTel span must opt out of Datadog auto LLMObs conversion" + ); +} + +#[test] +fn llm_parent_context_records_llm_span_and_dispatch_parent_ids() { + use opentelemetry::trace::TracerProvider as _; + use opentelemetry_sdk::trace::SdkTracerProvider; + use tracing_subscriber::prelude::*; + + let tracer_provider = SdkTracerProvider::builder().build(); + let subscriber = tracing_subscriber::registry().with( + tracing_opentelemetry::layer() + .with_tracer(tracer_provider.tracer("temper-server-llm-parent-test")), + ); + let _subscriber_guard = tracing::subscriber::set_default(subscriber); + + let dispatch_parent = tracing::info_span!("dispatch.dispatch_tenant_action_core"); + let (expected_trace_id, expected_parent_span_id) = dispatch_parent.in_scope(|| { + let span_context = tracing::Span::current() + .context() + .span() + .span_context() + .clone(); + ( + span_context.trace_id().to_string(), + span_context.span_id().to_string(), + ) + }); + + let mut callback_params = json!({}); + let entity_state = EntityState { + entity_type: "Session".to_string(), + entity_id: "session-1".to_string(), + status: "CallingProvider".to_string(), + item_count: 0, + counters: std::collections::BTreeMap::new(), + booleans: std::collections::BTreeMap::new(), + lists: std::collections::BTreeMap::new(), + fields: json!({}), + events: std::collections::VecDeque::new(), + total_event_count: 0, + events_since_snapshot: 0, + last_snapshot_sequence_nr: 0, + sequence_nr: 0, + processed_idempotency_keys: std::collections::BTreeMap::new(), + }; + let (llm_trace_id, llm_span_id) = dispatch_parent.in_scope(|| { + let llm_span = tracing::info_span!("llm_caller.trace"); + let span_context = llm_span.context().span().span_context().clone(); + attach_llm_parent_context( + &llm_span, + Some(&expected_parent_span_id), + &entity_state, + "session-1", + 1_234, + &mut callback_params, + ); + ( + span_context.trace_id().to_string(), + span_context.span_id().to_string(), + ) + }); + + assert_eq!(llm_trace_id, expected_trace_id); + assert_ne!(llm_span_id, expected_parent_span_id); + assert_eq!( + callback_params["_gen_ai_parent_trace_id"], + expected_trace_id + ); + assert_eq!(callback_params["_gen_ai_parent_span_id"], llm_span_id); + assert_eq!( + callback_params["_gen_ai_llm_parent_span_id"], + expected_parent_span_id + ); + assert_eq!( + callback_params["gen_ai_llm_parent_span_id"], + expected_parent_span_id + ); + let expected_agent_span_id = + temper_observe::llmobs_api::derive_span_id(&format!("{expected_trace_id}:session-1:agent")); + assert_eq!( + callback_params["_gen_ai_llmobs_agent_span_id"], + expected_agent_span_id + ); + assert_eq!( + callback_params["llmobs_agent_span_id"], + expected_agent_span_id + ); + assert_ne!( + callback_params["_gen_ai_llmobs_agent_span_id"], + expected_parent_span_id + ); + assert!( + callback_params["_gen_ai_llmobs_workflow_span_id"] + .as_str() + .is_some_and(|workflow_span_id| !workflow_span_id.is_empty() + && workflow_span_id != expected_parent_span_id + && workflow_span_id != llm_span_id) + ); + assert_eq!( + callback_params["llmobs_workflow_span_id"], + callback_params["_gen_ai_llmobs_workflow_span_id"] + ); + assert!( + callback_params["llmobs_agent_start_ns"] + .as_u64() + .is_some_and(|start_ns| start_ns > 0) + ); +} + +#[test] +fn llm_parent_context_reuses_existing_llmobs_agent_root() { + use opentelemetry::trace::TracerProvider as _; + use opentelemetry_sdk::trace::SdkTracerProvider; + use tracing_subscriber::prelude::*; + + let tracer_provider = SdkTracerProvider::builder().build(); + let subscriber = tracing_subscriber::registry().with( + tracing_opentelemetry::layer() + .with_tracer(tracer_provider.tracer("temper-server-llm-parent-reuse-test")), + ); + let _subscriber_guard = tracing::subscriber::set_default(subscriber); + + let entity_state = EntityState { + entity_type: "Session".to_string(), + entity_id: "session-1".to_string(), + status: "CallingProvider".to_string(), + item_count: 0, + counters: std::collections::BTreeMap::new(), + booleans: std::collections::BTreeMap::new(), + lists: std::collections::BTreeMap::new(), + fields: json!({ + "llmobs_agent_span_id": "stable-agent-root", + "llmobs_agent_start_ns": 12345_u64, + }), + events: std::collections::VecDeque::new(), + total_event_count: 0, + events_since_snapshot: 0, + last_snapshot_sequence_nr: 0, + sequence_nr: 0, + processed_idempotency_keys: std::collections::BTreeMap::new(), + }; + + let mut callback_params = json!({}); + let llm_span = tracing::info_span!("llm_caller.trace"); + attach_llm_parent_context( + &llm_span, + Some("turn-parent-span"), + &entity_state, + "session-1", + 99, + &mut callback_params, + ); + + assert_eq!( + callback_params["_gen_ai_llmobs_agent_span_id"], + "stable-agent-root" + ); + assert_eq!(callback_params["llmobs_agent_span_id"], "stable-agent-root"); + assert_eq!(callback_params["_gen_ai_llmobs_agent_start_ns"], 12345_u64); + assert_eq!(callback_params["llmobs_agent_start_ns"], 12345_u64); +} + +#[test] +fn llmobs_tool_parent_prefers_workflow_span_id() { + let entity_state = EntityState { + entity_type: "Session".to_string(), + entity_id: "ss-1".to_string(), + status: "CallingTools".to_string(), + item_count: 0, + counters: std::collections::BTreeMap::new(), + booleans: std::collections::BTreeMap::new(), + lists: std::collections::BTreeMap::new(), + fields: json!({ + "gen_ai_parent_trace_id": "trace-1", + "gen_ai_parent_span_id": "legacy-llm-parent", + "llmobs_workflow_span_id": "workflow-parent", + }), + events: std::collections::VecDeque::new(), + total_event_count: 0, + events_since_snapshot: 0, + last_snapshot_sequence_nr: 0, + sequence_nr: 0, + processed_idempotency_keys: std::collections::BTreeMap::new(), + }; + + assert_eq!( + llmobs_tool_trace_and_parent(&entity_state, &json!({})), + Some(("trace-1".to_string(), "workflow-parent".to_string())) + ); +} diff --git a/crates/temper-server/src/state/dispatch/wasm_secrets.rs b/crates/temper-server/src/state/dispatch/wasm_secrets.rs index acf943d28..88ab2f496 100644 --- a/crates/temper-server/src/state/dispatch/wasm_secrets.rs +++ b/crates/temper-server/src/state/dispatch/wasm_secrets.rs @@ -4,7 +4,7 @@ use std::sync::Arc; use temper_runtime::tenant::TenantId; use temper_wasm::{SecretResolverFn, WasmAuthzContext, WasmAuthzDecision, WasmAuthzGate}; -const WASM_BOOTSTRAP_SECRET_KEYS: [&str; 3] = ["blob_endpoint", "temper_api_key", "temper_api_url"]; +const WASM_BOOTSTRAP_SECRET_KEYS: [&str; 1] = ["blob_endpoint"]; fn is_wasm_bootstrap_secret(key: &str) -> bool { WASM_BOOTSTRAP_SECRET_KEYS.contains(&key) || key.starts_with("ca_cert:") @@ -54,6 +54,12 @@ impl crate::state::ServerState { let tenant_str = tenant.to_string(); Some(Arc::new(move |key: &str| { + if key.eq_ignore_ascii_case("temper_api_key") { + return Err( + "secret 'temper_api_key' is reserved and unavailable to WASM guests" + .to_string(), + ); + } match gate.authorize_secret_access(key, &authz_ctx) { WasmAuthzDecision::Allow => vault .get_secret(&tenant_str, key) @@ -190,21 +196,16 @@ mod tests { let secrets = state.get_authorized_wasm_host_bootstrap_secrets(&tenant, &gate, &test_authz_ctx()); - assert_eq!(secrets.len(), 4); + assert_eq!(secrets.len(), 2); assert!(secrets.contains_key("blob_endpoint")); - assert!(secrets.contains_key("temper_api_key")); - assert!(secrets.contains_key("temper_api_url")); + assert!(!secrets.contains_key("temper_api_key")); + assert!(!secrets.contains_key("temper_api_url")); assert!(secrets.contains_key("ca_cert:internal")); assert!(!secrets.contains_key("provider_api_key")); assert!(!secrets.contains_key("unrelated_secret")); assert_eq!( gate.requested_keys(), - vec![ - "blob_endpoint".to_string(), - "ca_cert:internal".to_string(), - "temper_api_key".to_string(), - "temper_api_url".to_string() - ] + vec!["blob_endpoint".to_string(), "ca_cert:internal".to_string()] ); } @@ -243,4 +244,28 @@ mod tests { ] ); } + + #[test] + fn lazy_secret_resolver_never_exposes_reserved_root_key() { + let tenant = TenantId::new("tenant-a"); + let vault = SecretsVault::new(&[9_u8; 32]); + vault + .cache_secret(tenant.as_str(), "temper_api_key", "deployment-root".into()) + .unwrap(); + let state = state_with_vault(vault); + let gate = RecordingSecretGate::allowing(&["temper_api_key"]); + let requested = gate.requested.clone(); + let resolver = state + .authorized_wasm_secret_resolver(&tenant, Arc::new(gate), test_authz_ctx()) + .expect("resolver should exist when vault is configured"); + + let error = resolver("temper_api_key").expect_err("root key must stay unavailable"); + assert!(error.contains("reserved")); + assert!( + requested + .lock() + .expect("lock should not be poisoned") + .is_empty() + ); + } } diff --git a/crates/temper-server/src/state/entity_ops.rs b/crates/temper-server/src/state/entity_ops.rs index 633df47b1..7fc561ccf 100644 --- a/crates/temper-server/src/state/entity_ops.rs +++ b/crates/temper-server/src/state/entity_ops.rs @@ -282,21 +282,57 @@ impl ServerState { .get_tenant_entity_state(tenant, entity_type, entity_id) .await?; + let resource_attrs = self + .build_authz_resource_attrs( + tenant, + entity_type, + entity_id, + ¤t_state.state.status, + ¤t_state.state.fields, + ) + .await?; + + Ok(AuthzResourceSnapshot { + current_state, + resource_attrs, + }) + } + + /// Build the Cedar resource view for a prospective entity representation. + /// + /// Mutation handlers use this after applying PATCH/PUT fields so policies + /// evaluate the state that would be committed, including refreshed context + /// entity status attributes. + pub(crate) async fn build_authz_resource_attrs( + &self, + tenant: &TenantId, + entity_type: &str, + entity_id: &str, + status: &str, + fields: &serde_json::Value, + ) -> Result, String> { let mut resource_attrs = BTreeMap::new(); - resource_attrs.insert( - "id".to_string(), - serde_json::Value::String(entity_id.to_string()), - ); - resource_attrs.insert( - "status".to_string(), - serde_json::Value::String(current_state.state.status.clone()), - ); - if let serde_json::Value::Object(fields) = ¤t_state.state.fields { + if let serde_json::Value::Object(fields) = fields { for (k, v) in fields { - resource_attrs.insert(k.clone(), v.clone()); + if !temper_spec::automaton::is_server_derived_field_name(k) { + resource_attrs.insert(k.clone(), v.clone()); + } } } + for key in ["id", "Id"] { + resource_attrs.insert( + key.to_string(), + serde_json::Value::String(entity_id.to_string()), + ); + } + for key in ["status", "Status"] { + resource_attrs.insert( + key.to_string(), + serde_json::Value::String(status.to_string()), + ); + } + let context_entities: Vec = self .registry .read() @@ -306,9 +342,7 @@ impl ServerState { .unwrap_or_default(); for ce in &context_entities { - let target_id = current_state - .state - .fields + let target_id = fields .get(&ce.id_field) .and_then(|v| v.as_str()) .unwrap_or(""); @@ -327,11 +361,42 @@ impl ServerState { let has_spec = self.has_registered_spec(tenant, entity_type)?; resource_attrs.insert("has_spec".to_string(), serde_json::Value::Bool(has_spec)); + Ok(resource_attrs) + } - Ok(AuthzResourceSnapshot { - current_state, - resource_attrs, - }) + /// Return the spec-defined initial state used by a true entity create. + pub(crate) fn initial_entity_status( + &self, + tenant: &TenantId, + entity_type: &str, + ) -> Result { + if let Some(table) = self + .registry + .read() + .map_err(|error| format!("registry lock poisoned: {error}"))? + .get_table(tenant, entity_type) + { + return Ok(table.initial_state.clone()); + } + self.transition_tables + .get(entity_type) + .map(|table| table.initial_state.clone()) + .ok_or_else(|| { + format!("No transition table for tenant '{tenant}', entity type '{entity_type}'") + }) + } + + /// Build trusted Cedar attributes for a durably absent create target. + pub(crate) async fn build_create_authz_resource_attrs( + &self, + tenant: &TenantId, + entity_type: &str, + entity_id: &str, + fields: &serde_json::Value, + ) -> Result, String> { + let initial_status = self.initial_entity_status(tenant, entity_type)?; + self.build_authz_resource_attrs(tenant, entity_type, entity_id, &initial_status, fields) + .await } /// Mark every entity type observed in `entities` as fully hydrated from the @@ -840,28 +905,11 @@ impl ServerState { .unwrap_or_default() } - /// Check authorization for an action using the Cedar ABAC engine. - /// - /// Returns a typed [`AuthzDenial`] on failure, preserving the denial kind - /// (policy denied, no matching permit, invalid principal, etc.). - /// - /// Accepts `BTreeMap` for DST compliance; converts at the authz boundary. - pub fn authorize( - &self, - headers: &[(String, String)], - action: &str, - resource_type: &str, - resource_attrs: &BTreeMap, - ) -> Result<(), AuthzDenial> { - let ctx = SecurityContext::from_headers(headers); - self.authorize_with_context(&ctx, action, resource_type, resource_attrs, "default") - } - /// Check authorization using a pre-built `SecurityContext`. /// - /// Unlike [`authorize`] which builds the context from raw headers, this - /// method accepts an already-constructed context enriched with agent - /// identity and resource attributes. + /// This method accepts an already-constructed, credential-derived context + /// enriched with agent identity and resource attributes. There is no + /// header-based authorization entry point. /// /// Returns a typed [`AuthzDenial`] on failure, preserving the denial kind. /// @@ -896,13 +944,27 @@ impl ServerState { AuthzDecision::Allow { .. } => "Allow", AuthzDecision::Deny(_) => "Deny", }; + // Correlate the decision with the resource it governed and the request + // that triggered it. `resource_attrs["id"]` is the Cedar resource id + // every caller populates (see `resource_attrs_from_body`); the trace id + // comes from the active span, which is the same request span the HTTP + // handler and the dispatch both run under. + let entity_id = resource_attrs + .get("id") + .and_then(serde_json::Value::as_str) + .unwrap_or_default(); + let trace_id = crate::request_context::current_span_trace_context_ids() + .map(|(trace_id, _span_id)| trace_id) + .unwrap_or_default(); let wide = wide_event::from_authz_decision(wide_event::AuthzDecisionInput { action, resource_type, + entity_id, principal_kind: &format!("{:?}", security_ctx.principal.kind), decision: decision_str, duration_ns, tenant, + trace_id: &trace_id, }); wide_event::emit_span(&wide); wide_event::emit_metrics(&wide); @@ -1103,13 +1165,14 @@ impl ServerState { }; let persistence_id = format!("{tenant}:{entity_type}:{entity_id}"); + let initial_fields = + crate::entity_actor::effects::sanitize_action_params(&initial_fields).into_owned(); let mut fields = initial_fields.clone(); - if let Some(obj) = fields.as_object_mut() { - obj.entry("Id".to_string()) - .or_insert(serde_json::Value::String(entity_id.to_string())); - obj.entry("Status".to_string()) - .or_insert(serde_json::Value::String(table.initial_state.clone())); - } + crate::entity_actor::effects::canonicalize_entity_fields( + &mut fields, + entity_id, + &table.initial_state, + ); let mut state = EntityState { entity_type: entity_type.to_string(), @@ -1345,19 +1408,82 @@ impl ServerState { fields: serde_json::Value, replace: bool, ) -> Result { + let response = self + .update_tenant_entity_fields_checked( + tenant, + entity_type, + entity_id, + fields, + replace, + None, + ) + .await?; + if response.success { + Ok(response) + } else { + Err(response + .error + .clone() + .unwrap_or_else(|| "entity field update was rejected".to_string())) + } + } + + /// Update fields only if the actor still matches the state that Cedar + /// authorized. The comparison and mutation are serialized by the actor + /// mailbox, closing the read/authorize/write race. + pub(crate) async fn update_tenant_entity_fields_if_current( + &self, + tenant: &TenantId, + entity_type: &str, + entity_id: &str, + fields: serde_json::Value, + replace: bool, + expected_precondition: String, + ) -> Result { + self.update_tenant_entity_fields_checked( + tenant, + entity_type, + entity_id, + fields, + replace, + Some(expected_precondition), + ) + .await + } + + async fn update_tenant_entity_fields_checked( + &self, + tenant: &TenantId, + entity_type: &str, + entity_id: &str, + fields: serde_json::Value, + replace: bool, + expected_precondition: Option, + ) -> Result { + if !fields.is_object() { + return Err("entity field update must be a JSON object".to_string()); + } let actor_ref = self .get_or_spawn_tenant_actor(tenant, entity_type, entity_id) .ok_or_else(|| { format!("No transition table for tenant '{tenant}', entity type '{entity_type}'") })?; - let policy = self.dispatch_retry_policy(); + let mut policy = self.dispatch_retry_policy(); + if expected_precondition.is_some() { + // A timed-out compare-and-set may already have committed. Retrying + // the same precondition would turn that ambiguity into a false + // conflict, so the caller must re-read and re-authorize instead. + policy.max_attempts = 1; + } let fields_for_retry = fields; + let precondition_for_retry = expected_precondition; let response = retry::ask_with_backoff::<_, EntityResponse, _>( &actor_ref, || EntityMsg::UpdateFields { fields: fields_for_retry.clone(), replace, + expected_precondition: precondition_for_retry.clone(), }, &policy, ) @@ -1427,6 +1553,43 @@ impl ServerState { tenant: &TenantId, entity_type: &str, entity_id: &str, + ) -> Result { + let response = self + .delete_tenant_entity_checked(tenant, entity_type, entity_id, None) + .await?; + if response.success { + Ok(response) + } else { + Err(response + .error + .clone() + .unwrap_or_else(|| "entity delete was rejected".to_string())) + } + } + + /// Delete only if the actor still matches the state Cedar authorized. + pub(crate) async fn delete_tenant_entity_if_current( + &self, + tenant: &TenantId, + entity_type: &str, + entity_id: &str, + expected_authorization_precondition: String, + ) -> Result { + self.delete_tenant_entity_checked( + tenant, + entity_type, + entity_id, + Some(expected_authorization_precondition), + ) + .await + } + + async fn delete_tenant_entity_checked( + &self, + tenant: &TenantId, + entity_type: &str, + entity_id: &str, + expected_authorization_precondition: Option, ) -> Result { let actor_ref = self .get_or_spawn_tenant_actor(tenant, entity_type, entity_id) @@ -1434,10 +1597,16 @@ impl ServerState { format!("No transition table for tenant '{tenant}', entity type '{entity_type}'") })?; - let policy = self.dispatch_retry_policy(); + let mut policy = self.dispatch_retry_policy(); + if expected_authorization_precondition.is_some() { + policy.max_attempts = 1; + } + let precondition_for_retry = expected_authorization_precondition; let response = retry::ask_with_backoff::<_, EntityResponse, _>( &actor_ref, - || EntityMsg::Delete, + || EntityMsg::Delete { + expected_authorization_precondition: precondition_for_retry.clone(), + }, &policy, ) .await @@ -1715,11 +1884,12 @@ impl ServerState { } /// Update Agent.Hint annotations based on trajectory analysis. - pub fn enrich_metadata(&self, action_name: &str, hint: &str) { + pub fn enrich_metadata(&self, tenant: &TenantId, action_name: &str, hint: &str) { const AGENT_HINTS_BUDGET: usize = 1_000; - let Ok(mut hints) = self.agent_hints.write() else { + let Ok(mut all_hints) = self.agent_hints.write() else { return; }; + let hints = all_hints.entry(tenant.clone()).or_default(); hints.insert(action_name.to_string(), hint.to_string()); while hints.len() > AGENT_HINTS_BUDGET { let oldest_key = hints.iter().next().map(|(k, _)| k.clone()); diff --git a/crates/temper-server/src/state/evolution.rs b/crates/temper-server/src/state/evolution.rs index 3fffc36e1..a8c34ae78 100644 --- a/crates/temper-server/src/state/evolution.rs +++ b/crates/temper-server/src/state/evolution.rs @@ -6,6 +6,8 @@ use temper_store_turso::EvolutionRecordRow; use tracing::instrument; +use crate::storage::EvolutionRecordWrite; + use super::ServerState; impl ServerState { @@ -13,12 +15,13 @@ impl ServerState { #[instrument(skip_all, fields(otel.name = "evolution.list_records", record_type, status))] pub async fn list_evolution_records( &self, + tenant: &str, record_type: Option<&str>, status: Option<&str>, ) -> Result, String> { - if let Some(store) = self.platform_metadata_store() { + if let Some(store) = self.metadata_store_for_tenant(tenant).await { let rows = store - .list_evolution_records(record_type, status) + .list_evolution_records(tenant, record_type, status) .await .map_err(|e| { tracing::warn!( @@ -43,7 +46,7 @@ impl ServerState { // Fall through to Postgres. if let Some(pg) = &self.pg_record_store { let rows = pg - .list_records_generic(record_type, status) + .list_records_generic(tenant, record_type, status) .await .map_err(|e| { tracing::warn!( @@ -74,11 +77,12 @@ impl ServerState { #[instrument(skip_all, fields(otel.name = "evolution.get_record", id))] pub async fn get_evolution_record( &self, + tenant: &str, id: &str, ) -> Result, String> { - if let Some(store) = self.platform_metadata_store() { + if let Some(store) = self.metadata_store_for_tenant(tenant).await { let row = store - .get_evolution_record(id) + .get_evolution_record(tenant, id) .await .map_err(|e| { tracing::warn!(backend = store.backend_name(), record_id = id, error = %e, "evolution.store.read"); @@ -94,7 +98,7 @@ impl ServerState { } if let Some(pg) = &self.pg_record_store { - let row = pg.get_record_generic(id).await.map_err(|e| { + let row = pg.get_record_generic(tenant, id).await.map_err(|e| { tracing::warn!(backend = "postgres", record_id = id, error = %e, "evolution.store.read"); e.to_string() })?; @@ -114,9 +118,12 @@ impl ServerState { /// List ranked insights (I-Records) from the first available backend. #[instrument(skip_all, fields(otel.name = "evolution.list_ranked_insights"))] - pub async fn list_ranked_insights(&self) -> Result, String> { - if let Some(store) = self.platform_metadata_store() { - let rows = store.list_ranked_insights().await.map_err(|e| { + pub async fn list_ranked_insights( + &self, + tenant: &str, + ) -> Result, String> { + if let Some(store) = self.metadata_store_for_tenant(tenant).await { + let rows = store.list_ranked_insights(tenant).await.map_err(|e| { tracing::warn!(backend = store.backend_name(), error = %e, "evolution.store.read"); e.to_string() })?; @@ -129,7 +136,7 @@ impl ServerState { } if let Some(pg) = &self.pg_record_store { - let rows = pg.list_ranked_insights_generic().await.map_err(|e| { + let rows = pg.list_ranked_insights_generic(tenant).await.map_err(|e| { tracing::warn!(backend = "postgres", error = %e, "evolution.store.read"); e.to_string() })?; @@ -147,26 +154,31 @@ impl ServerState { } /// Insert a generic evolution record into the first available backend. - #[instrument(skip_all, fields(otel.name = "evolution.insert_record", id, record_type, status))] + #[instrument(skip_all, fields(otel.name = "evolution.insert_record"))] pub async fn insert_evolution_record( &self, - id: &str, - record_type: &str, - status: &str, - created_by: &str, - derived_from: Option<&str>, - data_json: &str, + record: EvolutionRecordWrite<'_>, ) -> Result<(), String> { - if let Some(store) = self.platform_metadata_store() { + let EvolutionRecordWrite { + tenant, + id, + record_type, + status, + created_by, + derived_from, + data_json, + } = record; + if let Some(store) = self.metadata_store_for_tenant(tenant).await { store - .insert_evolution_record( + .insert_evolution_record(EvolutionRecordWrite { + tenant, id, record_type, status, created_by, derived_from, data_json, - ) + }) .await .map_err(|e| { tracing::warn!( @@ -192,19 +204,27 @@ impl ServerState { } if let Some(pg) = &self.pg_record_store { - pg.insert_record_generic(id, record_type, status, created_by, derived_from, data_json) - .await - .map_err(|e| { - tracing::warn!( - backend = "postgres", - record_id = id, - record_type, - status, - error = %e, - "evolution.store.write" - ); - e.to_string() - })?; + pg.insert_record_generic(temper_evolution::GenericEvolutionRecordInsert { + tenant, + id, + record_type, + status, + created_by, + derived_from, + data_json, + }) + .await + .map_err(|e| { + tracing::warn!( + backend = "postgres", + record_id = id, + record_type, + status, + error = %e, + "evolution.store.write" + ); + e.to_string() + })?; tracing::info!( backend = "postgres", record_id = id, @@ -231,6 +251,7 @@ impl ServerState { fn pg_row_to_turso(row: temper_evolution::GenericEvolutionRow) -> EvolutionRecordRow { EvolutionRecordRow { id: row.id, + tenant: row.tenant, record_type: row.record_type, status: row.status, created_by: row.created_by, diff --git a/crates/temper-server/src/state/file_initial_writes.rs b/crates/temper-server/src/state/file_initial_writes.rs index 90ba4df9c..8677abd6c 100644 --- a/crates/temper-server/src/state/file_initial_writes.rs +++ b/crates/temper-server/src/state/file_initial_writes.rs @@ -319,6 +319,7 @@ impl ServerState { let fields = response.state.fields.clone(); let agent_ctx = agent_ctx.clone(); tokio::spawn(async move { + // determinism-ok: post-commit external reaction side effects // determinism-ok: post-commit reaction side effects mirror the // existing non-awaited File `$value` update behavior. dispatcher @@ -344,13 +345,13 @@ fn initial_file_state( table: &temper_jit::table::TransitionTable, initial_fields: serde_json::Value, ) -> EntityState { - let mut fields = initial_fields; - if let Some(obj) = fields.as_object_mut() { - obj.entry("Id".to_string()) - .or_insert(serde_json::Value::String(file_id.to_string())); - obj.entry("Status".to_string()) - .or_insert(serde_json::Value::String(table.initial_state.clone())); - } + let mut fields = + crate::entity_actor::effects::sanitize_action_params(&initial_fields).into_owned(); + crate::entity_actor::effects::canonicalize_entity_fields( + &mut fields, + file_id, + &table.initial_state, + ); EntityState { entity_type: "File".to_string(), diff --git a/crates/temper-server/src/state/file_read_blobs.rs b/crates/temper-server/src/state/file_read_blobs.rs index 70fdf0f30..1c7a04e09 100644 --- a/crates/temper-server/src/state/file_read_blobs.rs +++ b/crates/temper-server/src/state/file_read_blobs.rs @@ -3,6 +3,7 @@ use temper_runtime::tenant::TenantId; use super::ServerState; impl ServerState { + #[cfg(feature = "observe")] pub(crate) async fn fetch_blob_text_for_hash( &self, tenant: &TenantId, diff --git a/crates/temper-server/src/state/file_reads.rs b/crates/temper-server/src/state/file_reads.rs index f2e22b175..a78f3045e 100644 --- a/crates/temper-server/src/state/file_reads.rs +++ b/crates/temper-server/src/state/file_reads.rs @@ -1,7 +1,4 @@ -use std::collections::{BTreeMap, BTreeSet}; -use std::sync::Arc; - -use futures_util::stream::{self, StreamExt}; +use std::collections::BTreeMap; use temper_runtime::tenant::TenantId; use tracing::instrument; @@ -11,7 +8,11 @@ use super::file_read_projection::{ file_version_projection_from_row, file_version_projection_from_state, }; -const FILE_BATCH_READ_CONCURRENCY: usize = 8; +#[cfg(feature = "observe")] +mod batch_text; + +#[cfg(feature = "observe")] +pub(crate) use batch_text::{BatchTextReadError, validate_batch_text_ids}; #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] pub struct TextFileReadResult { @@ -169,99 +170,7 @@ impl ServerState { } } - #[instrument(skip_all, fields( - otel.name = "state.read_file_texts_batch", - tenant = %tenant, - file_count = file_ids.len(), - ))] - pub async fn read_file_texts_batch( - &self, - tenant: &TenantId, - file_ids: &[String], - ) -> Result, String> { - if file_ids.is_empty() { - return Ok(Vec::new()); - } - - let mut unique_ids = Vec::new(); - let mut seen = BTreeSet::new(); - for file_id in file_ids { - if seen.insert(file_id.clone()) { - unique_ids.push(file_id.clone()); - } - } - - let meta_by_id = self - .load_file_projection_metadata_batch(tenant, &unique_ids) - .await?; - - let meta_by_id = Arc::new(meta_by_id); - let results = stream::iter(file_ids.iter().cloned()) - .map(|file_id| { - let meta_by_id = Arc::clone(&meta_by_id); - async move { - let meta = meta_by_id.get(&file_id).cloned(); - self.read_single_file_text(tenant, file_id, meta).await - } - }) - .buffered(FILE_BATCH_READ_CONCURRENCY) - .collect::>() - .await; - - let mut out = Vec::with_capacity(results.len()); - for result in results { - out.push(result?); - } - Ok(out) - } - - #[instrument(skip_all, fields( - otel.name = "state.read_file_version_texts_batch", - tenant = %tenant, - file_version_count = file_version_ids.len(), - ))] - pub async fn read_file_version_texts_batch( - &self, - tenant: &TenantId, - file_version_ids: &[String], - ) -> Result, String> { - if file_version_ids.is_empty() { - return Ok(Vec::new()); - } - - let mut unique_ids = Vec::new(); - let mut seen = BTreeSet::new(); - for file_version_id in file_version_ids { - if seen.insert(file_version_id.clone()) { - unique_ids.push(file_version_id.clone()); - } - } - - let meta_by_id = self - .load_file_version_projection_metadata_batch(tenant, &unique_ids) - .await?; - - let meta_by_id = Arc::new(meta_by_id); - let results = stream::iter(file_version_ids.iter().cloned()) - .map(|file_version_id| { - let meta_by_id = Arc::clone(&meta_by_id); - async move { - let meta = meta_by_id.get(&file_version_id).cloned(); - self.read_single_file_version_text(tenant, file_version_id, meta) - .await - } - }) - .buffered(FILE_BATCH_READ_CONCURRENCY) - .collect::>() - .await; - - let mut out = Vec::with_capacity(results.len()); - for result in results { - out.push(result?); - } - Ok(out) - } - + #[cfg(feature = "observe")] async fn load_file_projection_metadata_batch( &self, tenant: &TenantId, @@ -286,6 +195,7 @@ impl ServerState { Ok(by_id) } + #[cfg(feature = "observe")] async fn load_file_version_projection_metadata_batch( &self, tenant: &TenantId, @@ -426,86 +336,6 @@ impl ServerState { ) .await } - - async fn read_single_file_text( - &self, - tenant: &TenantId, - file_id: String, - meta: Option, - ) -> Result { - let Some(meta) = meta else { - return Ok(TextFileReadResult { - file_id, - found: false, - content_hash: String::new(), - mime_type: String::new(), - text: String::new(), - }); - }; - - if !meta.has_content || meta.content_hash.is_empty() { - return Ok(TextFileReadResult { - file_id, - found: true, - content_hash: meta.content_hash, - mime_type: meta.mime_type, - text: String::new(), - }); - } - - let text = self - .fetch_blob_text_for_hash(tenant, &meta.content_hash) - .await? - .unwrap_or_default(); - - Ok(TextFileReadResult { - file_id, - found: true, - content_hash: meta.content_hash, - mime_type: meta.mime_type, - text, - }) - } - - async fn read_single_file_version_text( - &self, - tenant: &TenantId, - file_version_id: String, - meta: Option, - ) -> Result { - let Some(meta) = meta else { - return Ok(TextFileVersionReadResult { - file_version_id, - found: false, - content_hash: String::new(), - mime_type: String::new(), - text: String::new(), - }); - }; - - if !meta.has_content || meta.content_hash.is_empty() { - return Ok(TextFileVersionReadResult { - file_version_id, - found: true, - content_hash: meta.content_hash, - mime_type: meta.mime_type, - text: String::new(), - }); - } - - let text = self - .fetch_blob_text_for_hash(tenant, &meta.content_hash) - .await? - .unwrap_or_default(); - - Ok(TextFileVersionReadResult { - file_version_id, - found: true, - content_hash: meta.content_hash, - mime_type: meta.mime_type, - text, - }) - } } fn stream_read_content_hash(read: &IndexedFileStreamRead) -> Option<&str> { diff --git a/crates/temper-server/src/state/file_reads/batch_text.rs b/crates/temper-server/src/state/file_reads/batch_text.rs new file mode 100644 index 000000000..e583e4853 --- /dev/null +++ b/crates/temper-server/src/state/file_reads/batch_text.rs @@ -0,0 +1,264 @@ +//! Bounded JSON batch reads for file text. + +use std::collections::BTreeSet; + +use temper_runtime::tenant::TenantId; +use tracing::instrument; + +use super::{FileProjectionMeta, TextFileReadResult, TextFileVersionReadResult}; +use crate::state::ServerState; + +const MAX_BATCH_ITEMS: usize = 100; +const MAX_BATCH_ID_BYTES: usize = 512; +const MAX_BATCH_ITEM_BYTES: usize = 2 * 1024 * 1024; +const MAX_BATCH_RESPONSE_BYTES: usize = 16 * 1024 * 1024; + +/// Failure returned by a bounded text batch read. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum BatchTextReadError { + /// The caller supplied an invalid identifier list. + InvalidRequest(String), + /// The caller exceeded the positional item budget. + TooManyItems { items: usize }, + /// One item cannot safely be represented by this buffered JSON endpoint. + ItemTooLarge { id: String, bytes: usize }, + /// The aggregate buffered response would exceed its byte budget. + ResponseTooLarge { bytes: usize }, + /// A projection or blob backend failed. + Storage(String), +} + +impl std::fmt::Display for BatchTextReadError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::InvalidRequest(message) | Self::Storage(message) => formatter.write_str(message), + Self::TooManyItems { items } => write!( + formatter, + "batch contains {items} items; maximum is {MAX_BATCH_ITEMS}" + ), + Self::ItemTooLarge { id, bytes } => write!( + formatter, + "item {id:?} is {bytes} bytes; buffered batch items are limited to {MAX_BATCH_ITEM_BYTES} bytes" + ), + Self::ResponseTooLarge { bytes } => write!( + formatter, + "batch text is {bytes} bytes; buffered batch responses are limited to {MAX_BATCH_RESPONSE_BYTES} bytes" + ), + } + } +} + +/// Validate one positional batch without silently changing its result shape. +pub(crate) fn validate_batch_text_ids(ids: &[String]) -> Result<(), BatchTextReadError> { + if ids.len() > MAX_BATCH_ITEMS { + return Err(BatchTextReadError::TooManyItems { items: ids.len() }); + } + + let mut seen = BTreeSet::new(); + for id in ids { + if id.trim().is_empty() { + return Err(BatchTextReadError::InvalidRequest( + "batch identifiers must not be empty".to_string(), + )); + } + if id.len() > MAX_BATCH_ID_BYTES { + return Err(BatchTextReadError::InvalidRequest(format!( + "batch identifier is {} bytes; maximum is {MAX_BATCH_ID_BYTES}", + id.len() + ))); + } + if !seen.insert(id.as_str()) { + return Err(BatchTextReadError::InvalidRequest(format!( + "duplicate batch identifier {id:?}" + ))); + } + } + Ok(()) +} + +impl ServerState { + #[instrument(skip_all, fields( + otel.name = "state.read_file_texts_batch", + tenant = %tenant, + file_count = file_ids.len(), + ))] + pub(crate) async fn read_file_texts_batch( + &self, + tenant: &TenantId, + file_ids: &[String], + ) -> Result, BatchTextReadError> { + validate_batch_text_ids(file_ids)?; + let metadata = self + .load_file_projection_metadata_batch(tenant, file_ids) + .await + .map_err(BatchTextReadError::Storage)?; + let mut response_bytes = 0usize; + let mut files = Vec::with_capacity(file_ids.len()); + for file_id in file_ids { + let result = self + .read_single_file_text(tenant, file_id.clone(), metadata.get(file_id).cloned()) + .await?; + consume_response_budget(file_id, result.text.len(), &mut response_bytes)?; + files.push(result); + } + Ok(files) + } + + #[instrument(skip_all, fields( + otel.name = "state.read_file_version_texts_batch", + tenant = %tenant, + file_version_count = file_version_ids.len(), + ))] + pub(crate) async fn read_file_version_texts_batch( + &self, + tenant: &TenantId, + file_version_ids: &[String], + ) -> Result, BatchTextReadError> { + validate_batch_text_ids(file_version_ids)?; + let metadata = self + .load_file_version_projection_metadata_batch(tenant, file_version_ids) + .await + .map_err(BatchTextReadError::Storage)?; + let mut response_bytes = 0usize; + let mut files = Vec::with_capacity(file_version_ids.len()); + for file_version_id in file_version_ids { + let result = self + .read_single_file_version_text( + tenant, + file_version_id.clone(), + metadata.get(file_version_id).cloned(), + ) + .await?; + consume_response_budget(file_version_id, result.text.len(), &mut response_bytes)?; + files.push(result); + } + Ok(files) + } + + async fn read_single_file_text( + &self, + tenant: &TenantId, + file_id: String, + meta: Option, + ) -> Result { + let Some(meta) = meta else { + return Ok(TextFileReadResult { + file_id, + found: false, + content_hash: String::new(), + mime_type: String::new(), + text: String::new(), + }); + }; + if !meta.has_content || meta.content_hash.is_empty() { + return Ok(TextFileReadResult { + file_id, + found: true, + content_hash: meta.content_hash, + mime_type: meta.mime_type, + text: String::new(), + }); + } + let text = self + .fetch_blob_text_for_hash(tenant, &meta.content_hash) + .await + .map_err(BatchTextReadError::Storage)? + .unwrap_or_default(); + Ok(TextFileReadResult { + file_id, + found: true, + content_hash: meta.content_hash, + mime_type: meta.mime_type, + text, + }) + } + + async fn read_single_file_version_text( + &self, + tenant: &TenantId, + file_version_id: String, + meta: Option, + ) -> Result { + let Some(meta) = meta else { + return Ok(TextFileVersionReadResult { + file_version_id, + found: false, + content_hash: String::new(), + mime_type: String::new(), + text: String::new(), + }); + }; + if !meta.has_content || meta.content_hash.is_empty() { + return Ok(TextFileVersionReadResult { + file_version_id, + found: true, + content_hash: meta.content_hash, + mime_type: meta.mime_type, + text: String::new(), + }); + } + let text = self + .fetch_blob_text_for_hash(tenant, &meta.content_hash) + .await + .map_err(BatchTextReadError::Storage)? + .unwrap_or_default(); + Ok(TextFileVersionReadResult { + file_version_id, + found: true, + content_hash: meta.content_hash, + mime_type: meta.mime_type, + text, + }) + } +} + +fn consume_response_budget( + id: &str, + item_bytes: usize, + response_bytes: &mut usize, +) -> Result<(), BatchTextReadError> { + if item_bytes > MAX_BATCH_ITEM_BYTES { + return Err(BatchTextReadError::ItemTooLarge { + id: id.to_string(), + bytes: item_bytes, + }); + } + *response_bytes = response_bytes.saturating_add(item_bytes); + if *response_bytes > MAX_BATCH_RESPONSE_BYTES { + return Err(BatchTextReadError::ResponseTooLarge { + bytes: *response_bytes, + }); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rejects_duplicate_and_oversized_identifier_sets() { + assert!(matches!( + validate_batch_text_ids(&["same".to_string(), "same".to_string()]), + Err(BatchTextReadError::InvalidRequest(_)) + )); + assert!(matches!( + validate_batch_text_ids(&vec!["id".to_string(); MAX_BATCH_ITEMS + 1]), + Err(BatchTextReadError::TooManyItems { .. }) + )); + } + + #[test] + fn response_budget_rejects_large_items_and_aggregate_overflow() { + let mut used = 0; + assert!(matches!( + consume_response_budget("large", MAX_BATCH_ITEM_BYTES + 1, &mut used), + Err(BatchTextReadError::ItemTooLarge { .. }) + )); + used = MAX_BATCH_RESPONSE_BYTES; + assert!(matches!( + consume_response_budget("next", 1, &mut used), + Err(BatchTextReadError::ResponseTooLarge { .. }) + )); + } +} diff --git a/crates/temper-server/src/state/file_writes.rs b/crates/temper-server/src/state/file_writes.rs index 0e9cfe873..1c59024f1 100644 --- a/crates/temper-server/src/state/file_writes.rs +++ b/crates/temper-server/src/state/file_writes.rs @@ -3,7 +3,17 @@ use std::sync::{Arc, RwLock}; use sha2::{Digest, Sha256}; use temper_wasm::{StreamRegistry, WasmInvocationContext}; -use super::{DispatchExtOptions, ServerState}; +use super::{DispatchCommand, DispatchExtOptions, ServerState}; + +struct FileStreamAction<'a> { + tenant: &'a temper_runtime::tenant::TenantId, + file_id: &'a str, + action: &'a str, + params: serde_json::Value, + agent_ctx: &'a crate::request_context::AgentContext, + await_reactions: bool, + expected_authorization_precondition: Option, +} /// Error returned by the native/File `$value` content upload path. #[derive(Debug, Clone, PartialEq, Eq)] @@ -84,7 +94,7 @@ impl ServerState { mime_type: &str, agent_ctx: &crate::request_context::AgentContext, ) -> Result { - self.put_file_stream_content_checked(tenant, file_id, body, mime_type, agent_ctx) + self.put_file_stream_content_checked(tenant, file_id, body, mime_type, agent_ctx, None) .await .map_err(|error| error.to_string()) } @@ -104,13 +114,21 @@ impl ServerState { body: &[u8], mime_type: &str, agent_ctx: &crate::request_context::AgentContext, + expected_authorization_precondition: Option, ) -> Result { let blob_endpoint = self .secrets_vault .as_ref() .and_then(|vault| vault.get_secret(&tenant.to_string(), "blob_endpoint")); let native_result = self - .put_file_stream_content_native(tenant, file_id, body, mime_type, agent_ctx) + .put_file_stream_content_native( + tenant, + file_id, + body, + mime_type, + agent_ctx, + expected_authorization_precondition.clone(), + ) .await; match native_result { Ok(response) => return Ok(response), @@ -129,8 +147,15 @@ impl ServerState { Err(error) => return Err(error), } - self.put_file_stream_content_via_wasm(tenant, file_id, body, mime_type, agent_ctx) - .await + self.put_file_stream_content_via_wasm( + tenant, + file_id, + body, + mime_type, + agent_ctx, + expected_authorization_precondition, + ) + .await } #[tracing::instrument(skip_all, fields( @@ -146,6 +171,7 @@ impl ServerState { body: &[u8], mime_type: &str, agent_ctx: &crate::request_context::AgentContext, + expected_authorization_precondition: Option, ) -> Result { let (content_hash, blob_key) = content_hash_and_native_blob_key(body); @@ -194,12 +220,11 @@ impl ServerState { .and_then(|value| value.as_str()) .unwrap_or_default(); let created_by = agent_ctx.agent_id.clone().unwrap_or_default(); - self.dispatch_tenant_action_ext_typed( + self.dispatch_file_stream_action(FileStreamAction { tenant, - "File", file_id, - "StreamUpdated", - serde_json::json!({ + action: "StreamUpdated", + params: serde_json::json!({ "content_hash": content_hash, "size_bytes": body.len() as i64, "mime_type": mime_type, @@ -207,12 +232,10 @@ impl ServerState { "previous_version_id": previous_version_id, "created_by": created_by, }), - DispatchExtOptions { - agent_ctx, - await_integration: false, - await_reactions: false, - }, - ) + agent_ctx, + await_reactions: false, + expected_authorization_precondition, + }) .await .map_err(|error| FileStreamContentError::ActionRejected(error.to_string())) } @@ -230,6 +253,7 @@ impl ServerState { body: &[u8], mime_type: &str, agent_ctx: &crate::request_context::AgentContext, + expected_authorization_precondition: Option, ) -> Result { let mut entity_state = serde_json::to_value( &self @@ -283,8 +307,13 @@ impl ServerState { http_request: None, }; + let security_ctx = agent_ctx.security_ctx.as_ref().ok_or_else(|| { + FileStreamContentError::Wasm( + "blob_adapter requires the caller's authenticated security context".to_string(), + ) + })?; let wasm_result = self - .invoke_wasm_direct(tenant, "blob_adapter", inv_ctx, streams) + .invoke_wasm_direct(tenant, "blob_adapter", inv_ctx, streams, security_ctx) .await .map_err(|e| FileStreamContentError::Wasm(format!("blob_adapter failed: {e}")))?; @@ -307,17 +336,58 @@ impl ServerState { }); } - self.dispatch_tenant_action( + self.dispatch_file_stream_action(FileStreamAction { tenant, - "File", file_id, - &wasm_result.callback_action, - wasm_result.callback_params, + action: &wasm_result.callback_action, + params: wasm_result.callback_params, agent_ctx, - ) + await_reactions: true, + expected_authorization_precondition, + }) .await .map_err(FileStreamContentError::ActionRejected) } + + async fn dispatch_file_stream_action( + &self, + request: FileStreamAction<'_>, + ) -> Result { + let options = DispatchExtOptions { + agent_ctx: request.agent_ctx, + await_integration: false, + await_reactions: request.await_reactions, + }; + match request.expected_authorization_precondition { + Some(expected) => self + .dispatch_tenant_action_ext_typed_if_current( + DispatchCommand { + tenant: request.tenant, + entity_type: "File", + entity_id: request.file_id, + action: request.action, + params: request.params, + agent_ctx: options.agent_ctx, + await_integration: options.await_integration, + await_reactions: options.await_reactions, + }, + expected, + ) + .await + .map_err(|error| error.to_string()), + None => self + .dispatch_tenant_action_ext_typed( + request.tenant, + "File", + request.file_id, + request.action, + request.params, + options, + ) + .await + .map_err(|error| error.to_string()), + } + } } pub(super) fn content_hash_and_native_blob_key(body: &[u8]) -> (String, String) { diff --git a/crates/temper-server/src/state/mod.rs b/crates/temper-server/src/state/mod.rs index e9cd9796d..89b0bdc38 100644 --- a/crates/temper-server/src/state/mod.rs +++ b/crates/temper-server/src/state/mod.rs @@ -26,8 +26,13 @@ pub mod trajectory; pub mod wasm_invocation_log; pub use admission::{AdmissionController, AdmissionOutcome, AdmissionPermit}; +pub(crate) use dispatch::authorized_http_endpoint_host; +#[cfg(feature = "observe")] +pub(crate) use dispatch::internal_http_capability_issuer; pub use dispatch::{DispatchCommand, DispatchError, DispatchExtOptions, StateTimeoutTracker}; pub use entity_ops::{FailedLevelInfo, VerificationGateError}; +#[cfg(feature = "observe")] +pub(crate) use file_reads::{BatchTextReadError, validate_batch_text_ids}; pub use file_reads::{IndexedFileStreamRead, TextFileReadResult, TextFileVersionReadResult}; pub(crate) use file_writes::FileStreamContentError; pub use metrics::MetricsCollector; @@ -38,6 +43,10 @@ pub use pending_decisions::{ pub use persistence::WasmModuleSource; pub use policy_suggestions::PolicySuggestionEngine; pub use published_artifacts::PublishFileArtifactRequest; +#[cfg(feature = "observe")] +pub(crate) use published_artifacts::{ + PUBLISH_ARTIFACT_STALE_AUTHORIZATION, PublishArtifactAuthorization, +}; pub(crate) use query_projection_queue::{ProjectionEnqueueOutcome, QueryProjectionWriteQueue}; pub use trajectory::{TrajectoryEntry, TrajectorySource}; pub use wasm_invocation_log::WasmInvocationEntry; @@ -64,6 +73,7 @@ use crate::adapters::AdapterRegistry; use crate::entity_actor::{EntityMsg, SnapshotWriteQueue}; use crate::events::EntityStateChange; use crate::idempotency::IdempotencyCache; +use crate::internal_invocation::InternalInvocationCredentialStore; use crate::ots_trajectory_outbox::OtsTrajectoryOutbox; use crate::registry::SpecRegistry; use crate::secrets::vault::SecretsVault; @@ -286,6 +296,33 @@ fn env_local_tdata_hosts() -> BTreeSet { hosts } +/// Tenants that have opted into exporting raw LLM content to telemetry, loaded +/// once at startup from `TEMPER_LLM_CONTENT_EXPORT_TENANTS` (comma-separated +/// tenant ids; the literal `*` opts in every tenant). Empty by default, so +/// content is redacted for every tenant unless explicitly opted in. See +/// ADR-0166. +fn env_llm_content_export_tenants() -> BTreeSet { + let configured = std::env::var("TEMPER_LLM_CONTENT_EXPORT_TENANTS"); // determinism-ok: read once at startup + parse_llm_content_export_tenants(configured.as_deref().unwrap_or("")) +} + +/// Parse the opt-in list. Split out from the env read so the policy is testable +/// without mutating process environment (which races across parallel tests). +fn parse_llm_content_export_tenants(raw: &str) -> BTreeSet { + raw.split(',') + .map(str::trim) + .filter(|item| !item.is_empty()) + .map(str::to_string) + .collect() +} + +/// Whether `tenant` is opted into raw LLM content export. Redact-by-default: an +/// empty set exports for nobody. Split out from [`ServerState::export_llm_content`] +/// so the decision can be tested without building a whole server state. +fn tenant_exports_llm_content(opted_in: &BTreeSet, tenant: &str) -> bool { + opted_in.contains("*") || opted_in.contains(tenant) +} + fn state_cache_budget() -> usize { static STATE_CACHE_BUDGET: OnceLock = OnceLock::new(); *STATE_CACHE_BUDGET.get_or_init(|| env_usize("TEMPER_STATE_CACHE_BUDGET", 10_000)) @@ -348,6 +385,13 @@ pub struct QueryProjectionReplayParityDrift { #[derive(Clone)] // ADR-0025 Phase 4: remove record_store field after IOA entity migration complete pub struct ServerState { + /// Capture losses this server could not record against any session. + /// + /// Read by conformance checking: a non-zero count means some stored + /// session is missing rows and nothing durable says which, so no report + /// from this server can claim to have seen a whole run. + pub(crate) capture_health: crate::trajectory_outbox::CaptureHealth, + /// The actor system for spawning and managing legacy in-memory entity actors. pub actor_system: Arc, /// Optional PG-backed actor system. When configured and an entity type is in @@ -382,7 +426,7 @@ pub struct ServerState { /// Runtime data directory for persisted local metadata (e.g. specs registry). pub data_dir: std::path::PathBuf, /// Agent hints learned from trajectory analysis, keyed by action name. - pub agent_hints: Arc>>, + pub agent_hints: Arc>>>, /// Cedar ABAC authorization engine. pub authz: Arc, /// Multi-tenant specification registry (shared, mutable for live registration). @@ -459,11 +503,17 @@ pub struct ServerState { pub eventual_tracker: Arc>, /// Idempotency cache for deduplicating agent retries. pub idempotency_cache: Arc, + /// Bounded, single-use credentials for authenticated internal HTTP re-entry. + pub internal_invocation_credentials: InternalInvocationCredentialStore, /// Optional encrypted secrets vault for per-tenant secret management. /// Broadcast channel for new pending decisions (SSE subscriptions). pub pending_decision_tx: Arc>, /// Per-tenant Cedar policy text (tenant -> policy text). pub tenant_policies: Arc>>, + /// Serializes approval commit/activation so concurrent approvals cannot + /// replace one another with policy text derived from a stale cache. + #[cfg(feature = "observe")] + pub(crate) policy_approval_lock: Arc>, /// Tenants installed in commons mode. Collection creates for these tenants /// must pass Cedar so commons guardrail forbids apply to direct OData /// writes as well as bound actions and composite sub-writes. @@ -480,6 +530,9 @@ pub struct ServerState { /// writes; storage cap enforcement prefers freshness over narrow retention. pub(crate) commons_storage_projection_cache: Arc>>, + /// Pending owner-byte reservations held by admitted raw Blob uploads. + commons_storage_reservations: + Arc>>, /// Coarse commons-mode write guardrail lock. /// /// Held from preflight through persistence for commons writes so exact @@ -487,6 +540,9 @@ pub struct ServerState { /// between "check" and "write" while cross-actor transactions are still /// being built out. pub(crate) commons_write_guardrail_lock: Arc>, + /// Weighted declared-byte admission for disk-backed raw Blob ingest. + /// State ownership permits deterministic capacity injection in simulation. + pub(crate) raw_blob_ingest_budget: crate::blob_store::BlobIngestBudget, pub secrets_vault: Option>, /// Broadcast channel for agent progress events (SSE subscriptions). /// // determinism-ok: broadcast channel for external observation only @@ -538,6 +594,12 @@ pub struct ServerState { /// Public hostnames owned by this process that may use in-process TData /// dispatch from WASM guests instead of leaving through the public edge. pub(crate) local_tdata_hosts: Arc>, + /// Tenants that have opted into exporting raw LLM content (prompts, + /// completions, system instructions, tool arguments/results) to the + /// telemetry backend. Loaded once from `TEMPER_LLM_CONTENT_EXPORT_TENANTS`. + /// Empty by default: every tenant is redacted unless explicitly opted in. + /// See ADR-0166. + pub(crate) llm_content_export_tenants: Arc>, } /// Install a one-time hook so liveness violations surfaced by temper-spec @@ -561,6 +623,16 @@ impl ServerState { } } + /// Whether `tenant` may export raw LLM content (prompts, completions, + /// system instructions, tool arguments/results) to the telemetry backend. + /// + /// Redact-by-default: content is only exported for tenants listed in + /// `TEMPER_LLM_CONTENT_EXPORT_TENANTS` (or when it contains the wildcard + /// `*`). Every other tenant is redacted. See ADR-0166. + pub fn export_llm_content(&self, tenant: &str) -> bool { + tenant_exports_llm_content(&self.llm_content_export_tenants, tenant) + } + /// Whether commons-mode write guardrails are active for a tenant. pub fn commons_guardrails_enabled(&self, tenant: &TenantId) -> bool { self.commons_guardrail_tenants @@ -663,6 +735,7 @@ impl ServerState { let (agent_progress_tx, _) = tokio::sync::broadcast::channel(256); // determinism-ok: broadcast for external observation let (observe_refresh_tx, _) = tokio::sync::broadcast::channel(64); // determinism-ok: broadcast for external observation let state = Self { + capture_health: crate::trajectory_outbox::CaptureHealth::default(), actor_system: Arc::new(system), pg_actor_system: None, actor_backed_types: BTreeSet::new(), @@ -678,7 +751,10 @@ impl ServerState { ots_trajectory_outbox: Arc::new(Mutex::new(None)), data_dir: std::path::PathBuf::new(), agent_hints: Arc::new(RwLock::new(BTreeMap::new())), - authz: Arc::new(AuthzEngine::permissive()), + // Network and tenant-scoped authorization starts fail-closed. + // Tests/development that intentionally need a permissive tenant + // must install that tenant policy explicitly (ARN-230). + authz: Arc::new(AuthzEngine::empty()), registry: Arc::new(RwLock::new(SpecRegistry::new())), entity_index: Arc::new(RwLock::new(BTreeMap::new())), entity_index_hydrated: Arc::new(RwLock::new(BTreeSet::new())), @@ -708,12 +784,17 @@ impl ServerState { crate::eventual_invariants::EventualInvariantTracker::new(), )), idempotency_cache: Arc::new(IdempotencyCache::new()), + internal_invocation_credentials: InternalInvocationCredentialStore::runtime(), pending_decision_tx: Arc::new(pending_decision_tx), tenant_policies: Arc::new(RwLock::new(BTreeMap::new())), + #[cfg(feature = "observe")] + policy_approval_lock: Arc::new(tokio::sync::Mutex::new(())), commons_guardrail_tenants: Arc::new(RwLock::new(BTreeSet::new())), commons_rate_limit_buckets: Arc::new(Mutex::new(BTreeMap::new())), commons_storage_projection_cache: Arc::new(Mutex::new(BTreeMap::new())), + commons_storage_reservations: Arc::new(Mutex::new(BTreeMap::new())), commons_write_guardrail_lock: Arc::new(tokio::sync::Mutex::new(())), + raw_blob_ingest_budget: crate::blob_store::BlobIngestBudget::runtime(), secrets_vault: None, agent_progress_tx: Arc::new(agent_progress_tx), // determinism-ok: broadcast for external observation entity_event_sequences: Arc::new(Mutex::new(BTreeMap::new())), @@ -729,6 +810,7 @@ impl ServerState { http_stream_registry: Arc::new(temper_wasm::http_stream::HttpStreamRegistry::new()), workflow_spans: Arc::new(crate::workflow_tracing::WorkflowSpanRegistry::default()), local_tdata_hosts: Arc::new(env_local_tdata_hosts()), + llm_content_export_tenants: Arc::new(env_llm_content_export_tenants()), }; // Pre-register built-in WASM modules (http_fetch for generic HTTP integrations). @@ -908,6 +990,7 @@ impl ServerState { let (agent_progress_tx, _) = tokio::sync::broadcast::channel(256); // determinism-ok: broadcast for external observation let (observe_refresh_tx, _) = tokio::sync::broadcast::channel(64); // determinism-ok: broadcast for external observation let state = Self { + capture_health: crate::trajectory_outbox::CaptureHealth::default(), actor_system: Arc::new(system), pg_actor_system: None, actor_backed_types: BTreeSet::new(), @@ -926,7 +1009,9 @@ impl ServerState { ots_trajectory_outbox: Arc::new(Mutex::new(None)), data_dir: std::path::PathBuf::new(), agent_hints: Arc::new(RwLock::new(BTreeMap::new())), - authz: Arc::new(AuthzEngine::permissive()), + // Missing tenant policy state is never an implicit permit-all + // compatibility mode (ARN-230). + authz: Arc::new(AuthzEngine::empty()), registry, entity_index: Arc::new(RwLock::new(BTreeMap::new())), entity_index_hydrated: Arc::new(RwLock::new(BTreeSet::new())), @@ -956,12 +1041,17 @@ impl ServerState { crate::eventual_invariants::EventualInvariantTracker::new(), )), idempotency_cache: Arc::new(IdempotencyCache::new()), + internal_invocation_credentials: InternalInvocationCredentialStore::runtime(), pending_decision_tx: Arc::new(pending_decision_tx), tenant_policies: Arc::new(RwLock::new(BTreeMap::new())), + #[cfg(feature = "observe")] + policy_approval_lock: Arc::new(tokio::sync::Mutex::new(())), commons_guardrail_tenants: Arc::new(RwLock::new(BTreeSet::new())), commons_rate_limit_buckets: Arc::new(Mutex::new(BTreeMap::new())), commons_storage_projection_cache: Arc::new(Mutex::new(BTreeMap::new())), + commons_storage_reservations: Arc::new(Mutex::new(BTreeMap::new())), commons_write_guardrail_lock: Arc::new(tokio::sync::Mutex::new(())), + raw_blob_ingest_budget: crate::blob_store::BlobIngestBudget::runtime(), secrets_vault: None, agent_progress_tx: Arc::new(agent_progress_tx), // determinism-ok: broadcast for external observation entity_event_sequences: Arc::new(Mutex::new(BTreeMap::new())), @@ -977,6 +1067,7 @@ impl ServerState { http_stream_registry: Arc::new(temper_wasm::http_stream::HttpStreamRegistry::new()), workflow_spans: Arc::new(crate::workflow_tracing::WorkflowSpanRegistry::default()), local_tdata_hosts: Arc::new(env_local_tdata_hosts()), + llm_content_export_tenants: Arc::new(env_llm_content_export_tenants()), }; state.register_builtin_wasm_modules(); state @@ -1255,8 +1346,17 @@ impl ServerState { http_request: None, }; + let security_ctx = agent_ctx.security_ctx.as_ref().ok_or_else(|| { + "blob_adapter requires the caller's authenticated security context".to_string() + })?; let wasm_result = self - .invoke_wasm_direct(tenant, "blob_adapter", inv_ctx, streams.clone()) + .invoke_wasm_direct( + tenant, + "blob_adapter", + inv_ctx, + streams.clone(), + security_ctx, + ) .await .map_err(|e| format!("blob_adapter download failed: {e}"))?; @@ -1292,36 +1392,31 @@ impl ServerState { None } - /// Load aggregated unmet-intent failure groups from durable metadata stores. - /// - /// Uses fan-out across all tenant stores in TenantRouted mode. - /// Returns an empty vec when Turso is not configured. + /// Load aggregated unmet-intent failure groups for one tenant. pub async fn load_unmet_intent_rows_aggregated( &self, + tenant: &str, ) -> ( Vec, std::collections::BTreeMap, ) { - let stores = self.collect_all_metadata_stores().await; - if stores.is_empty() { + let Some(store) = self.metadata_store_for_tenant(tenant).await else { return (Vec::new(), std::collections::BTreeMap::new()); - } + }; let mut failures = Vec::new(); let mut submitted_specs = std::collections::BTreeMap::new(); - for store in &stores { - match store.load_unmet_intent_rows().await { - Ok(rows) => failures.extend(rows), - Err(e) => { - tracing::warn!(error = %e, backend = store.backend_name(), "failed to load unmet intent rows"); - } + match store.load_unmet_intent_rows(tenant).await { + Ok(rows) => failures.extend(rows), + Err(e) => { + tracing::warn!(error = %e, backend = store.backend_name(), tenant, "failed to load unmet intent rows"); } - match store.load_submit_spec_timestamps().await { - Ok(map) => submitted_specs.extend(map), - Err(e) => { - tracing::warn!(error = %e, backend = store.backend_name(), "failed to load submit-spec timestamps"); - } + } + match store.load_submit_spec_timestamps(tenant).await { + Ok(map) => submitted_specs.extend(map), + Err(e) => { + tracing::warn!(error = %e, backend = store.backend_name(), tenant, "failed to load submit-spec timestamps"); } } (failures, submitted_specs) @@ -1352,50 +1447,46 @@ impl ServerState { counts } - /// Load trajectory entries from durable metadata stores. - /// - /// Uses fan-out across all tenant stores in TenantRouted mode. - pub async fn load_trajectory_entries(&self, limit: i64) -> Vec { - let stores = self.collect_all_metadata_stores().await; - if stores.is_empty() { + /// Load trajectory entries owned by one tenant. + pub async fn load_trajectory_entries(&self, tenant: &str, limit: i64) -> Vec { + let Some(store) = self.metadata_store_for_tenant(tenant).await else { return Vec::new(); - } + }; let mut all_entries = Vec::new(); - for store in &stores { - match store.load_recent_trajectories(limit).await { - Ok(rows) => { - all_entries.extend(rows.into_iter().map(|r| TrajectoryEntry { - timestamp: r.created_at, - tenant: r.tenant, - entity_type: r.entity_type, - entity_id: r.entity_id, - action: r.action, - success: r.success, - from_status: r.from_status, - to_status: r.to_status, - error: r.error, - agent_id: r.agent_id, - session_id: r.session_id, - authz_denied: r.authz_denied, - denied_resource: r.denied_resource, - denied_module: r.denied_module, - source: r.source.as_deref().and_then(|s| match s { - "Entity" => Some(TrajectorySource::Entity), - "Platform" => Some(TrajectorySource::Platform), - "Authz" => Some(TrajectorySource::Authz), - _ => None, - }), - spec_governed: r.spec_governed, - agent_type: None, - request_body: r.request_body.and_then(|s| serde_json::from_str(&s).ok()), - intent: r.intent, - matched_policy_ids: r.matched_policy_ids, - })); - } - Err(e) => { - tracing::warn!(error = %e, backend = store.backend_name(), "failed to load trajectories"); - } + match store.load_recent_trajectories(tenant, limit).await { + Ok(rows) => { + all_entries.extend(rows.into_iter().map(|r| TrajectoryEntry { + timestamp: r.created_at, + tenant: r.tenant, + entity_type: r.entity_type, + entity_id: r.entity_id, + action: r.action, + success: r.success, + from_status: r.from_status, + to_status: r.to_status, + error: r.error, + agent_id: r.agent_id, + session_id: r.session_id, + authz_denied: r.authz_denied, + denied_resource: r.denied_resource, + denied_module: r.denied_module, + source: r.source.as_deref().and_then(|s| match s { + "Entity" => Some(TrajectorySource::Entity), + "Platform" => Some(TrajectorySource::Platform), + "Authz" => Some(TrajectorySource::Authz), + _ => None, + }), + spec_governed: r.spec_governed, + agent_type: None, + request_body: r.request_body.and_then(|s| serde_json::from_str(&s).ok()), + intent: r.intent, + matched_policy_ids: r.matched_policy_ids, + capture_seq: r.capture_seq, + })); + } + Err(e) => { + tracing::warn!(error = %e, backend = store.backend_name(), tenant, "failed to load trajectories"); } } // Sort by timestamp descending and limit @@ -1435,7 +1526,58 @@ impl ServerState { #[cfg(test)] mod tests { - use super::normalize_local_tdata_host; + use super::{ + normalize_local_tdata_host, parse_llm_content_export_tenants, tenant_exports_llm_content, + }; + + /// The redaction gate is only as good as the policy that opens it. An empty + /// or unset opt-in list must export for nobody — a bug that flipped this to + /// default-allow would leak every tenant's prompts while every redaction unit + /// test stayed green. + #[test] + fn llm_content_export_is_deny_by_default() { + let unset = parse_llm_content_export_tenants(""); + assert!(unset.is_empty()); + assert!(!tenant_exports_llm_content(&unset, "acme")); + assert!(!tenant_exports_llm_content(&unset, "")); + + // Whitespace/empty entries must not be read as a tenant that opts in. + let blank = parse_llm_content_export_tenants(" , ,\t,"); + assert!(blank.is_empty(), "got {blank:?}"); + assert!(!tenant_exports_llm_content(&blank, "acme")); + } + + #[test] + fn llm_content_export_opts_in_only_listed_tenants() { + let set = parse_llm_content_export_tenants(" acme , globex "); + assert!( + tenant_exports_llm_content(&set, "acme"), + "trimmed entry opts in" + ); + assert!(tenant_exports_llm_content(&set, "globex")); + assert!( + !tenant_exports_llm_content(&set, "initech"), + "an unlisted tenant must stay redacted" + ); + // Not a prefix/substring match: neighbours of a listed tenant stay out. + assert!(!tenant_exports_llm_content(&set, "acme2")); + assert!(!tenant_exports_llm_content(&set, "acm")); + assert!( + !tenant_exports_llm_content(&set, "ACME"), + "match is case-sensitive" + ); + } + + #[test] + fn llm_content_export_wildcard_opts_in_every_tenant() { + let all = parse_llm_content_export_tenants("*"); + assert!(tenant_exports_llm_content(&all, "acme")); + assert!(tenant_exports_llm_content(&all, "anything-else")); + + // The wildcard still applies when it arrives alongside named tenants. + let mixed = parse_llm_content_export_tenants("acme,*"); + assert!(tenant_exports_llm_content(&mixed, "initech")); + } #[test] fn normalize_local_tdata_host_accepts_urls_domains_and_ports() { diff --git a/crates/temper-server/src/state/persistence/logs_and_secrets.rs b/crates/temper-server/src/state/persistence/logs_and_secrets.rs index 9d1ebf1db..b8c5aa70d 100644 --- a/crates/temper-server/src/state/persistence/logs_and_secrets.rs +++ b/crates/temper-server/src/state/persistence/logs_and_secrets.rs @@ -126,6 +126,60 @@ impl ServerState { Ok(()) } + /// Whether `session_id` is a server-validated session grant for `agent_id`. + /// + /// True only when an APPROVED decision in `tenant` carries an approved scope + /// with `duration = session`, the same session id, and the same agent — a + /// human explicitly approved this principal for this session. This is the + /// server-side record that lets a caller-asserted session header become a + /// Cedar input (ADR-0157); without it the header stays telemetry, so a + /// session-scoped permit can only ever match the principal it was approved + /// for. Fails closed: no backend or a storage error means "not verified". + pub async fn session_grant_verified( + &self, + tenant: &str, + agent_id: &str, + session_id: &str, + ) -> bool { + let Some(backend) = self.tenant_metadata_backend(tenant).await else { + return false; + }; + let blobs = match backend { + TenantMetadataBackend::Postgres(pool) => { + temper_store_postgres::PostgresEventStore::new(pool) + .load_approved_session_decisions(tenant, session_id) + .await + .map_err(|e| e.to_string()) + } + TenantMetadataBackend::Turso(turso) => turso + .load_approved_session_decisions(tenant, session_id) + .await + .map_err(|e| e.to_string()), + TenantMetadataBackend::Redis => { + Err(Self::redis_ephemeral_error("Session grant validation")) + } + }; + let blobs = match blobs { + Ok(blobs) => blobs, + Err(e) => { + tracing::warn!(tenant, session_id, error = %e, "session grant lookup failed; treating as unverified"); + return false; + } + }; + blobs.iter().any(|blob| { + serde_json::from_str::(blob) + .map(|d| { + d.status == super::super::DecisionStatus::Approved + && d.agent_id == agent_id + && d.approved_scope.as_ref().is_some_and(|scope| { + scope.duration == temper_authz::DurationScope::Session + && scope.session_id.as_deref() == Some(session_id) + }) + }) + .unwrap_or(false) + }) + } + /// Upsert an encrypted secret in the persistence backend. pub async fn upsert_secret( &self, @@ -323,3 +377,142 @@ mod tests { let _ = std::fs::remove_file(db_path); // determinism-ok: test-only cleanup } } + +#[cfg(test)] +mod session_grant_tests { + use temper_authz::{DurationScope, PolicyScopeMatrix}; + use temper_runtime::ActorSystem; + + use crate::registry::SpecRegistry; + use crate::state::{DecisionStatus, PendingDecision, ServerState}; + use crate::storage::StorageStack; + + async fn state_with_turso(test_name: &str) -> ServerState { + let mut path = std::env::temp_dir(); + path.push(format!( + "temper-session-grant-{test_name}-{}.db", + uuid::Uuid::new_v4() + )); + let turso = + temper_store_turso::TursoEventStore::new(&format!("file:{}", path.display()), None) + .await + .expect("create local turso db"); + let mut state = + ServerState::from_registry(ActorSystem::new("session-grant-test"), SpecRegistry::new()); + state.set_storage_stack(StorageStack::from_turso(turso)); + state + } + + fn decision( + tenant: &str, + agent_id: &str, + status: DecisionStatus, + scope: Option, + ) -> PendingDecision { + let mut d = PendingDecision::from_denial( + tenant, + agent_id, + "Delete", + "Order", + "order-1", + serde_json::json!({}), + "denied by policy", + None, + ); + d.status = status; + d.approved_scope = scope; + d + } + + fn session_scope(session_id: &str) -> PolicyScopeMatrix { + let mut scope = PolicyScopeMatrix::default_for(Some("operator")); + scope.duration = DurationScope::Session; + scope.session_id = Some(session_id.to_string()); + scope + } + + /// The grant is exact: approved decision, same agent, same session, session + /// duration. Anything less must not turn a caller-asserted header into a + /// Cedar input (ADR-0157) — each negative arm below is one relaxation. + #[tokio::test] + async fn a_session_grant_binds_exactly_one_agent_and_session() { + let state = state_with_turso("exact-binding").await; + + state + .persist_pending_decision(&decision( + "default", + "agent-a", + DecisionStatus::Approved, + Some(session_scope("sess-approved")), + )) + .await + .expect("persist approved grant"); + + assert!( + state + .session_grant_verified("default", "agent-a", "sess-approved") + .await, + "the approved (agent, session) pair must verify" + ); + assert!( + !state + .session_grant_verified("default", "agent-b", "sess-approved") + .await, + "another agent asserting the approved session must not verify" + ); + assert!( + !state + .session_grant_verified("default", "agent-a", "sess-other") + .await, + "the approved agent asserting a different session must not verify" + ); + assert!( + !state + .session_grant_verified("other-tenant", "agent-a", "sess-approved") + .await, + "the grant must not verify outside its tenant" + ); + } + + #[tokio::test] + async fn an_unapproved_or_unscoped_decision_is_not_a_grant() { + let state = state_with_turso("not-a-grant").await; + + // Still pending: the human has not approved anything. + state + .persist_pending_decision(&decision( + "default", + "agent-a", + DecisionStatus::Pending, + Some(session_scope("sess-pending")), + )) + .await + .expect("persist pending decision"); + assert!( + !state + .session_grant_verified("default", "agent-a", "sess-pending") + .await, + "a pending decision must not act as a session grant" + ); + + // Approved, but not session-scoped: an Always-duration approval names no + // session, so no session assertion may borrow it. + let mut always = PolicyScopeMatrix::default_for(Some("operator")); + always.session_id = Some("sess-always".to_string()); + state + .persist_pending_decision(&decision( + "default", + "agent-a", + DecisionStatus::Approved, + Some(always), + )) + .await + .expect("persist always-duration decision"); + assert!( + !state + .session_grant_verified("default", "agent-a", "sess-always") + .await, + "an approval without session duration must not act as a session grant" + ); + } +} diff --git a/crates/temper-server/src/state/published_artifacts.rs b/crates/temper-server/src/state/published_artifacts.rs index 3f74b2213..023a32e27 100644 --- a/crates/temper-server/src/state/published_artifacts.rs +++ b/crates/temper-server/src/state/published_artifacts.rs @@ -1,4 +1,5 @@ -use reqwest::header::{AUTHORIZATION, CONTENT_TYPE, HeaderMap, HeaderValue}; +use reqwest::header::{CONTENT_TYPE, HeaderMap}; +use std::collections::BTreeMap; use std::sync::OnceLock; use temper_runtime::tenant::TenantId; use tracing::{Span, instrument}; @@ -12,6 +13,16 @@ use telemetry::{PublishedArtifactTelemetry, emit_published_artifact_persisted_lo mod telemetry; const DEFAULT_PUBLIC_ARTIFACT_NAMESPACE: &str = "published-artifacts"; +pub(crate) const PUBLISH_ARTIFACT_STALE_AUTHORIZATION: &str = + "publish source authorization became stale; retry against current state"; + +/// Exact source state/resource view used for a public-artifact Cedar decision. +pub(crate) struct PublishArtifactAuthorization { + pub source_entity_type: String, + pub source_entity_id: String, + pub state_precondition: String, + pub resource_attrs: BTreeMap, +} #[derive(Debug, Clone)] pub struct PublishFileArtifactRequest { @@ -23,7 +34,67 @@ pub struct PublishFileArtifactRequest { pub namespace: Option, } +impl PublishFileArtifactRequest { + /// Validate every caller-controlled object-key component as one segment. + pub fn validate(&self) -> Result<(), String> { + validate_path_segment("label", &self.label)?; + validate_path_segment("owner_ref_type", &self.owner_ref_type)?; + validate_path_segment("owner_ref_id", &self.owner_ref_id)?; + if let Some(namespace) = self.namespace.as_deref() { + validate_path_segment("namespace", namespace)?; + } + Ok(()) + } +} + impl ServerState { + /// Resolve the immutable parent File recorded by a FileVersion. + /// + /// The durable query projection is the fast path. Actor state is the + /// read-after-write fallback when the projection has not caught up yet. + #[cfg(feature = "observe")] + pub(crate) async fn file_version_source_file_id( + &self, + tenant: &TenantId, + file_version_id: &str, + ) -> Result { + if let Some(query_plane) = self.query_plane_store() { + let ids = [file_version_id.to_string()]; + let rows = query_plane + .load_projection_fields_many(tenant.as_str(), "FileVersion", &ids, &["file_id"]) + .await + .map_err(|error| { + format!("failed to load FileVersion '{file_version_id}' relationship: {error}") + })? + .unwrap_or_default(); + if let Some(file_id) = rows + .first() + .and_then(|row| row.fields.get("file_id")) + .and_then(Option::as_deref) + .filter(|file_id| !file_id.is_empty()) + { + return Ok(file_id.to_string()); + } + } + + let response = self + .get_tenant_entity_state(tenant, "FileVersion", file_version_id) + .await + .map_err(|error| { + format!("failed to load FileVersion '{file_version_id}' relationship: {error}") + })?; + response + .state + .fields + .get("file_id") + .and_then(serde_json::Value::as_str) + .filter(|file_id| !file_id.is_empty()) + .map(str::to_string) + .ok_or_else(|| { + format!("FileVersion '{file_version_id}' has no immutable file_id relationship") + }) + } + #[instrument(skip_all, fields( otel.name = "state.publish_file_artifact", tenant = %tenant, @@ -50,6 +121,19 @@ impl ServerState { tenant: &TenantId, request: PublishFileArtifactRequest, ) -> Result { + self.publish_file_artifact_authorized(tenant, request, Vec::new()) + .await + } + + /// Publish only while the source still matches the exact Cedar-authorized + /// state and derived resource attributes. + pub(crate) async fn publish_file_artifact_authorized( + &self, + tenant: &TenantId, + request: PublishFileArtifactRequest, + authorizations: Vec, + ) -> Result { + request.validate()?; let source_display = if request.source_file_version_id.trim().is_empty() { format!("File('{}')", request.file_id) } else { @@ -83,6 +167,26 @@ impl ServerState { } }; + for authorization in authorizations { + let snapshot = self + .load_authz_resource_snapshot( + tenant, + &authorization.source_entity_type, + &authorization.source_entity_id, + ) + .await + .map_err(|_| PUBLISH_ARTIFACT_STALE_AUTHORIZATION.to_string())?; + let current_precondition = + crate::entity_actor::effects::entity_authorization_precondition( + &snapshot.current_state.state, + ); + if current_precondition != authorization.state_precondition + || snapshot.resource_attrs != authorization.resource_attrs + { + return Err(PUBLISH_ARTIFACT_STALE_AUTHORIZATION.to_string()); + } + } + let public_base_url = self .secret(tenant, "published_blob_public_base_url") .ok_or_else(|| "missing published_blob_public_base_url secret".to_string())?; @@ -110,7 +214,7 @@ impl ServerState { &request.label, &content_hash, &mime_type, - ); + )?; put_public_blob( self, tenant, @@ -196,6 +300,32 @@ async fn put_public_blob( mime_type: &str, bytes: &[u8], ) -> Result<(), String> { + if crate::blob_store::is_local_internal_blob_endpoint(endpoint) { + let object_key = format!( + "{}/{}", + bucket.trim_matches('/'), + storage_key.trim_start_matches('/') + ); + state + .put_blob_object(tenant, &object_key, bytes, None) + .await + .map_err(|error| { + format!( + "direct tenant-scoped public blob write failed for bucket '{bucket}' key '{storage_key}': {error}" + ) + })?; + tracing::Span::current().record("http.status_code", 204_u16); + tracing::info!( + tenant = %tenant, + bucket, + storage_key, + mime_type = %stream_content_type(mime_type), + byte_length = bytes.len(), + "public blob stored through tenant-scoped local API" + ); + return Ok(()); + } + let url = format!( "{}/{}/{}", endpoint.trim_end_matches('/'), @@ -277,21 +407,7 @@ fn build_public_blob_put_headers( mime_type: &str, bytes: &[u8], ) -> Result { - let mut headers = HeaderMap::new(); - if crate::blob_store::is_local_internal_blob_endpoint(url) { - if let Some(api_key) = std::env::var("TEMPER_API_KEY") // determinism-ok: deployment config read - .ok() - .filter(|value| !value.trim().is_empty()) - { - headers.insert( - AUTHORIZATION, - HeaderValue::from_str(&format!("Bearer {api_key}")) - .map_err(|e| format!("invalid internal blob authorization header: {e}"))?, - ); - } - return Ok(headers); - } - + let headers = HeaderMap::new(); let access_key = state .secret(tenant, "published_blob_access_key") .or_else(|| state.secret(tenant, "blob_access_key")); @@ -326,17 +442,22 @@ fn public_storage_key( label: &str, content_hash: &str, mime_type: &str, -) -> String { +) -> Result { let hash = content_hash.trim_start_matches("sha256:"); - format!( + validate_path_segment("namespace", namespace)?; + validate_path_segment("owner_ref_type", owner_ref_type)?; + validate_path_segment("owner_ref_id", owner_ref_id)?; + validate_path_segment("label", label)?; + validate_path_segment("content_hash", hash)?; + Ok(format!( "{}/{}/{}/{}-{}.{}", - sanitize_path_segment(namespace).trim_matches('/'), - sanitize_path_segment(owner_ref_type), - sanitize_path_segment(owner_ref_id), - sanitize_path_segment(label), + namespace, + owner_ref_type, + owner_ref_id, + label, hash, extension_for_mime(mime_type) - ) + )) } fn published_artifact_id( @@ -351,16 +472,22 @@ fn published_artifact_id( format!("part-{}", &digest[..32]) } -fn sanitize_path_segment(value: &str) -> String { - let mut out = String::with_capacity(value.len()); - for ch in value.chars() { - if ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_' | '/' | '.') { - out.push(ch); - } else { - out.push('-'); - } +fn validate_path_segment(field: &str, value: &str) -> Result<(), String> { + if value.is_empty() { + return Err(format!("{field} must not be empty")); + } + if value == "." || value == ".." { + return Err(format!("{field} must not be a relative path segment")); } - out.trim_matches('-').to_string() + if !value + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_' | '.' | '~')) + { + return Err(format!( + "{field} must be one URI-safe path segment using only letters, digits, '-', '_', '.' or '~'" + )); + } + Ok(()) } fn extension_for_mime(mime_type: &str) -> &'static str { diff --git a/crates/temper-server/src/state/published_artifacts/tests.rs b/crates/temper-server/src/state/published_artifacts/tests.rs index e0ff786b1..f0f9c0df1 100644 --- a/crates/temper-server/src/state/published_artifacts/tests.rs +++ b/crates/temper-server/src/state/published_artifacts/tests.rs @@ -10,8 +10,8 @@ use tracing_subscriber::{Layer, registry::LookupSpan}; use crate::storage::PublishedArtifactStoreRow; use super::{ - PublishedArtifactTelemetry, emit_published_artifact_persisted_log, - public_blob_put_status_error, public_storage_key, published_artifact_id, + PublishFileArtifactRequest, PublishedArtifactTelemetry, emit_published_artifact_persisted_log, + public_blob_put_status_error, public_storage_key, published_artifact_id, validate_path_segment, }; #[derive(Clone, Default)] @@ -60,20 +60,67 @@ impl Visit for FieldVisitor { #[test] fn public_storage_key_is_generic_and_content_addressed() { let key = public_storage_key( - "public/demo artifacts", + "public-demo-artifacts", "Report", "quarterly-2026", - "preview image", + "preview-image", "sha256:abc123", "image/png", - ); + ) + .expect("valid path segments"); assert_eq!( key, - "public/demo-artifacts/Report/quarterly-2026/preview-image-abc123.png" + "public-demo-artifacts/Report/quarterly-2026/preview-image-abc123.png" ); } +#[test] +fn artifact_path_segments_reject_breakout_and_ambiguous_values() { + for value in ["", ".", "..", "a/b", r"a\b", "line\nbreak", "has space"] { + assert!( + validate_path_segment("label", value).is_err(), + "segment should be rejected: {value:?}" + ); + } + for value in ["artifact", "artifact_1", "Artifact-2026", "v1.2"] { + assert!( + validate_path_segment("label", value).is_ok(), + "segment should be accepted: {value:?}" + ); + } +} + +#[test] +fn publish_request_rejects_invalid_namespace_owner_and_label() { + let valid = PublishFileArtifactRequest { + file_id: "file-a".to_string(), + label: "latest".to_string(), + owner_ref_type: "Document".to_string(), + owner_ref_id: "doc-a".to_string(), + source_file_version_id: String::new(), + namespace: Some("published-artifacts".to_string()), + }; + assert!(valid.validate().is_ok()); + + for invalid in [ + PublishFileArtifactRequest { + namespace: Some("../escape".to_string()), + ..valid.clone() + }, + PublishFileArtifactRequest { + owner_ref_id: "owner/escape".to_string(), + ..valid.clone() + }, + PublishFileArtifactRequest { + label: "..".to_string(), + ..valid.clone() + }, + ] { + assert!(invalid.validate().is_err()); + } +} + #[test] fn published_artifact_id_uses_generic_owner_ref_label_and_hash() { let first = published_artifact_id( diff --git a/crates/temper-server/src/state/rate_limit.rs b/crates/temper-server/src/state/rate_limit.rs index 9d6b1bda6..2b44cb4b2 100644 --- a/crates/temper-server/src/state/rate_limit.rs +++ b/crates/temper-server/src/state/rate_limit.rs @@ -152,6 +152,7 @@ impl ServerState { }), &agent, false, + None, ) .await .map_err(rate_limit_dispatch_error); diff --git a/crates/temper-server/src/state/storage_caps.rs b/crates/temper-server/src/state/storage_caps.rs index 0e50e621f..747e66905 100644 --- a/crates/temper-server/src/state/storage_caps.rs +++ b/crates/temper-server/src/state/storage_caps.rs @@ -5,6 +5,11 @@ use temper_runtime::tenant::TenantId; use super::ServerState; +mod profile; +mod reservations; +use profile::OwnerStorageProfile; +pub(crate) use reservations::CommonsStorageReservationEntry; + const OWNER_ENTITY_TYPE: &str = "Owner"; const REPOSITORY_ENTITY_TYPE: &str = "Repository"; const BLOB_ENTITY_TYPE: &str = "Blob"; @@ -35,6 +40,7 @@ pub(crate) struct CommonsStorageCapExceeded { #[derive(Debug, Clone)] pub(crate) enum CommonsStorageCapError { Exceeded(CommonsStorageCapExceeded), + ReservationCapacityExhausted, OwnerSuspended(String), MissingAttribution(String), Internal(String), @@ -51,6 +57,9 @@ impl std::fmt::Display for CommonsStorageCapError { exceeded.additional_bytes, exceeded.cap_bytes ), + CommonsStorageCapError::ReservationCapacityExhausted => { + f.write_str("pending storage reservation capacity is exhausted") + } CommonsStorageCapError::OwnerSuspended(owner_id) => { write!(f, "owner '{owner_id}' is suspended") } @@ -62,13 +71,6 @@ impl std::fmt::Display for CommonsStorageCapError { impl std::error::Error for CommonsStorageCapError {} -#[derive(Debug, Clone)] -struct OwnerStorageProfile { - owner_id: String, - cap_bytes: i64, - suspended: bool, -} - impl ServerState { pub(crate) async fn acquire_commons_write_guardrail_lock( &self, @@ -205,11 +207,13 @@ impl ServerState { .get(&profile.owner_id) .copied() .unwrap_or(0); - if projection.used_bytes.saturating_add(additional_bytes) > projection.cap_bytes { + let reserved_bytes = self.pending_reserved_bytes(tenant, &profile.owner_id)?; + let effective_used = projection.used_bytes.saturating_add(reserved_bytes); + if effective_used.saturating_add(additional_bytes) > projection.cap_bytes { return Err(CommonsStorageCapError::Exceeded( CommonsStorageCapExceeded { owner_id: projection.owner_id, - used_bytes: projection.used_bytes, + used_bytes: effective_used, additional_bytes, cap_bytes: projection.cap_bytes, }, @@ -444,30 +448,6 @@ impl ServerState { } } -impl OwnerStorageProfile { - fn from_fields(entity_id: &str, fields: &Value) -> Result { - let owner_id = read_string(fields, "AccountId") - .or_else(|| read_string(fields, "Id")) - .unwrap_or_else(|| entity_id.to_string()); - let Some(cap_bytes) = read_i64(fields, "StorageCapBytes") else { - return Ok(Self { - owner_id, - cap_bytes: i64::MAX, - suspended: false, - }); - }; - let suspended = fields - .get("Status") - .and_then(|v| v.as_str()) - .is_some_and(|status| status == "Suspended"); - Ok(Self { - owner_id, - cap_bytes: cap_bytes.max(0), - suspended, - }) - } -} - fn is_create_action(action: &str) -> bool { action == "Create" } diff --git a/crates/temper-server/src/state/storage_caps/profile.rs b/crates/temper-server/src/state/storage_caps/profile.rs new file mode 100644 index 000000000..2f154ded9 --- /dev/null +++ b/crates/temper-server/src/state/storage_caps/profile.rs @@ -0,0 +1,37 @@ +use serde_json::Value; + +use super::{CommonsStorageCapError, read_i64, read_string}; + +#[derive(Debug, Clone)] +pub(super) struct OwnerStorageProfile { + pub(super) owner_id: String, + pub(super) cap_bytes: i64, + pub(super) suspended: bool, +} + +impl OwnerStorageProfile { + pub(super) fn from_fields( + entity_id: &str, + fields: &Value, + ) -> Result { + let owner_id = read_string(fields, "AccountId") + .or_else(|| read_string(fields, "Id")) + .unwrap_or_else(|| entity_id.to_string()); + let Some(cap_bytes) = read_i64(fields, "StorageCapBytes") else { + return Ok(Self { + owner_id, + cap_bytes: i64::MAX, + suspended: false, + }); + }; + let suspended = fields + .get("Status") + .and_then(|value| value.as_str()) + .is_some_and(|status| status == "Suspended"); + Ok(Self { + owner_id, + cap_bytes: cap_bytes.max(0), + suspended, + }) + } +} diff --git a/crates/temper-server/src/state/storage_caps/reservations.rs b/crates/temper-server/src/state/storage_caps/reservations.rs new file mode 100644 index 000000000..869e59463 --- /dev/null +++ b/crates/temper-server/src/state/storage_caps/reservations.rs @@ -0,0 +1,139 @@ +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; + +use temper_runtime::scheduler::sim_uuid; +use temper_runtime::tenant::TenantId; + +use super::{CommonsStorageCapError, CommonsStorageCapExceeded, CommonsStorageProjection}; +use crate::state::ServerState; + +const MAX_PENDING_STORAGE_RESERVATIONS: usize = 4096; + +#[derive(Debug, Clone)] +pub(crate) struct CommonsStorageReservationEntry { + tenant: String, + owner_id: String, + bytes: i64, +} + +/// RAII reservation for owner-attributed raw Blob bytes. +/// +/// Dropping it removes the pending bytes on success, failure, or cancelled +/// request futures. Persisted Blob metadata remains the durable source used by +/// the next storage projection. +pub(crate) struct CommonsStorageReservation { + id: String, + reservations: Arc>>, +} + +impl Drop for CommonsStorageReservation { + fn drop(&mut self) { + if let Ok(mut reservations) = self.reservations.lock() { + reservations.remove(&self.id); + } + } +} + +impl ServerState { + /// Reserve the exact owner-attributed bytes for an admitted raw Blob. + /// + /// The caller must hold [`Self::acquire_commons_write_guardrail_lock`] + /// while creating the reservation so the durable projection and pending + /// ledger form one admission snapshot. The returned guard can then live + /// across streaming I/O without holding the coarse mutation lock. + pub(crate) async fn reserve_commons_blob_storage( + &self, + tenant: &TenantId, + blob_id: &str, + repository_id: &str, + size_bytes: i64, + ) -> Result, CommonsStorageCapError> { + if !self.commons_guardrails_enabled(tenant) + || !self.storage_cap_entities_available(tenant)? + || size_bytes <= 0 + { + return Ok(None); + } + if self.blob_already_exists(tenant, blob_id).await { + return Ok(None); + } + + let owner_id = self + .repository_owner_id(tenant, repository_id) + .await? + .ok_or_else(|| { + CommonsStorageCapError::MissingAttribution(format!( + "Repository '{repository_id}' is required for commons storage attribution" + )) + })?; + let Some(profile) = self.owner_storage_profile(tenant, &owner_id).await? else { + return Ok(None); + }; + if profile.suspended { + return Err(CommonsStorageCapError::OwnerSuspended(profile.owner_id)); + } + let projection = self + .commons_storage_projection_for_owner(tenant, &owner_id) + .await? + .unwrap_or(CommonsStorageProjection { + owner_id: owner_id.clone(), + used_bytes: 0, + cap_bytes: profile.cap_bytes, + }); + let reserved_bytes = self.pending_reserved_bytes(tenant, &owner_id)?; + let effective_used = projection.used_bytes.saturating_add(reserved_bytes); + if effective_used.saturating_add(size_bytes) > projection.cap_bytes { + return Err(CommonsStorageCapError::Exceeded( + CommonsStorageCapExceeded { + owner_id, + used_bytes: effective_used, + additional_bytes: size_bytes, + cap_bytes: projection.cap_bytes, + }, + )); + } + + let reservation_id = sim_uuid().to_string(); + let mut reservations = self.commons_storage_reservations.lock().map_err(|error| { + CommonsStorageCapError::Internal(format!("storage reservation lock poisoned: {error}")) + })?; + if reservations.len() >= MAX_PENDING_STORAGE_RESERVATIONS { + return Err(CommonsStorageCapError::ReservationCapacityExhausted); + } + let previous = reservations.insert( + reservation_id.clone(), + CommonsStorageReservationEntry { + tenant: tenant.to_string(), + owner_id, + bytes: size_bytes, + }, + ); + debug_assert!( + previous.is_none(), + "sim_uuid reservation IDs must be unique" + ); + drop(reservations); + Ok(Some(CommonsStorageReservation { + id: reservation_id, + reservations: self.commons_storage_reservations.clone(), + })) + } + + pub(super) fn pending_reserved_bytes( + &self, + tenant: &TenantId, + owner_id: &str, + ) -> Result { + let reservations = self.commons_storage_reservations.lock().map_err(|error| { + CommonsStorageCapError::Internal(format!("storage reservation lock poisoned: {error}")) + })?; + Ok(reservations + .values() + .filter(|reservation| { + reservation.tenant == tenant.as_str() && reservation.owner_id == owner_id + }) + .fold(0i64, |total, reservation| { + total.saturating_add(reservation.bytes.max(0)) + })) + } +} diff --git a/crates/temper-server/src/state/trajectory.rs b/crates/temper-server/src/state/trajectory.rs index a2c026fa8..31159bb12 100644 --- a/crates/temper-server/src/state/trajectory.rs +++ b/crates/temper-server/src/state/trajectory.rs @@ -70,6 +70,16 @@ pub struct TrajectoryEntry { /// Cedar policy IDs that contributed to the authorization decision (allow or deny). #[serde(default, skip_serializing_if = "Option::is_none")] pub matched_policy_ids: Option>, + /// Position of this entry in the capturing process's capture order. + /// + /// Stamped by `ServerState::enqueue_trajectory_entry`, the one choke point + /// every capture site passes through; capture sites leave it `None`. + /// Persistence runs in independently spawned tasks, so the storage row id + /// records the order the writes landed rather than the order the kernel + /// captured them. The session read orders by this instead, so a replay + /// sees the run in the order it happened. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub capture_seq: Option, } /// Bounded, append-only trajectory log. diff --git a/crates/temper-server/src/storage/metadata_impls.rs b/crates/temper-server/src/storage/metadata_impls.rs new file mode 100644 index 000000000..30d289c62 --- /dev/null +++ b/crates/temper-server/src/storage/metadata_impls.rs @@ -0,0 +1,196 @@ +//! Backend adapters for observe and evolution metadata capabilities. + +use super::*; + +impl BackendNamedStore for PostgresEventStore { + fn backend_name(&self) -> &'static str { + "postgres" + } +} + +impl BackendNamedStore for TursoEventStore { + fn backend_name(&self) -> &'static str { + "turso" + } +} + +#[async_trait::async_trait] +impl EvolutionStore for PostgresEventStore { + async fn upsert_feature_request( + &self, + tenant: &str, + id: &str, + category: &str, + description: &str, + frequency: i64, + trajectory_refs_json: &str, + disposition: &str, + developer_notes: Option<&str>, + ) -> Result<(), PersistenceError> { + self.upsert_feature_request( + tenant, + id, + category, + description, + frequency, + trajectory_refs_json, + disposition, + developer_notes, + ) + .await + } + + async fn list_feature_requests( + &self, + tenant: &str, + disposition: Option<&str>, + ) -> Result, PersistenceError> { + self.list_feature_requests(tenant, disposition) + .await + .map(|rows| rows.into_iter().map(pg_feature_request_to_turso).collect()) + } + + async fn update_feature_request( + &self, + tenant: &str, + id: &str, + disposition: &str, + developer_notes: Option<&str>, + ) -> Result { + self.update_feature_request(tenant, id, disposition, developer_notes) + .await + } + + async fn insert_evolution_record( + &self, + record: EvolutionRecordWrite<'_>, + ) -> Result<(), PersistenceError> { + self.insert_evolution_record(PostgresEvolutionRecordInsert { + tenant: record.tenant, + id: record.id, + record_type: record.record_type, + status: record.status, + created_by: record.created_by, + derived_from: record.derived_from, + data_json: record.data_json, + }) + .await + } + + async fn get_evolution_record( + &self, + tenant: &str, + id: &str, + ) -> Result, PersistenceError> { + self.get_evolution_record(tenant, id) + .await + .map(|row| row.map(pg_evolution_record_to_turso)) + } + + async fn list_evolution_records( + &self, + tenant: &str, + record_type: Option<&str>, + status: Option<&str>, + ) -> Result, PersistenceError> { + self.list_evolution_records(tenant, record_type, status) + .await + .map(|rows| rows.into_iter().map(pg_evolution_record_to_turso).collect()) + } + + async fn list_ranked_insights( + &self, + tenant: &str, + ) -> Result, PersistenceError> { + self.list_ranked_insights(tenant) + .await + .map(|rows| rows.into_iter().map(pg_evolution_record_to_turso).collect()) + } +} + +#[async_trait::async_trait] +impl EvolutionStore for TursoEventStore { + async fn upsert_feature_request( + &self, + tenant: &str, + id: &str, + category: &str, + description: &str, + frequency: i64, + trajectory_refs_json: &str, + disposition: &str, + developer_notes: Option<&str>, + ) -> Result<(), PersistenceError> { + self.upsert_feature_request( + tenant, + id, + category, + description, + frequency, + trajectory_refs_json, + disposition, + developer_notes, + ) + .await + } + + async fn list_feature_requests( + &self, + tenant: &str, + disposition: Option<&str>, + ) -> Result, PersistenceError> { + self.list_feature_requests(tenant, disposition).await + } + + async fn update_feature_request( + &self, + tenant: &str, + id: &str, + disposition: &str, + developer_notes: Option<&str>, + ) -> Result { + self.update_feature_request(tenant, id, disposition, developer_notes) + .await + } + + async fn insert_evolution_record( + &self, + record: EvolutionRecordWrite<'_>, + ) -> Result<(), PersistenceError> { + self.insert_evolution_record(TursoEvolutionRecordInsert { + tenant: record.tenant, + id: record.id, + record_type: record.record_type, + status: record.status, + created_by: record.created_by, + derived_from: record.derived_from, + data_json: record.data_json, + }) + .await + } + + async fn get_evolution_record( + &self, + tenant: &str, + id: &str, + ) -> Result, PersistenceError> { + self.get_evolution_record(tenant, id).await + } + + async fn list_evolution_records( + &self, + tenant: &str, + record_type: Option<&str>, + status: Option<&str>, + ) -> Result, PersistenceError> { + self.list_evolution_records(tenant, record_type, status) + .await + } + + async fn list_ranked_insights( + &self, + tenant: &str, + ) -> Result, PersistenceError> { + self.list_ranked_insights(tenant).await + } +} diff --git a/crates/temper-server/src/storage/mod.rs b/crates/temper-server/src/storage/mod.rs index f689b7db0..2d461d998 100644 --- a/crates/temper-server/src/storage/mod.rs +++ b/crates/temper-server/src/storage/mod.rs @@ -20,23 +20,31 @@ use sqlx::PgPool; use temper_runtime::persistence::{ EventStore, PersistenceAppend, PersistenceAppendResult, PersistenceEnvelope, PersistenceError, }; -use temper_store_postgres::{PostgresEventStore, PostgresPolicyRow, PostgresTrajectoryInsert}; +use temper_store_postgres::{ + PostgresEventStore, PostgresEvolutionRecordInsert, PostgresPolicyApprovalCommit, + PostgresPolicyRow, PostgresTrajectoryInsert, +}; use temper_store_turso::{ ActionStats, AgentSummary, DesignTimeEventRow, EvolutionRecordRow, FeatureRequestRow, - OtsQueuedTrajectoryRow, OtsTrajectoryParams, OtsTrajectoryRow, PolicyDenialPatternRow, - PolicyRow as TursoPolicyRow, TenantStoreRouter, TenantUserRow, TursoEventStore, - TursoTrajectoryInsert, TursoTrajectoryRow, TursoWasmInvocationInsert, TursoWasmInvocationRow, + OtsQueuedTrajectoryRow, OtsTrajectoryDocument, OtsTrajectoryParams, OtsTrajectoryRow, + PolicyDenialPatternRow, PolicyRow as TursoPolicyRow, TenantStoreRouter, TenantUserRow, + TursoEventStore, TursoEvolutionRecordInsert, TursoPolicyApprovalCommit, TursoTrajectoryInsert, + TursoTrajectoryRow, TursoWasmInvocationInsert, TursoWasmInvocationRow, TursoWasmModuleMetadataRow, UnmetIntentAggRow, store::TrajectoryStats, }; use crate::platform_store::PlatformStore; #[cfg(feature = "sim")] use crate::platform_store::SimPlatformStore; -use crate::state::trajectory::{TrajectoryEntry, TrajectorySource}; +use crate::state::trajectory::TrajectoryEntry; +mod metadata_impls; +mod observe_read; mod published_artifacts; mod query_plane_impls; mod query_plane_read; +mod redaction; +mod trajectory_row; pub use published_artifacts::{ PublishedArtifactStore, PublishedArtifactStoreRow, PublishedArtifactStoreUpsert, }; @@ -827,13 +835,18 @@ pub trait PolicyStore: Send + Sync { pub trait ObserveReadStore: Send + Sync { async fn load_recent_trajectories( &self, + tenant: &str, limit: i64, ) -> Result, PersistenceError>; - async fn load_unmet_intent_rows(&self) -> Result, PersistenceError>; + async fn load_unmet_intent_rows( + &self, + tenant: &str, + ) -> Result, PersistenceError>; async fn load_submit_spec_timestamps( &self, + tenant: &str, ) -> Result, PersistenceError>; async fn count_trajectories_by_tenant(&self) @@ -841,6 +854,7 @@ pub trait ObserveReadStore: Send + Sync { async fn query_trajectory_stats( &self, + tenant: &str, entity_type: Option<&str>, action: Option<&str>, success_filter: Option, @@ -855,6 +869,18 @@ pub trait ObserveReadStore: Send + Sync { limit: i64, ) -> Result, PersistenceError>; + /// One session's rows, oldest first, in the order the kernel wrote them. + /// + /// Conformance replays a session as a state-machine run, so the ordering + /// is part of the contract, not an implementation detail. + async fn query_trajectories_by_session( + &self, + session_id: &str, + tenant: Option<&str>, + entity_type: Option<&str>, + limit: i64, + ) -> Result, PersistenceError>; + async fn query_agent_summaries( &self, tenant: Option<&str>, @@ -862,11 +888,30 @@ pub trait ObserveReadStore: Send + Sync { } /// Evolution engine durable metadata capability. +#[derive(Clone, Copy, Debug)] +pub struct EvolutionRecordWrite<'a> { + /// Tenant that owns the record. + pub tenant: &'a str, + /// Stable evolution record identifier. + pub id: &'a str, + /// Evolution record kind. + pub record_type: &'a str, + /// Current record status. + pub status: &'a str, + /// Principal that created the record. + pub created_by: &'a str, + /// Optional predecessor record identifier. + pub derived_from: Option<&'a str>, + /// Serialized record payload. + pub data_json: &'a str, +} + #[async_trait::async_trait] pub trait EvolutionStore: Send + Sync { #[allow(clippy::too_many_arguments)] async fn upsert_feature_request( &self, + tenant: &str, id: &str, category: &str, description: &str, @@ -878,11 +923,13 @@ pub trait EvolutionStore: Send + Sync { async fn list_feature_requests( &self, + tenant: &str, disposition: Option<&str>, ) -> Result, PersistenceError>; async fn update_feature_request( &self, + tenant: &str, id: &str, disposition: &str, developer_notes: Option<&str>, @@ -890,26 +937,26 @@ pub trait EvolutionStore: Send + Sync { async fn insert_evolution_record( &self, - id: &str, - record_type: &str, - status: &str, - created_by: &str, - derived_from: Option<&str>, - data_json: &str, + record: EvolutionRecordWrite<'_>, ) -> Result<(), PersistenceError>; async fn get_evolution_record( &self, + tenant: &str, id: &str, ) -> Result, PersistenceError>; async fn list_evolution_records( &self, + tenant: &str, record_type: Option<&str>, status: Option<&str>, ) -> Result, PersistenceError>; - async fn list_ranked_insights(&self) -> Result, PersistenceError>; + async fn list_ranked_insights( + &self, + tenant: &str, + ) -> Result, PersistenceError>; } /// Design-time verification event capability. @@ -948,13 +995,21 @@ pub trait OtsStore: Send + Sync { params: &OtsTrajectoryParams<'_>, ) -> Result<(), PersistenceError>; + /// Mark a queued trajectory as persisted. + /// + /// Addressed by `(tenant, trajectory_id)`, the identity the row is keyed + /// by: the id comes from the uploading harness, so two tenants can hold + /// the same one. async fn mark_ots_trajectory_persisted( &self, + tenant: &str, trajectory_id: &str, ) -> Result<(), PersistenceError>; + /// Mark a queued trajectory as failed, addressed the same way. async fn mark_ots_trajectory_failed( &self, + tenant: &str, trajectory_id: &str, error: &str, ) -> Result<(), PersistenceError>; @@ -972,10 +1027,15 @@ pub trait OtsStore: Send + Sync { limit: i64, ) -> Result, PersistenceError>; + /// Load a full OTS trajectory by tenant and ID. + /// + /// Tenant is part of the lookup so a trajectory id taken from a request + /// path cannot read another tenant's trace out of a shared store. async fn get_ots_trajectory( &self, + tenant: &str, trajectory_id: &str, - ) -> Result, PersistenceError>; + ) -> Result, PersistenceError>; } /// Legacy database-backed blob capability. @@ -993,6 +1053,14 @@ pub trait BlobStore: Send + Sync { async fn sweep_expired_blobs(&self, max_rows: u64) -> Result; async fn get_blob(&self, key: &str) -> Result>, String>; + + /// Read a legacy database blob only when its stored size is within the + /// caller's allocation budget. `None` means missing or over budget. + async fn get_blob_if_size_at_most( + &self, + key: &str, + max_bytes: usize, + ) -> Result>, String>; } /// Authorization analytics capability. @@ -1015,6 +1083,22 @@ pub trait AuthzAnalyticsStore: Send + Sync { } /// Pending decision query capability. +#[derive(Clone, Copy, Debug)] +pub struct PolicyApprovalCommit<'a> { + /// Tenant that owns both rows. + pub tenant: &'a str, + /// Pending decision to transition. + pub decision_id: &'a str, + /// Serialized approved decision. + pub approved_decision_json: &'a str, + /// Policy row created by the decision. + pub policy_id: &'a str, + /// Approved Cedar source. + pub cedar_text: &'a str, + /// Principal that approved the decision. + pub created_by: &'a str, +} + #[async_trait::async_trait] pub trait DecisionStore: Send + Sync { async fn query_decisions( @@ -1028,7 +1112,24 @@ pub trait DecisionStore: Send + Sync { status: Option<&str>, ) -> Result, PersistenceError>; - async fn get_pending_decision(&self, id: &str) -> Result, PersistenceError>; + async fn get_pending_decision( + &self, + tenant: &str, + id: &str, + ) -> Result, PersistenceError>; + + async fn commit_policy_approval( + &self, + commit: PolicyApprovalCommit<'_>, + ) -> Result<(), PersistenceError>; + + async fn rollback_policy_approval( + &self, + tenant: &str, + decision_id: &str, + pending_decision_json: &str, + policy_id: &str, + ) -> Result<(), PersistenceError>; } /// WASM module metadata capability. @@ -1726,292 +1827,6 @@ impl PolicyStore for TenantStoreRouter { } } -impl BackendNamedStore for PostgresEventStore { - fn backend_name(&self) -> &'static str { - "postgres" - } -} - -impl BackendNamedStore for TursoEventStore { - fn backend_name(&self) -> &'static str { - "turso" - } -} - -#[async_trait::async_trait] -impl ObserveReadStore for PostgresEventStore { - async fn load_recent_trajectories( - &self, - limit: i64, - ) -> Result, PersistenceError> { - self.load_recent_trajectories(limit) - .await - .map(|rows| rows.into_iter().map(pg_trajectory_to_turso).collect()) - } - - async fn load_unmet_intent_rows(&self) -> Result, PersistenceError> { - self.load_unmet_intent_rows() - .await - .map(|rows| rows.into_iter().map(pg_unmet_to_turso).collect()) - } - - async fn load_submit_spec_timestamps( - &self, - ) -> Result, PersistenceError> { - self.load_submit_spec_timestamps().await - } - - async fn count_trajectories_by_tenant( - &self, - ) -> Result, PersistenceError> { - self.count_trajectories_by_tenant().await - } - - async fn query_trajectory_stats( - &self, - entity_type: Option<&str>, - action: Option<&str>, - success_filter: Option, - failed_limit: i64, - ) -> Result { - self.query_trajectory_stats(entity_type, action, success_filter, failed_limit) - .await - .map(pg_stats_to_turso) - } - - async fn query_trajectories_by_agent( - &self, - agent_id: &str, - tenant: Option<&str>, - entity_type: Option<&str>, - limit: i64, - ) -> Result, PersistenceError> { - self.query_trajectories_by_agent(agent_id, tenant, entity_type, limit) - .await - .map(|rows| rows.into_iter().map(pg_trajectory_to_turso).collect()) - } - - async fn query_agent_summaries( - &self, - tenant: Option<&str>, - ) -> Result, PersistenceError> { - self.query_agent_summaries(tenant) - .await - .map(|rows| rows.into_iter().map(pg_agent_summary_to_turso).collect()) - } -} - -#[async_trait::async_trait] -impl ObserveReadStore for TursoEventStore { - async fn load_recent_trajectories( - &self, - limit: i64, - ) -> Result, PersistenceError> { - self.load_recent_trajectories(limit).await - } - - async fn load_unmet_intent_rows(&self) -> Result, PersistenceError> { - self.load_unmet_intent_rows().await - } - - async fn load_submit_spec_timestamps( - &self, - ) -> Result, PersistenceError> { - self.load_submit_spec_timestamps().await - } - - async fn count_trajectories_by_tenant( - &self, - ) -> Result, PersistenceError> { - self.count_trajectories_by_tenant().await - } - - async fn query_trajectory_stats( - &self, - entity_type: Option<&str>, - action: Option<&str>, - success_filter: Option, - failed_limit: i64, - ) -> Result { - self.query_trajectory_stats(entity_type, action, success_filter, failed_limit) - .await - } - - async fn query_trajectories_by_agent( - &self, - agent_id: &str, - tenant: Option<&str>, - entity_type: Option<&str>, - limit: i64, - ) -> Result, PersistenceError> { - self.query_trajectories_by_agent(agent_id, tenant, entity_type, limit) - .await - } - - async fn query_agent_summaries( - &self, - tenant: Option<&str>, - ) -> Result, PersistenceError> { - self.query_agent_summaries(tenant).await - } -} - -#[async_trait::async_trait] -impl EvolutionStore for PostgresEventStore { - async fn upsert_feature_request( - &self, - id: &str, - category: &str, - description: &str, - frequency: i64, - trajectory_refs_json: &str, - disposition: &str, - developer_notes: Option<&str>, - ) -> Result<(), PersistenceError> { - self.upsert_feature_request( - id, - category, - description, - frequency, - trajectory_refs_json, - disposition, - developer_notes, - ) - .await - } - - async fn list_feature_requests( - &self, - disposition: Option<&str>, - ) -> Result, PersistenceError> { - self.list_feature_requests(disposition) - .await - .map(|rows| rows.into_iter().map(pg_feature_request_to_turso).collect()) - } - - async fn update_feature_request( - &self, - id: &str, - disposition: &str, - developer_notes: Option<&str>, - ) -> Result { - self.update_feature_request(id, disposition, developer_notes) - .await - } - - async fn insert_evolution_record( - &self, - id: &str, - record_type: &str, - status: &str, - created_by: &str, - derived_from: Option<&str>, - data_json: &str, - ) -> Result<(), PersistenceError> { - self.insert_evolution_record(id, record_type, status, created_by, derived_from, data_json) - .await - } - - async fn get_evolution_record( - &self, - id: &str, - ) -> Result, PersistenceError> { - self.get_evolution_record(id) - .await - .map(|row| row.map(pg_evolution_record_to_turso)) - } - - async fn list_evolution_records( - &self, - record_type: Option<&str>, - status: Option<&str>, - ) -> Result, PersistenceError> { - self.list_evolution_records(record_type, status) - .await - .map(|rows| rows.into_iter().map(pg_evolution_record_to_turso).collect()) - } - - async fn list_ranked_insights(&self) -> Result, PersistenceError> { - self.list_ranked_insights() - .await - .map(|rows| rows.into_iter().map(pg_evolution_record_to_turso).collect()) - } -} - -#[async_trait::async_trait] -impl EvolutionStore for TursoEventStore { - async fn upsert_feature_request( - &self, - id: &str, - category: &str, - description: &str, - frequency: i64, - trajectory_refs_json: &str, - disposition: &str, - developer_notes: Option<&str>, - ) -> Result<(), PersistenceError> { - self.upsert_feature_request( - id, - category, - description, - frequency, - trajectory_refs_json, - disposition, - developer_notes, - ) - .await - } - - async fn list_feature_requests( - &self, - disposition: Option<&str>, - ) -> Result, PersistenceError> { - self.list_feature_requests(disposition).await - } - - async fn update_feature_request( - &self, - id: &str, - disposition: &str, - developer_notes: Option<&str>, - ) -> Result { - self.update_feature_request(id, disposition, developer_notes) - .await - } - - async fn insert_evolution_record( - &self, - id: &str, - record_type: &str, - status: &str, - created_by: &str, - derived_from: Option<&str>, - data_json: &str, - ) -> Result<(), PersistenceError> { - self.insert_evolution_record(id, record_type, status, created_by, derived_from, data_json) - .await - } - - async fn get_evolution_record( - &self, - id: &str, - ) -> Result, PersistenceError> { - self.get_evolution_record(id).await - } - - async fn list_evolution_records( - &self, - record_type: Option<&str>, - status: Option<&str>, - ) -> Result, PersistenceError> { - self.list_evolution_records(record_type, status).await - } - - async fn list_ranked_insights(&self) -> Result, PersistenceError> { - self.list_ranked_insights().await - } -} - #[async_trait::async_trait] impl DesignTimeEventStore for PostgresEventStore { async fn insert_design_time_event( @@ -2124,17 +1939,21 @@ impl OtsStore for PostgresEventStore { async fn mark_ots_trajectory_persisted( &self, + tenant: &str, trajectory_id: &str, ) -> Result<(), PersistenceError> { - self.mark_ots_trajectory_persisted(trajectory_id).await + self.mark_ots_trajectory_persisted(tenant, trajectory_id) + .await } async fn mark_ots_trajectory_failed( &self, + tenant: &str, trajectory_id: &str, error: &str, ) -> Result<(), PersistenceError> { - self.mark_ots_trajectory_failed(trajectory_id, error).await + self.mark_ots_trajectory_failed(tenant, trajectory_id, error) + .await } async fn list_queued_ots_trajectories( @@ -2160,9 +1979,12 @@ impl OtsStore for PostgresEventStore { async fn get_ots_trajectory( &self, + tenant: &str, trajectory_id: &str, - ) -> Result, PersistenceError> { - self.get_ots_trajectory(trajectory_id).await + ) -> Result, PersistenceError> { + self.get_ots_trajectory(tenant, trajectory_id) + .await + .map(|document| document.map(pg_ots_document_to_turso)) } } @@ -2184,17 +2006,21 @@ impl OtsStore for TursoEventStore { async fn mark_ots_trajectory_persisted( &self, + tenant: &str, trajectory_id: &str, ) -> Result<(), PersistenceError> { - self.mark_ots_trajectory_persisted(trajectory_id).await + self.mark_ots_trajectory_persisted(tenant, trajectory_id) + .await } async fn mark_ots_trajectory_failed( &self, + tenant: &str, trajectory_id: &str, error: &str, ) -> Result<(), PersistenceError> { - self.mark_ots_trajectory_failed(trajectory_id, error).await + self.mark_ots_trajectory_failed(tenant, trajectory_id, error) + .await } async fn list_queued_ots_trajectories( @@ -2217,9 +2043,10 @@ impl OtsStore for TursoEventStore { async fn get_ots_trajectory( &self, + tenant: &str, trajectory_id: &str, - ) -> Result, PersistenceError> { - self.get_ots_trajectory(trajectory_id).await + ) -> Result, PersistenceError> { + self.get_ots_trajectory(tenant, trajectory_id).await } } @@ -2245,6 +2072,14 @@ impl BlobStore for PostgresEventStore { async fn get_blob(&self, key: &str) -> Result>, String> { self.get_blob(key).await } + + async fn get_blob_if_size_at_most( + &self, + key: &str, + max_bytes: usize, + ) -> Result>, String> { + self.get_blob_if_size_at_most(key, max_bytes).await + } } #[async_trait::async_trait] @@ -2269,6 +2104,14 @@ impl BlobStore for TursoEventStore { async fn get_blob(&self, key: &str) -> Result>, String> { self.get_blob(key).await } + + async fn get_blob_if_size_at_most( + &self, + key: &str, + max_bytes: usize, + ) -> Result>, String> { + self.get_blob_if_size_at_most(key, max_bytes).await + } } #[async_trait::async_trait] @@ -2350,8 +2193,38 @@ impl DecisionStore for PostgresEventStore { self.query_all_decisions(status).await } - async fn get_pending_decision(&self, id: &str) -> Result, PersistenceError> { - self.get_pending_decision(id).await + async fn get_pending_decision( + &self, + tenant: &str, + id: &str, + ) -> Result, PersistenceError> { + self.get_pending_decision(tenant, id).await + } + + async fn commit_policy_approval( + &self, + commit: PolicyApprovalCommit<'_>, + ) -> Result<(), PersistenceError> { + self.commit_policy_approval(PostgresPolicyApprovalCommit { + tenant: commit.tenant, + decision_id: commit.decision_id, + approved_decision_json: commit.approved_decision_json, + policy_id: commit.policy_id, + cedar_text: commit.cedar_text, + created_by: commit.created_by, + }) + .await + } + + async fn rollback_policy_approval( + &self, + tenant: &str, + decision_id: &str, + pending_decision_json: &str, + policy_id: &str, + ) -> Result<(), PersistenceError> { + self.rollback_policy_approval(tenant, decision_id, pending_decision_json, policy_id) + .await } } @@ -2372,8 +2245,38 @@ impl DecisionStore for TursoEventStore { self.query_all_decisions(status).await } - async fn get_pending_decision(&self, id: &str) -> Result, PersistenceError> { - self.get_pending_decision(id).await + async fn get_pending_decision( + &self, + tenant: &str, + id: &str, + ) -> Result, PersistenceError> { + self.get_pending_decision(tenant, id).await + } + + async fn commit_policy_approval( + &self, + commit: PolicyApprovalCommit<'_>, + ) -> Result<(), PersistenceError> { + self.commit_policy_approval(TursoPolicyApprovalCommit { + tenant: commit.tenant, + decision_id: commit.decision_id, + approved_decision_json: commit.approved_decision_json, + policy_id: commit.policy_id, + cedar_text: commit.cedar_text, + created_by: commit.created_by, + }) + .await + } + + async fn rollback_policy_approval( + &self, + tenant: &str, + decision_id: &str, + pending_decision_json: &str, + policy_id: &str, + ) -> Result<(), PersistenceError> { + self.rollback_policy_approval(tenant, decision_id, pending_decision_json, policy_id) + .await } } @@ -2482,6 +2385,7 @@ fn pg_trajectory_to_turso(row: temper_store_postgres::PostgresTrajectoryRow) -> request_body: row.request_body, intent: row.intent, matched_policy_ids: row.matched_policy_ids, + capture_seq: row.capture_seq, } } @@ -2541,6 +2445,7 @@ fn pg_feature_request_to_turso( ) -> FeatureRequestRow { FeatureRequestRow { id: row.id, + tenant: row.tenant, category: row.category, description: row.description, frequency: row.frequency, @@ -2557,6 +2462,7 @@ fn pg_evolution_record_to_turso( ) -> EvolutionRecordRow { EvolutionRecordRow { id: row.id, + tenant: row.tenant, record_type: row.record_type, status: row.status, created_by: row.created_by, @@ -2614,6 +2520,19 @@ fn pg_queued_ots_to_turso( } } +fn pg_ots_document_to_turso( + document: temper_store_postgres::PostgresOtsTrajectoryDocument, +) -> OtsTrajectoryDocument { + OtsTrajectoryDocument { + trajectory_id: document.trajectory_id, + tenant: document.tenant, + agent_id: document.agent_id, + session_id: document.session_id, + outcome: document.outcome, + data: document.data, + } +} + fn pg_denial_pattern_to_turso( row: temper_store_postgres::PostgresPolicyDenialPatternRow, ) -> PolicyDenialPatternRow { @@ -2677,35 +2596,11 @@ impl DataOnlyCreateStore for PostgresEventStore { } } -fn trajectory_source_label(source: &TrajectorySource) -> &'static str { - match source { - TrajectorySource::Entity => "Entity", - TrajectorySource::Platform => "Platform", - TrajectorySource::Authz => "Authz", - } -} - -fn trajectory_request_body_json(entry: &TrajectoryEntry) -> Option { - entry.request_body.as_ref().and_then(|value| { - let serialized = serde_json::to_string(value).ok()?; - Some(if serialized.len() > 4096 { - let mut end = 4096; - while !serialized.is_char_boundary(end) { - end -= 1; - } - serialized[..end].to_string() - } else { - serialized - }) - }) -} - -fn trajectory_matched_policy_ids_json(entry: &TrajectoryEntry) -> Option { - entry - .matched_policy_ids - .as_ref() - .and_then(|ids| serde_json::to_string(ids).ok()) -} +pub(crate) use redaction::redact_secrets; +pub(crate) use trajectory_row::bounded_request_body; +use trajectory_row::{ + trajectory_matched_policy_ids_json, trajectory_request_body_json, trajectory_source_label, +}; #[async_trait::async_trait] impl TrajectorySink for PostgresEventStore { @@ -2734,6 +2629,7 @@ impl TrajectorySink for PostgresEventStore { request_body: request_body_json.as_deref(), intent: entry.intent.as_deref(), matched_policy_ids: matched_policy_ids_json.as_deref(), + capture_seq: entry.capture_seq, }) .await .map_err(|e| { @@ -2772,6 +2668,7 @@ impl TrajectorySink for TursoEventStore { request_body: request_body_json.as_deref(), intent: entry.intent.as_deref(), matched_policy_ids: matched_policy_ids_json.as_deref(), + capture_seq: entry.capture_seq, }) .await .map_err(|e| { @@ -2817,6 +2714,7 @@ impl TrajectorySink for TenantStoreRouter { request_body: request_body_json.as_deref(), intent: entry.intent.as_deref(), matched_policy_ids: matched_policy_ids_json.as_deref(), + capture_seq: entry.capture_seq, }) .await .map_err(|e| { diff --git a/crates/temper-server/src/storage/observe_read.rs b/crates/temper-server/src/storage/observe_read.rs new file mode 100644 index 000000000..872f6d3e6 --- /dev/null +++ b/crates/temper-server/src/storage/observe_read.rs @@ -0,0 +1,172 @@ +//! `ObserveReadStore` implementations for the durable backends. +//! +//! The observe read path — trajectory listings, per-session and per-agent +//! replay, aggregate stats — lives here rather than in the storage module so +//! the composite-trait module stays a declaration of the storage surface +//! instead of also carrying every backend's query bodies. + +use std::collections::BTreeMap; + +use temper_runtime::persistence::PersistenceError; +use temper_store_postgres::PostgresEventStore; +use temper_store_turso::{ + AgentSummary, TursoEventStore, TursoTrajectoryRow, UnmetIntentAggRow, store::TrajectoryStats, +}; + +use super::{ + ObserveReadStore, pg_agent_summary_to_turso, pg_stats_to_turso, pg_trajectory_to_turso, + pg_unmet_to_turso, +}; + +#[async_trait::async_trait] +impl ObserveReadStore for PostgresEventStore { + async fn load_recent_trajectories( + &self, + tenant: &str, + limit: i64, + ) -> Result, PersistenceError> { + self.load_recent_trajectories(tenant, limit) + .await + .map(|rows| rows.into_iter().map(pg_trajectory_to_turso).collect()) + } + + async fn load_unmet_intent_rows( + &self, + tenant: &str, + ) -> Result, PersistenceError> { + self.load_unmet_intent_rows(tenant) + .await + .map(|rows| rows.into_iter().map(pg_unmet_to_turso).collect()) + } + + async fn load_submit_spec_timestamps( + &self, + tenant: &str, + ) -> Result, PersistenceError> { + self.load_submit_spec_timestamps(tenant).await + } + + async fn count_trajectories_by_tenant( + &self, + ) -> Result, PersistenceError> { + self.count_trajectories_by_tenant().await + } + + async fn query_trajectory_stats( + &self, + tenant: &str, + entity_type: Option<&str>, + action: Option<&str>, + success_filter: Option, + failed_limit: i64, + ) -> Result { + self.query_trajectory_stats(tenant, entity_type, action, success_filter, failed_limit) + .await + .map(pg_stats_to_turso) + } + + async fn query_trajectories_by_agent( + &self, + agent_id: &str, + tenant: Option<&str>, + entity_type: Option<&str>, + limit: i64, + ) -> Result, PersistenceError> { + self.query_trajectories_by_agent(agent_id, tenant, entity_type, limit) + .await + .map(|rows| rows.into_iter().map(pg_trajectory_to_turso).collect()) + } + + async fn query_trajectories_by_session( + &self, + session_id: &str, + tenant: Option<&str>, + entity_type: Option<&str>, + limit: i64, + ) -> Result, PersistenceError> { + self.query_trajectories_by_session(session_id, tenant, entity_type, limit) + .await + .map(|rows| rows.into_iter().map(pg_trajectory_to_turso).collect()) + } + + async fn query_agent_summaries( + &self, + tenant: Option<&str>, + ) -> Result, PersistenceError> { + self.query_agent_summaries(tenant) + .await + .map(|rows| rows.into_iter().map(pg_agent_summary_to_turso).collect()) + } +} + +#[async_trait::async_trait] +impl ObserveReadStore for TursoEventStore { + async fn load_recent_trajectories( + &self, + tenant: &str, + limit: i64, + ) -> Result, PersistenceError> { + self.load_recent_trajectories(tenant, limit).await + } + + async fn load_unmet_intent_rows( + &self, + tenant: &str, + ) -> Result, PersistenceError> { + self.load_unmet_intent_rows(tenant).await + } + + async fn load_submit_spec_timestamps( + &self, + tenant: &str, + ) -> Result, PersistenceError> { + self.load_submit_spec_timestamps(tenant).await + } + + async fn count_trajectories_by_tenant( + &self, + ) -> Result, PersistenceError> { + self.count_trajectories_by_tenant().await + } + + async fn query_trajectory_stats( + &self, + tenant: &str, + entity_type: Option<&str>, + action: Option<&str>, + success_filter: Option, + failed_limit: i64, + ) -> Result { + self.query_trajectory_stats(tenant, entity_type, action, success_filter, failed_limit) + .await + } + + async fn query_trajectories_by_agent( + &self, + agent_id: &str, + tenant: Option<&str>, + entity_type: Option<&str>, + limit: i64, + ) -> Result, PersistenceError> { + self.query_trajectories_by_agent(agent_id, tenant, entity_type, limit) + .await + } + + async fn query_trajectories_by_session( + &self, + session_id: &str, + tenant: Option<&str>, + entity_type: Option<&str>, + limit: i64, + ) -> Result, PersistenceError> { + self.query_trajectories_by_session(session_id, tenant, entity_type, limit) + .await + } + + async fn query_agent_summaries( + &self, + tenant: Option<&str>, + ) -> Result, PersistenceError> { + self.query_agent_summaries(tenant).await + } +} diff --git a/crates/temper-server/src/storage/redaction.rs b/crates/temper-server/src/storage/redaction.rs new file mode 100644 index 000000000..1cafdbdce --- /dev/null +++ b/crates/temper-server/src/storage/redaction.rs @@ -0,0 +1,224 @@ +//! Secret redaction for captured request bodies. +//! +//! Every dispatch records the parameters it was called with, successful ones +//! included, and those parameters are read back through trajectory observation +//! surfaces and exported into training data. An action that takes an +//! `api_token`, a `password`, or a card number would put that value in a +//! durable, widely-readable row. +//! +//! Redaction runs at the single enqueue choke point +//! (`ServerState::enqueue_trajectory_entry`), before the size cap, so: +//! +//! - every capture site is covered — dispatch, authorization denials, the +//! audit endpoint, unmet intents — and a new one cannot forget it; +//! - a truncated preview can never contain a value the full body would have +//! had redacted. +//! +//! # What is redacted +//! +//! The name of the field, not its contents: value-shape guessing produces both +//! misses and false positives, while a field called `password` is a password +//! whatever it holds. A key is redacted when its normalized form — lowercased, +//! with `_`, `-`, `.`, and spaces removed — is one of [`SECRET_KEY_NAMES`] or +//! ends with one of [`SECRET_KEY_SUFFIXES`]. The key stays, so a reader still +//! sees which arguments were passed; only the value becomes +//! [`REDACTED_PLACEHOLDER`]. +//! +//! The suffix list is deliberately singular (`token`, not `tokens`) and does +//! not include bare `key`: `prompt_tokens`, `completion_token_ids`, and +//! `idempotency_key` are operational data, and redacting them would corrupt +//! the trajectory to no benefit. +//! +//! This is a name-based net, not a proof. An action that puts a credential in +//! a field named `value` is not caught; catching that needs the spec to +//! declare which parameters are sensitive, which the IOA action grammar does +//! not yet express. + +use serde_json::{Map, Value}; + +/// Value written in place of a redacted field. +pub(crate) const REDACTED_PLACEHOLDER: &str = "[redacted]"; + +/// Normalized key names whose value is a secret only as the whole name. +/// +/// These are short or common enough that matching them as an ending would +/// catch ordinary words: `auth` would take `oauth_provider_name`, `pin` would +/// take `spin`. Names that are safe to match as endings live in +/// [`SECRET_KEY_SUFFIXES`] instead and are not repeated here. +pub(crate) const SECRET_KEY_NAMES: &[&str] = &[ + "accountnumber", + "auth", + "authorization", + "cvc", + "cvv", + "iban", + "pin", + "pwd", + "routingnumber", + "securitycode", + "ssn", +]; + +/// Normalized key endings whose value is always a secret. +/// +/// Matching the ending is what catches the qualified forms every codebase +/// grows — `client_secret`, `stripe_api_key`, `user_password`. +pub(crate) const SECRET_KEY_SUFFIXES: &[&str] = &[ + "accesskey", + "apikey", + "cardnumber", + "credential", + "credentials", + "passphrase", + "passwd", + "password", + "privatekey", + "secret", + "secretkey", + "sessionkey", + "token", +]; + +/// Replace the value of every secret-named field, at any depth. +pub(crate) fn redact_secrets(value: Value) -> Value { + match value { + Value::Object(map) => Value::Object(redact_object(map)), + Value::Array(items) => Value::Array(items.into_iter().map(redact_secrets).collect()), + other => other, + } +} + +fn redact_object(map: Map) -> Map { + map.into_iter() + .map(|(key, value)| { + if is_secret_key(&key) { + (key, Value::String(REDACTED_PLACEHOLDER.to_string())) + } else { + (key, redact_secrets(value)) + } + }) + .collect() +} + +/// Whether a field name names a secret. +fn is_secret_key(key: &str) -> bool { + let normalized = normalize_key(key); + SECRET_KEY_NAMES.contains(&normalized.as_str()) + || SECRET_KEY_SUFFIXES + .iter() + .any(|suffix| normalized.ends_with(suffix)) +} + +/// Lowercase and drop the separators that only vary by naming convention, so +/// `apiToken`, `api_token`, `API-TOKEN`, and `api.token` are one name. +fn normalize_key(key: &str) -> String { + key.chars() + .filter(|c| !matches!(c, '_' | '-' | '.' | ' ')) + .flat_map(char::to_lowercase) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn secret_named_fields_lose_their_values() { + let body = serde_json::json!({ + "password": "hunter2", + "api_token": "sk-live-abc", + "apiKey": "k-1", + "Authorization": "Bearer abc", + "card_number": "4111111111111111", + "cvv": 123, + "client_secret": "cs-1", + }); + let redacted = redact_secrets(body); + for key in [ + "password", + "api_token", + "apiKey", + "Authorization", + "card_number", + "cvv", + "client_secret", + ] { + assert_eq!( + redacted[key], REDACTED_PLACEHOLDER, + "`{key}` must not be stored verbatim" + ); + } + } + + #[test] + fn nested_and_arrayed_secrets_are_reached() { + let body = serde_json::json!({ + "connection": {"host": "db", "password": "hunter2"}, + "accounts": [{"id": "a", "access_token": "t-1"}, {"id": "b"}], + }); + let redacted = redact_secrets(body); + assert_eq!(redacted["connection"]["host"], "db"); + assert_eq!(redacted["connection"]["password"], REDACTED_PLACEHOLDER); + assert_eq!(redacted["accounts"][0]["id"], "a"); + assert_eq!( + redacted["accounts"][0]["access_token"], + REDACTED_PLACEHOLDER + ); + } + + #[test] + fn a_secret_object_is_replaced_whole_rather_than_walked_into() { + let body = serde_json::json!({"credentials": {"user": "rita", "password": "hunter2"}}); + let redacted = redact_secrets(body); + assert_eq!( + redacted["credentials"], REDACTED_PLACEHOLDER, + "a secret-named object leaks through its own fields if only its children are redacted" + ); + } + + #[test] + fn whole_name_only_entries_do_not_match_as_endings() { + // `auth` and `pin` are whole names; matching them as endings would + // take ordinary words with them. + let body = serde_json::json!({ + "auth": "Bearer abc", + "pin": "1234", + "oauth": "provider", + "spin": 3, + "checkin": "10:00", + }); + let redacted = redact_secrets(body); + assert_eq!(redacted["auth"], REDACTED_PLACEHOLDER); + assert_eq!(redacted["pin"], REDACTED_PLACEHOLDER); + assert_eq!(redacted["oauth"], "provider"); + assert_eq!(redacted["spin"], serde_json::json!(3)); + assert_eq!(redacted["checkin"], "10:00"); + } + + #[test] + fn operational_fields_that_merely_look_secretish_are_kept() { + let body = serde_json::json!({ + "prompt_tokens": 128, + "completion_token_ids": [1, 2, 3], + "idempotency_key": "idem-1", + "author": "rita", + "keyboard": "qwerty", + "monkey": "george", + "public_key_id": "kid-1", + }); + let redacted = redact_secrets(body.clone()); + assert_eq!( + redacted, body, + "redaction must not corrupt ordinary trajectory data" + ); + } + + #[test] + fn a_non_object_body_passes_through() { + assert_eq!( + redact_secrets(serde_json::json!("plain string")), + serde_json::json!("plain string") + ); + assert_eq!(redact_secrets(Value::Null), Value::Null); + } +} diff --git a/crates/temper-server/src/storage/trajectory_row.rs b/crates/temper-server/src/storage/trajectory_row.rs new file mode 100644 index 000000000..b0ece0651 --- /dev/null +++ b/crates/temper-server/src/storage/trajectory_row.rs @@ -0,0 +1,85 @@ +//! Column encoding for a persisted trajectory row. +//! +//! Backend-agnostic: every `TrajectorySink` implementation encodes the same +//! way, so a row means the same thing whichever store wrote it. + +use crate::state::trajectory::{TrajectoryEntry, TrajectorySource}; + +pub(super) fn trajectory_source_label(source: &TrajectorySource) -> &'static str { + match source { + TrajectorySource::Entity => "Entity", + TrajectorySource::Platform => "Platform", + TrajectorySource::Authz => "Authz", + } +} + +/// Maximum stored size, in bytes, of a captured trajectory request body. +pub(crate) const TRAJECTORY_REQUEST_BODY_MAX_BYTES: usize = 4096; + +/// Serialize a captured request body for storage, bounded to +/// [`TRAJECTORY_REQUEST_BODY_MAX_BYTES`]. +/// +/// An oversized body becomes a valid JSON envelope carrying a bounded preview +/// rather than a byte-sliced prefix. The column is read back with +/// `serde_json::from_str`, so a sliced prefix parses as nothing and the row +/// loses its body entirely. Every action records its params, successful ones +/// included, which makes a large body an ordinary case rather than a rare one. +pub(super) fn trajectory_request_body_json(entry: &TrajectoryEntry) -> Option { + let serialized = serde_json::to_string(entry.request_body.as_ref()?).ok()?; + if serialized.len() <= TRAJECTORY_REQUEST_BODY_MAX_BYTES { + return Some(serialized); + } + Some(truncated_request_body_value(&serialized).to_string()) +} + +/// Bound a request body before the entry enters the persistence outbox. +/// +/// Applies the same cap as the sink, one step earlier. Every dispatch now +/// records its params, and entries wait in the bounded outbox before they are +/// written, so admitting whole bodies would let a stalled drain hold megabytes +/// per in-flight entry. Enforced at the single enqueue choke point rather than +/// at each capture site, so a new capture site cannot forget it. +pub(crate) fn bounded_request_body(value: serde_json::Value) -> serde_json::Value { + match serde_json::to_string(&value) { + Ok(serialized) if serialized.len() > TRAJECTORY_REQUEST_BODY_MAX_BYTES => { + truncated_request_body_value(&serialized) + } + _ => value, + } +} + +/// Build the truncation envelope for an oversized request body. +/// +/// JSON string escaping can expand the preview, so the preview shrinks until +/// the rendered envelope fits the cap. An empty preview always fits. +fn truncated_request_body_value(serialized: &str) -> serde_json::Value { + let original_bytes = serialized.len(); + let mut preview_len = TRAJECTORY_REQUEST_BODY_MAX_BYTES; + loop { + while preview_len > 0 && !serialized.is_char_boundary(preview_len) { + preview_len -= 1; + } + let envelope = serde_json::json!({ + "_truncated": true, + "_original_bytes": original_bytes, + "_preview": &serialized[..preview_len], + }); + let rendered_len = envelope.to_string().len(); + if rendered_len <= TRAJECTORY_REQUEST_BODY_MAX_BYTES || preview_len == 0 { + return envelope; + } + let overflow = rendered_len - TRAJECTORY_REQUEST_BODY_MAX_BYTES; + preview_len = preview_len.saturating_sub(overflow.max(1)); + } +} + +pub(super) fn trajectory_matched_policy_ids_json(entry: &TrajectoryEntry) -> Option { + entry + .matched_policy_ids + .as_ref() + .and_then(|ids| serde_json::to_string(ids).ok()) +} + +#[cfg(test)] +#[path = "trajectory_row_test.rs"] +mod tests; diff --git a/crates/temper-server/src/storage/trajectory_row_test.rs b/crates/temper-server/src/storage/trajectory_row_test.rs new file mode 100644 index 000000000..ac8aa9224 --- /dev/null +++ b/crates/temper-server/src/storage/trajectory_row_test.rs @@ -0,0 +1,131 @@ +//! Bounds and JSON validity of the captured trajectory request body. + +use super::*; + +fn entry_with_body(request_body: Option) -> TrajectoryEntry { + TrajectoryEntry { + timestamp: "2026-08-11T00:00:00Z".to_string(), + tenant: "default".to_string(), + entity_type: "Order".to_string(), + entity_id: "ord-1".to_string(), + action: "AddItem".to_string(), + success: true, + from_status: Some("Draft".to_string()), + to_status: Some("Draft".to_string()), + error: None, + agent_id: None, + session_id: None, + authz_denied: None, + denied_resource: None, + denied_module: None, + source: Some(TrajectorySource::Entity), + spec_governed: Some(true), + agent_type: None, + request_body, + intent: None, + matched_policy_ids: None, + capture_seq: None, + } +} + +#[test] +fn small_request_body_is_stored_verbatim() { + let body = serde_json::json!({"ProductId": "p-1", "Quantity": 2}); + let stored = + trajectory_request_body_json(&entry_with_body(Some(body.clone()))).expect("stored"); + assert_eq!( + serde_json::from_str::(&stored).expect("valid json"), + body + ); +} + +#[test] +fn absent_request_body_stores_nothing() { + assert!(trajectory_request_body_json(&entry_with_body(None)).is_none()); +} + +#[test] +fn oversized_request_body_stays_parseable_json() { + let body = serde_json::json!({"Notes": "x".repeat(20_000)}); + let stored = trajectory_request_body_json(&entry_with_body(Some(body))).expect("stored"); + + assert!( + stored.len() <= TRAJECTORY_REQUEST_BODY_MAX_BYTES, + "stored body must respect the cap, got {} bytes", + stored.len() + ); + let parsed: serde_json::Value = + serde_json::from_str(&stored).expect("truncated body must still parse as JSON"); + assert_eq!(parsed["_truncated"], serde_json::json!(true)); + assert!( + parsed["_original_bytes"].as_u64().expect("original bytes") > 20_000, + "envelope records the pre-truncation size" + ); + assert!( + parsed["_preview"] + .as_str() + .expect("preview") + .starts_with("{\"Notes\":\"xxx"), + "envelope keeps a readable prefix of the original body" + ); +} + +#[test] +fn oversized_multibyte_request_body_is_not_split_mid_character() { + // A body made entirely of 4-byte characters: naive byte slicing at the + // cap would land mid-character and produce invalid UTF-8/JSON. + let body = serde_json::json!({"Notes": "\u{1F600}".repeat(5_000)}); + let stored = trajectory_request_body_json(&entry_with_body(Some(body))).expect("stored"); + + assert!(stored.len() <= TRAJECTORY_REQUEST_BODY_MAX_BYTES); + let parsed: serde_json::Value = + serde_json::from_str(&stored).expect("multibyte truncation must still parse"); + assert_eq!(parsed["_truncated"], serde_json::json!(true)); +} + +#[test] +fn escape_heavy_request_body_still_fits_the_cap() { + // Quotes and backslashes double under JSON escaping, so a preview cut + // at the raw cap would render past it. + let body = serde_json::json!({"Notes": "\"\\".repeat(6_000)}); + let stored = trajectory_request_body_json(&entry_with_body(Some(body))).expect("stored"); + + assert!( + stored.len() <= TRAJECTORY_REQUEST_BODY_MAX_BYTES, + "escape expansion must not push the envelope past the cap, got {} bytes", + stored.len() + ); + serde_json::from_str::(&stored).expect("valid json"); +} + +#[test] +fn enqueue_bounding_passes_small_bodies_through_unchanged() { + let body = serde_json::json!({"ProductId": "p-1", "Quantity": 2}); + assert_eq!(bounded_request_body(body.clone()), body); +} + +#[test] +fn enqueue_bounding_caps_oversized_bodies_before_the_outbox() { + let body = serde_json::json!({"Notes": "x".repeat(20_000)}); + let bounded = bounded_request_body(body); + + assert_eq!(bounded["_truncated"], serde_json::json!(true)); + assert!( + bounded.to_string().len() <= TRAJECTORY_REQUEST_BODY_MAX_BYTES, + "an enqueued entry must never carry more than the cap into the outbox" + ); +} + +#[test] +fn enqueue_bounding_and_sink_encoding_agree() { + // Whether a body is bounded on the way into the outbox or at the sink, the + // stored column must come out identical — otherwise the same action yields + // two shapes depending on which path wrote it. + let body = serde_json::json!({"Notes": "y".repeat(20_000)}); + let via_sink = + trajectory_request_body_json(&entry_with_body(Some(body.clone()))).expect("sink"); + let via_enqueue = + trajectory_request_body_json(&entry_with_body(Some(bounded_request_body(body)))) + .expect("enqueue"); + assert_eq!(via_sink, via_enqueue); +} diff --git a/crates/temper-server/src/trajectory_outbox.rs b/crates/temper-server/src/trajectory_outbox.rs index 50a7bd751..78b593bbe 100644 --- a/crates/temper-server/src/trajectory_outbox.rs +++ b/crates/temper-server/src/trajectory_outbox.rs @@ -1,4 +1,10 @@ //! Bounded background persistence for observe trajectory entries. +//! +//! The queue is bounded and its writes can fail, so this path can lose a +//! captured entry. A loss that only shows up as a log line lets a run with +//! holes in it later pass a conformance check, so every loss is counted and — +//! once per session — written to storage as a marker row the checker reads as +//! an evidence gap (`crate::conformance::CAPTURE_LOSS_ENTITY_TYPE`). use std::sync::{ Arc, OnceLock, @@ -15,6 +21,16 @@ use tracing::Instrument; use crate::state::trajectory::TrajectoryEntry; use crate::storage::TrajectorySink; +mod capture_loss; + +pub(crate) use capture_loss::CaptureHealth; +use capture_loss::record_capture_loss; +#[cfg(test)] +use capture_loss::{ + MAX_MARKED_SESSIONS, MarkerClaim, claim_capture_loss_marker, claim_in, + persist_capture_loss_marker, queued_marker_for_test, release_capture_loss_marker, +}; + const DEFAULT_CAPACITY: usize = 8_192; struct TrajectoryOutboxMetrics { @@ -22,6 +38,7 @@ struct TrajectoryOutboxMetrics { outbox_capacity: Gauge, enqueued_total: Counter, dropped_total: Counter, + capture_loss_marker_total: Counter, persist_latency_ms: Histogram, } @@ -44,7 +61,11 @@ fn metrics() -> &'static TrajectoryOutboxMetrics { .build(), dropped_total: meter .u64_counter("temper_trajectory_outbox_dropped_total") - .with_description("Trajectory entries dropped because the persistence outbox was unavailable or full.") + .with_description("Captured trajectory entries that never reached storage, by reason: the outbox was full or unavailable, or the write failed.") + .build(), + capture_loss_marker_total: meter + .u64_counter("temper_trajectory_capture_loss_marker_total") + .with_description("Attempts to write the per-session marker that records a capture loss, by result.") .build(), persist_latency_ms: meter .f64_histogram("temper_trajectory_outbox_persist_latency_ms") @@ -122,6 +143,7 @@ struct QueuedTrajectory { sink: Option>, backend: &'static str, entry: TrajectoryEntry, + health: CaptureHealth, } pub(crate) struct TrajectoryOutbox { @@ -150,8 +172,9 @@ impl TrajectoryOutbox { backend: &'static str, sink: Arc, entry: TrajectoryEntry, + health: CaptureHealth, ) -> bool { - self.try_enqueue(Some(sink), backend, entry) + self.try_enqueue(Some(sink), backend, entry, health) } fn try_enqueue( @@ -159,6 +182,7 @@ impl TrajectoryOutbox { sink: Option>, backend: &'static str, entry: TrajectoryEntry, + health: CaptureHealth, ) -> bool { let metric_entry = entry.clone(); // Backpressure: cap the in-flight depth at `capacity`. Drop-newest on @@ -168,14 +192,7 @@ impl TrajectoryOutbox { self.depth.fetch_sub(1, Ordering::Relaxed); self.dropped_total.fetch_add(1, Ordering::Relaxed); record_depth(self.depth.load(Ordering::Relaxed)); - record_dropped(&metric_entry, backend, "outbox_full"); - tracing::warn!( - tenant = %metric_entry.tenant, - entity_type = %metric_entry.entity_type, - entity_id = %metric_entry.entity_id, - action = %metric_entry.action, - "trajectory outbox full; dropping entry" - ); + record_capture_loss(sink.clone(), backend, &metric_entry, "outbox_full", &health); return false; } record_enqueued(&metric_entry, backend); @@ -190,6 +207,7 @@ impl TrajectoryOutbox { sink, backend, entry, + health, }; // In unit tests built via `for_tests`, skip the spawn so the bounded // depth/drop semantics can be exercised without a tokio runtime. @@ -220,7 +238,7 @@ impl TrajectoryOutbox { #[cfg(test)] fn try_record_for_test(&self, entry: TrajectoryEntry) -> bool { debug_assert!(self.inflight.is_some()); - self.try_enqueue(None, "test", entry) + self.try_enqueue(None, "test", entry, CaptureHealth::default()) } #[cfg(test)] @@ -240,6 +258,7 @@ async fn persist_drained(item: QueuedTrajectory) { }; let backend = item.backend; let entry = item.entry; + let health = item.health; let span = tracing::info_span!( "trajectory_outbox.persist", tenant = %entry.tenant, @@ -258,6 +277,16 @@ async fn persist_drained(item: QueuedTrajectory) { Err(error) => { record_persist_latency(&entry, backend, "error", started_at.elapsed()); tracing::error!(error = %error, "failed to persist trajectory entry from outbox"); + // A write that failed loses the entry exactly as surely as a + // full queue does. Both go through one place, so neither can + // be the one that stays silent. + record_capture_loss( + Some(Arc::clone(&sink)), + backend, + &entry, + "persist_failed", + &health, + ); } } } @@ -274,55 +303,48 @@ pub(crate) fn try_record( backend: &'static str, sink: Arc, entry: TrajectoryEntry, + health: CaptureHealth, ) -> bool { - global().try_record(backend, sink, entry) + global().try_record(backend, sink, entry, health) +} + +/// Next position in this process's capture order. +/// +/// Monotonic and gap-free within the process, which is all the session read +/// needs: it is a tie-break inside one `created_at` tick, and a restart always +/// advances the wall clock past the tick it was in. +fn next_capture_seq() -> i64 { + static CAPTURE_SEQ: AtomicU64 = AtomicU64::new(0); + // Saturating rather than wrapping: a wrap would sort later rows before + // earlier ones, and 2^63 captures in one process is not reachable. + CAPTURE_SEQ + .fetch_add(1, Ordering::Relaxed) + .min(i64::MAX as u64) as i64 } impl crate::state::ServerState { - pub(crate) fn enqueue_trajectory_entry(&self, entry: TrajectoryEntry) -> bool { + pub(crate) fn enqueue_trajectory_entry(&self, mut entry: TrajectoryEntry) -> bool { let Some((backend, sink)) = self.trajectory_sink() else { return true; }; - try_record(backend, sink, entry) + // Stamped here rather than at each capture site: this is the single + // point every captured entry passes through, and it is still on the + // capturing thread, before the entry is handed to a persistence task + // that may land in any order. + entry.capture_seq = Some(next_capture_seq()); + // Single choke point for every capture site: a captured body is + // scrubbed of secret-named fields and then bounded, so a new capture + // site cannot forget either, a stalled drain cannot accumulate whole + // request bodies in memory, and the truncation preview can never carry + // a value the full body would have had redacted. + if let Some(body) = entry.request_body.take() { + let redacted = crate::storage::redact_secrets(body); + entry.request_body = Some(crate::storage::bounded_request_body(redacted)); + } + try_record(backend, sink, entry, self.capture_health.clone()) } } #[cfg(test)] -mod tests { - use super::*; - use crate::state::trajectory::{TrajectoryEntry, TrajectorySource}; - - fn entry(id: &str) -> TrajectoryEntry { - TrajectoryEntry { - timestamp: "2026-04-28T00:00:00Z".to_string(), - tenant: "tenant".to_string(), - entity_type: "Session".to_string(), - entity_id: id.to_string(), - action: "ProgressMade".to_string(), - success: true, - from_status: Some("Running".to_string()), - to_status: Some("Running".to_string()), - error: None, - agent_id: Some("agent".to_string()), - session_id: Some("session".to_string()), - authz_denied: None, - denied_resource: None, - denied_module: None, - source: Some(TrajectorySource::Entity), - spec_governed: Some(true), - agent_type: None, - request_body: None, - intent: None, - matched_policy_ids: None, - } - } - - #[test] - fn try_record_reports_drop_when_full() { - let outbox = TrajectoryOutbox::for_tests(1); - assert!(outbox.try_record_for_test(entry("one"))); - assert!(!outbox.try_record_for_test(entry("two"))); - assert_eq!(outbox.dropped_total(), 1); - assert_eq!(outbox.depth(), 1); - } -} +#[path = "trajectory_outbox_test.rs"] +mod tests; diff --git a/crates/temper-server/src/trajectory_outbox/capture_loss.rs b/crates/temper-server/src/trajectory_outbox/capture_loss.rs new file mode 100644 index 000000000..95e19bee4 --- /dev/null +++ b/crates/temper-server/src/trajectory_outbox/capture_loss.rs @@ -0,0 +1,407 @@ +//! Recording a captured entry that never reached storage. +//! +//! The outbox next door is bounded and its writes can fail. Either way an +//! action the kernel captured is gone, and a loss that leaves no trace lets a +//! run with holes in it pass a conformance check later. So every loss is +//! counted, and the session it belonged to gets a marker row the checker reads +//! as an evidence gap (`crate::conformance::CAPTURE_LOSS_ENTITY_TYPE`). +//! +//! # The count leads the write +//! +//! [`CaptureHealth`] is incremented **before** the marker write is scheduled and +//! decremented only once storage confirms it. That ordering is the whole +//! guarantee: a process killed, a runtime cancelled, or a simulation ended +//! between the loss and the write leaves the count standing, so the server +//! reads degraded rather than clean. Counting after the write would make every +//! interruption look like a healthy capture — the exact fail-open this +//! mechanism exists to close. +//! +//! Two other things can go wrong, and neither is allowed to be silent: the +//! dedupe set can fill up, in which case the marker is written without being +//! remembered rather than skipped; and the write can keep failing, in which +//! case it is retried and the count simply stays up. +//! +//! # Scheduling +//! +//! Markers go through one bounded queue drained by one long-lived worker, the +//! same shape `crate::ots_trajectory_outbox` uses. `temper-server` is +//! simulation-visible, so the capture path spawns nothing per loss and holds no +//! `HashMap`/`HashSet`: a detached task per marker gives a simulation a +//! different interleaving on every run, and a hashed set gives it a different +//! iteration order. + +use std::collections::BTreeSet; +use std::sync::{ + Arc, Mutex, OnceLock, + atomic::{AtomicU64, Ordering}, +}; +use std::time::Duration; + +use opentelemetry::KeyValue; +use temper_runtime::scheduler::sim_now; +use tokio::sync::mpsc; + +use super::{metrics, next_capture_seq, record_dropped}; +use crate::conformance::{CAPTURE_LOSS_ACTION, CAPTURE_LOSS_ENTITY_TYPE}; +use crate::state::trajectory::{TrajectoryEntry, TrajectorySource}; +use crate::storage::TrajectorySink; + +/// Cap on sessions this process remembers having marked. +/// +/// The set exists only to keep a burst of losses in one session from becoming +/// a burst of identical marker writes. Past the cap the marker is still +/// written — the session that lost a row is the one that needs it — and only +/// the memory of having written it is dropped, so a long-lived process cannot +/// grow the set without bound and no loss goes unmarked to buy that. +pub(super) const MAX_MARKED_SESSIONS: usize = 4_096; + +/// In-flight marker writes the queue will hold. +/// +/// Small on purpose: a marker is one row, the worker drains them in order, and +/// a queue that cannot accept one leaves the loss counted rather than dropping +/// it silently. +const MARKER_QUEUE_CAPACITY: usize = 1_024; + +/// Attempts a marker write gets before the worker gives up on it. +const CAPTURE_MARKER_ATTEMPTS: u32 = 4; + +/// First backoff between marker attempts; doubles each time. +const CAPTURE_MARKER_RETRY_DELAY: Duration = Duration::from_millis(50); + +/// Capture losses this server has not confirmed recorded. +/// +/// Incremented when an entry is lost, decremented only when the marker naming +/// its session is durably stored. Non-zero therefore covers every state in +/// which a reader could be misled: the write is still in flight, it exhausted +/// its retries, the queue would not take it, or the process died before any of +/// that finished. In all of them some stored session is missing rows with +/// nothing to say so, which is why a conformance check reports it and cannot +/// pass. +/// +/// Carried on [`crate::state::ServerState`] rather than in a global, so it +/// belongs to one server's capture path and a check can ask its own server +/// whether the record it is about to read can be trusted. +#[derive(Clone, Default)] +pub(crate) struct CaptureHealth { + unconfirmed_losses: Arc, +} + +impl CaptureHealth { + /// Losses whose marker this server has not seen stored. + #[allow(dead_code)] // False positive: read by api/trajectory_analysis.rs under the observe feature + pub(crate) fn unconfirmed_losses(&self) -> u64 { + self.unconfirmed_losses.load(Ordering::Relaxed) + } + + /// Whether any loss is still unaccounted for. + #[allow(dead_code)] // False positive: read by api/trajectory_analysis.rs under the observe feature + pub(crate) fn is_degraded(&self) -> bool { + self.unconfirmed_losses() > 0 + } + + /// Count a loss, before anything is attempted on its behalf. + pub(super) fn record_unconfirmed_loss(&self) { + self.unconfirmed_losses.fetch_add(1, Ordering::Relaxed); + } + + /// Clear one loss, once its marker is durably stored. + fn confirm_loss_recorded(&self) { + // Saturating: a decrement that ran without a matching increment would + // wrap to a permanently degraded server. + let _ = + self.unconfirmed_losses + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |count| { + Some(count.saturating_sub(1)) + }); + } +} + +fn record_capture_loss_marker(tenant: &str, result: &'static str) { + metrics().capture_loss_marker_total.add( + 1, + &[ + KeyValue::new("tenant", tenant.to_string()), + KeyValue::new("result", result), + ], + ); +} + +/// One session's marker, waiting for the worker. +pub(super) struct QueuedMarker { + sink: Arc, + tenant: String, + session_id: String, + reason: &'static str, + health: CaptureHealth, +} + +/// The queue every marker write goes through. +/// +/// One sender, one worker, started once. Markers are written in the order they +/// were queued, so a simulation replaying the same losses sees the same writes +/// in the same order. +fn marker_queue() -> &'static mpsc::Sender { + static QUEUE: OnceLock> = OnceLock::new(); + QUEUE.get_or_init(|| { + let (sender, receiver) = mpsc::channel(MARKER_QUEUE_CAPACITY); + // determinism-ok: one long-lived drain task for external observe + // persistence, started once — the same shape as the OTS trajectory + // outbox worker, and not a task per captured loss. + tokio::spawn(run_marker_worker(receiver)); + sender + }) +} + +/// Drain markers in queue order, one at a time. +async fn run_marker_worker(mut receiver: mpsc::Receiver) { + while let Some(queued) = receiver.recv().await { + persist_capture_loss_marker(queued, CAPTURE_MARKER_RETRY_DELAY).await; + } +} + +/// Sessions this process has already written a capture-loss marker for. +/// +/// One marker per session is the whole signal: it says this session's stored +/// record has holes. Writing another on every subsequent loss would put a +/// write storm on the backend that is already the reason entries are being +/// lost. +/// +/// A `BTreeSet` rather than a hashed one: this crate is simulation-visible and +/// ordered collections are what keep a replay identical. +fn marked_sessions() -> &'static Mutex> { + static MARKED: OnceLock>> = OnceLock::new(); + MARKED.get_or_init(|| Mutex::new(BTreeSet::new())) +} + +/// What the remembered set says about writing this session's marker. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum MarkerClaim { + /// Not marked yet, and now remembered: write it. + Claimed, + /// Already marked, or a marker is in flight: writing another says nothing + /// the first one did not. + AlreadyMarked, + /// The set is full, so this write cannot be remembered. Write it anyway — + /// a duplicate marker is noise, an unmarked lossy session is a silent hole. + Unremembered, +} + +/// Claim the right to write this session's marker. +pub(super) fn claim_capture_loss_marker(tenant: &str, session_id: &str) -> MarkerClaim { + let Ok(mut marked) = marked_sessions().lock() else { + // The set is poisoned, so nothing can be deduplicated against it. Write + // rather than skip: the point of the marker is that its absence is + // indistinguishable from a clean run. + return MarkerClaim::Unremembered; + }; + claim_in(&mut marked, tenant, session_id) +} + +/// The claim decision itself, over whichever set is being deduplicated against. +/// +/// Split from the lock so the full-set behaviour can be exercised on a set of +/// its own rather than by filling the one every other capture path shares. +pub(super) fn claim_in( + marked: &mut BTreeSet<(String, String)>, + tenant: &str, + session_id: &str, +) -> MarkerClaim { + let key = (tenant.to_string(), session_id.to_string()); + if marked.contains(&key) { + return MarkerClaim::AlreadyMarked; + } + if marked.len() >= MAX_MARKED_SESSIONS { + tracing::warn!( + marked_sessions = marked.len(), + tenant, + session_id, + "capture-loss marker dedupe set is full; marking without remembering" + ); + return MarkerClaim::Unremembered; + } + marked.insert(key); + MarkerClaim::Claimed +} + +/// Give the claim back so a later loss in the same session tries again. +pub(super) fn release_capture_loss_marker(tenant: &str, session_id: &str) { + if let Ok(mut marked) = marked_sessions().lock() { + marked.remove(&(tenant.to_string(), session_id.to_string())); + } +} + +/// Record that a captured entry never reached storage. +/// +/// The loss is counted first — on the metric and on [`CaptureHealth`] — and the +/// marker naming its session is queued after. Nothing between those two points +/// can make the loss disappear: if the queue refuses it, if the worker never +/// runs it, if the process dies first, the count is already standing and the +/// server reads degraded. +pub(super) fn record_capture_loss( + sink: Option>, + backend: &'static str, + lost: &TrajectoryEntry, + reason: &'static str, + health: &CaptureHealth, +) { + record_dropped(lost, backend, reason); + tracing::warn!( + tenant = %lost.tenant, + entity_type = %lost.entity_type, + entity_id = %lost.entity_id, + action = %lost.action, + session_id = lost.session_id.as_deref().unwrap_or(""), + reason, + "trajectory capture lost an entry" + ); + + // An entry with no session was never part of a run a conformance check can + // read — a session read would not have returned it — so no session's report + // is wrong for its absence and there is nothing to mark. + let Some(session_id) = lost + .session_id + .as_deref() + .filter(|session| !session.is_empty()) + else { + return; + }; + // No sink means no storage at all, and a conformance check against this + // server answers 503 rather than reading a short session. + let Some(sink) = sink else { + return; + }; + if claim_capture_loss_marker(&lost.tenant, session_id) == MarkerClaim::AlreadyMarked { + return; + } + + health.record_unconfirmed_loss(); + let queued = QueuedMarker { + sink, + tenant: lost.tenant.clone(), + session_id: session_id.to_string(), + reason, + health: health.clone(), + }; + if let Err(error) = marker_queue().try_send(queued) { + // The count stays up, which is the point: a marker that was never + // queued is a loss nothing durable will report. + record_capture_loss_marker(&lost.tenant, "unqueued"); + release_capture_loss_marker(&lost.tenant, session_id); + tracing::error!( + tenant = %lost.tenant, + session_id, + reason = %error, + "could not queue a trajectory capture-loss marker; the loss stays unconfirmed" + ); + } +} + +/// Write one session's capture-loss marker, retrying until it lands. +/// +/// The retry belongs here rather than to the next loss: a session that loses +/// one row and fails to mark it would otherwise stay silently incomplete until +/// something else in the same session happened to be lost, which for a finished +/// run is never. +/// +/// The loss is cleared from [`CaptureHealth`] only on a confirmed write. Every +/// other ending — exhausted retries, a cancelled worker, a dead process — +/// leaves it counted. +pub(super) async fn persist_capture_loss_marker(queued: QueuedMarker, retry_delay: Duration) { + let QueuedMarker { + sink, + tenant, + session_id, + reason, + health, + } = queued; + let marker = capture_loss_marker(&tenant, &session_id, reason); + let mut delay = retry_delay; + let mut last_error = String::new(); + for attempt in 1..=CAPTURE_MARKER_ATTEMPTS { + match sink.persist_trajectory_entry(&marker).await { + Ok(()) => { + record_capture_loss_marker(&tenant, "stored"); + health.confirm_loss_recorded(); + return; + } + Err(error) => { + last_error = error.to_string(); + tracing::warn!( + error = %last_error, + tenant = %tenant, + session_id = %session_id, + attempt, + max_attempts = CAPTURE_MARKER_ATTEMPTS, + "failed to persist trajectory capture-loss marker; retrying" + ); + if attempt < CAPTURE_MARKER_ATTEMPTS && !delay.is_zero() { + tokio::time::sleep(delay).await; + } + delay = delay.saturating_mul(2); + } + } + } + + record_capture_loss_marker(&tenant, "failed"); + // The count is left standing, and the session is no longer remembered as + // marked, so a later loss in it gets a fresh attempt. + release_capture_loss_marker(&tenant, &session_id); + tracing::error!( + error = %last_error, + tenant = %tenant, + session_id = %session_id, + attempts = CAPTURE_MARKER_ATTEMPTS, + "trajectory capture-loss marker exhausted its retries; the loss stays unconfirmed" + ); +} + +/// Build a queued marker without going through the shared queue. +#[cfg(test)] +pub(super) fn queued_marker_for_test( + sink: Arc, + tenant: &str, + session_id: &str, + health: CaptureHealth, +) -> QueuedMarker { + QueuedMarker { + sink, + tenant: tenant.to_string(), + session_id: session_id.to_string(), + reason: "outbox_full", + health, + } +} + +/// The row that tells a later reader this session's record is incomplete. +/// +/// Not an actor's entity type and not a declared action, so the conformance +/// checker counts it as an evidence gap rather than judging it +/// (`crate::conformance::walk::row_disposition`). +fn capture_loss_marker(tenant: &str, session_id: &str, reason: &str) -> TrajectoryEntry { + TrajectoryEntry { + timestamp: sim_now().to_rfc3339(), + tenant: tenant.to_string(), + entity_type: CAPTURE_LOSS_ENTITY_TYPE.to_string(), + entity_id: session_id.to_string(), + action: CAPTURE_LOSS_ACTION.to_string(), + success: false, + from_status: None, + to_status: None, + error: Some(format!( + "trajectory capture lost at least one entry for this session ({reason}); the stored \ + record of this run is incomplete" + )), + agent_id: None, + session_id: Some(session_id.to_string()), + authz_denied: None, + denied_resource: None, + denied_module: None, + source: Some(TrajectorySource::Platform), + spec_governed: Some(false), + agent_type: None, + request_body: None, + intent: None, + matched_policy_ids: None, + capture_seq: Some(next_capture_seq()), + } +} diff --git a/crates/temper-server/src/trajectory_outbox_test.rs b/crates/temper-server/src/trajectory_outbox_test.rs new file mode 100644 index 000000000..ac36f0285 --- /dev/null +++ b/crates/temper-server/src/trajectory_outbox_test.rs @@ -0,0 +1,295 @@ +use super::*; +use crate::state::trajectory::{TrajectoryEntry, TrajectorySource}; + +fn entry(id: &str) -> TrajectoryEntry { + TrajectoryEntry { + timestamp: "2026-04-28T00:00:00Z".to_string(), + tenant: "tenant".to_string(), + entity_type: "Session".to_string(), + entity_id: id.to_string(), + action: "ProgressMade".to_string(), + success: true, + from_status: Some("Running".to_string()), + to_status: Some("Running".to_string()), + error: None, + agent_id: Some("agent".to_string()), + session_id: Some("session".to_string()), + authz_denied: None, + denied_resource: None, + denied_module: None, + source: Some(TrajectorySource::Entity), + spec_governed: Some(true), + agent_type: None, + request_body: None, + intent: None, + matched_policy_ids: None, + capture_seq: None, + } +} + +#[test] +fn try_record_reports_drop_when_full() { + let outbox = TrajectoryOutbox::for_tests(1); + assert!(outbox.try_record_for_test(entry("one"))); + assert!(!outbox.try_record_for_test(entry("two"))); + assert_eq!(outbox.dropped_total(), 1); + assert_eq!(outbox.depth(), 1); +} + +/// A sink whose every write fails, standing in for a backend that is down. +struct FailingSink; + +#[async_trait::async_trait] +impl crate::storage::TrajectorySink for FailingSink { + async fn persist_trajectory_entry(&self, _entry: &TrajectoryEntry) -> Result<(), String> { + Err("backend unavailable".to_string()) + } +} + +#[test] +fn a_session_is_marked_once() { + // A backlog drops entries in bursts. One marker says the session's record + // has holes; a thousand say the same thing to a backend that is already + // the reason they are being dropped. + release_capture_loss_marker("tenant", "burst-session"); + + assert_eq!( + claim_capture_loss_marker("tenant", "burst-session"), + MarkerClaim::Claimed + ); + assert_eq!( + claim_capture_loss_marker("tenant", "burst-session"), + MarkerClaim::AlreadyMarked + ); + release_capture_loss_marker("tenant", "burst-session"); + assert_eq!( + claim_capture_loss_marker("tenant", "burst-session"), + MarkerClaim::Claimed, + "a marker that failed to store leaves the session unmarked, so the next loss retries" + ); + release_capture_loss_marker("tenant", "burst-session"); +} + +#[test] +fn a_full_dedupe_set_still_marks_the_session() { + // The overflow behaviour that matters: past the cap the marker is written + // without being remembered. Skipping it instead would make a lossy session + // indistinguishable from a clean one, which is the whole failure the + // marker exists to prevent. + // On a set of its own, so filling it cannot make every other capture path + // in the process look full. + let mut marked = std::collections::BTreeSet::new(); + for index in 0..MAX_MARKED_SESSIONS { + assert_eq!( + claim_in(&mut marked, "tenant", &format!("session-{index}")), + MarkerClaim::Claimed + ); + } + + assert_eq!( + claim_in(&mut marked, "tenant", "one-too-many"), + MarkerClaim::Unremembered, + "past the cap the marker is still written, only the memory of it is dropped" + ); + assert_eq!( + claim_in(&mut marked, "tenant", "one-too-many"), + MarkerClaim::Unremembered, + "an unremembered session must not later read as already marked" + ); + assert_eq!(marked.len(), MAX_MARKED_SESSIONS, "the set stays bounded"); +} + +#[tokio::test] +async fn a_loss_interrupted_before_its_marker_lands_leaves_the_server_degraded() { + // The process-death edge. The loss is counted before the write is even + // queued, so every way the write can fail to happen — killed process, + // cancelled runtime, ended simulation, worker that never ran — leaves the + // count standing and the next conformance check degraded. + let health = CaptureHealth::default(); + assert!(!health.is_degraded()); + + health.record_unconfirmed_loss(); + + assert!( + health.is_degraded(), + "a loss counted but not yet written must read as degraded, not clean" + ); + assert_eq!(health.unconfirmed_losses(), 1); + // Dropping everything here is the cancellation: nothing else runs. +} + +#[tokio::test] +async fn a_marker_that_never_lands_leaves_the_loss_unconfirmed() { + // Retries run on the worker rather than waiting for the next loss — for a + // finished run there is no next loss — and when they are exhausted the + // count simply stays up, because nothing durable will tell a reader this + // session has a hole in it. + release_capture_loss_marker("tenant", "doomed-session"); + let health = CaptureHealth::default(); + health.record_unconfirmed_loss(); + + persist_capture_loss_marker( + queued_marker_for_test( + Arc::new(FailingSink), + "tenant", + "doomed-session", + health.clone(), + ), + Duration::ZERO, + ) + .await; + + assert_eq!( + health.unconfirmed_losses(), + 1, + "an unmarkable loss has to stay visible, and storage is not available" + ); + assert_eq!( + claim_capture_loss_marker("tenant", "doomed-session"), + MarkerClaim::Claimed, + "the exhausted session is released so a later loss in it tries again" + ); + release_capture_loss_marker("tenant", "doomed-session"); +} + +#[tokio::test] +async fn only_a_confirmed_write_clears_the_loss() { + let dir = tempfile::tempdir().expect("create temp dir"); + let db_url = format!("file:{}", dir.path().join("marker-ok.db").display()); + let store = temper_store_turso::TursoEventStore::new(&db_url, None) + .await + .expect("create local turso store"); + let health = CaptureHealth::default(); + health.record_unconfirmed_loss(); + assert!(health.is_degraded()); + + persist_capture_loss_marker( + queued_marker_for_test(Arc::new(store), "tenant", "healthy-session", health.clone()), + Duration::ZERO, + ) + .await; + + assert!( + !health.is_degraded(), + "the marker is stored, so the loss it records is accounted for" + ); +} + +#[test] +fn a_degraded_capture_stops_any_session_from_passing() { + // A marker says which session lost a row. The degraded count says a row + // was lost and could not be marked at all, so no session read from this + // server can be assumed whole — including one that looks clean. + let automaton = temper_spec::automaton::parse_automaton(include_str!( + "../../../test-fixtures/specs/order.ioa.toml" + )) + .expect("order fixture parses"); + let rows = vec![temper_store_turso::TursoTrajectoryRow { + tenant: "tenant".to_string(), + entity_type: "Order".to_string(), + entity_id: "order-1".to_string(), + action: "AddItem".to_string(), + success: true, + from_status: Some("Draft".to_string()), + to_status: Some("Draft".to_string()), + error: None, + agent_id: None, + session_id: Some("session".to_string()), + authz_denied: None, + denied_resource: None, + denied_module: None, + source: Some("Entity".to_string()), + spec_governed: Some(true), + created_at: "2026-01-01T00:00:00Z".to_string(), + request_body: None, + intent: None, + matched_policy_ids: None, + capture_seq: None, + }]; + + let input = |capture_degraded| crate::conformance::ConformanceInput { + automaton: &automaton, + kernel_rows: &rows, + ots_trajectory: None, + rows_truncated: false, + spec_resolution: crate::conformance::SpecResolution::Pinned, + capture_degraded, + }; + + assert!( + crate::conformance::check_conformance(input(false)).passed, + "the same run passes when the capture path is healthy" + ); + let degraded = crate::conformance::check_conformance(input(true)); + assert!(!degraded.passed); + assert!(!degraded.evidence_complete); + assert!( + degraded + .evidence_gaps + .iter() + .any(|gap| gap.contains("could not record against any session")), + "{:?}", + degraded.evidence_gaps + ); +} + +#[tokio::test] +async fn a_lost_entry_leaves_a_marker_the_checker_reads_as_missing_evidence() { + // The end the finding cares about: after a loss, a conformance check + // of that session must not come back `passed`. + let dir = tempfile::tempdir().expect("create temp dir"); + let db_url = format!("file:{}", dir.path().join("capture-loss.db").display()); + let store = temper_store_turso::TursoEventStore::new(&db_url, None) + .await + .expect("create local turso store"); + let session = "capture-loss-session"; + + crate::storage::TrajectorySink::persist_trajectory_entry( + &store, + &TrajectoryEntry { + session_id: Some(session.to_string()), + ..entry("order-1") + }, + ) + .await + .expect("persist the entry that survived"); + + persist_capture_loss_marker( + queued_marker_for_test( + Arc::new(store.clone()), + "tenant", + session, + CaptureHealth::default(), + ), + Duration::ZERO, + ) + .await; + release_capture_loss_marker("tenant", session); + + let rows = store + .query_trajectories_by_session(session, Some("tenant"), None, 10) + .await + .expect("read the session back"); + assert_eq!(rows.len(), 2, "the marker is stored alongside the run"); + + let automaton = temper_spec::automaton::parse_automaton(include_str!( + "../../../test-fixtures/specs/order.ioa.toml" + )) + .expect("order fixture parses"); + let report = crate::conformance::check_conformance(crate::conformance::ConformanceInput { + automaton: &automaton, + kernel_rows: &rows, + ots_trajectory: None, + rows_truncated: false, + spec_resolution: crate::conformance::SpecResolution::Pinned, + capture_degraded: false, + }); + + assert_eq!(report.stats.capture_loss_markers, 1); + assert!( + !report.passed, + "a run whose capture is known to have lost rows cannot pass: {:?}", + report.evidence_gaps + ); + assert!(!report.evidence_complete); +} diff --git a/crates/temper-server/src/trigger/dispatcher.rs b/crates/temper-server/src/trigger/dispatcher.rs index 96a4ec522..8aec4af0b 100644 --- a/crates/temper-server/src/trigger/dispatcher.rs +++ b/crates/temper-server/src/trigger/dispatcher.rs @@ -238,6 +238,7 @@ impl ReactionDispatcher { effective_params, &dispatch_ctx, false, + None, ) .await; @@ -397,7 +398,7 @@ fn effective_trigger_security_context(agent_ctx: &AgentContext) -> SecurityConte return security_ctx.clone(); } - let mut security_ctx = SecurityContext::from_headers(&[]).with_agent_context( + let mut security_ctx = SecurityContext::anonymous().with_agent_context( agent_ctx.agent_id.as_deref(), agent_ctx.session_id.as_deref(), agent_ctx.agent_type.as_deref(), diff --git a/crates/temper-server/src/webhooks/dispatcher.rs b/crates/temper-server/src/webhooks/dispatcher.rs index 434dd9d35..57ea3e8df 100644 --- a/crates/temper-server/src/webhooks/dispatcher.rs +++ b/crates/temper-server/src/webhooks/dispatcher.rs @@ -240,6 +240,7 @@ mod tests { request_body: None, intent: None, matched_policy_ids: None, + capture_seq: None, } } diff --git a/crates/temper-server/src/webhooks/receiver.rs b/crates/temper-server/src/webhooks/receiver.rs index bda797289..57c04f38f 100644 --- a/crates/temper-server/src/webhooks/receiver.rs +++ b/crates/temper-server/src/webhooks/receiver.rs @@ -7,17 +7,23 @@ use std::collections::BTreeMap; +use axum::body::{Body, Bytes, to_bytes}; use axum::extract::{Path, Query, State}; -use axum::http::{Method, StatusCode}; +use axum::http::{HeaderMap, Method, Request, StatusCode}; use axum::response::IntoResponse; use tracing::instrument; +use crate::aws_sigv4::{hex_encode, hmac_sha256}; use crate::request_context::AgentContext; +use crate::secrets::template::resolve_secret_templates; use crate::state::ServerState; +use temper_authz::SecurityContext; use temper_runtime::tenant::TenantId; use temper_spec::automaton::Webhook; +const WEBHOOK_BODY_BUDGET_BYTES: usize = 64 * 1024; + /// Handle an inbound webhook request. /// /// Route: `GET|POST /webhooks/{tenant}/{*path}` @@ -36,6 +42,7 @@ pub async fn handle_webhook( State(state): State, Path((tenant_str, webhook_path)): Path<(String, String)>, Query(query): Query>, + request: Request, ) -> impl IntoResponse { let tenant = TenantId::new(&tenant_str); @@ -80,6 +87,37 @@ pub async fn handle_webhook( ); }; + let path_and_query = request + .uri() + .path_and_query() + .map(|pq| pq.as_str().to_string()) + .unwrap_or_else(|| request.uri().path().to_string()); + let headers = request.headers().clone(); + let body = match to_bytes(request.into_body(), WEBHOOK_BODY_BUDGET_BYTES).await { + Ok(body) => body, + Err(_) => { + return ( + StatusCode::PAYLOAD_TOO_LARGE, + format!("Webhook body exceeds {WEBHOOK_BODY_BUDGET_BYTES} bytes"), + ); + } + }; + + let security_ctx = match admit_webhook( + &state, + &tenant, + &webhook, + &method, + &path_and_query, + &headers, + &body, + ) { + Ok(ctx) => ctx, + Err(status) => { + return (status, "Webhook admission denied".to_string()); + } + }; + // Extract action parameters from the configured extraction map. let mut params = serde_json::Map::new(); for (param_name, source) in &webhook.extract { @@ -89,11 +127,30 @@ pub async fn handle_webhook( } let action = &webhook.action; + let mut resource_attrs = BTreeMap::new(); + resource_attrs.insert( + "id".to_string(), + serde_json::Value::String(entity_id.clone()), + ); + if let Err(denial) = state.authorize_with_context( + &security_ctx, + action, + &entity_type, + &resource_attrs, + tenant.as_str(), + ) { + tracing::warn!(reason = %denial, webhook = %webhook.name, "webhook action denied"); + return ( + StatusCode::FORBIDDEN, + format!("Webhook action denied: {denial}"), + ); + } + let agent_ctx = AgentContext { - security_ctx: None, + security_ctx: Some(security_ctx), agent_id: Some(format!("webhook:{}", webhook.name)), session_id: None, - agent_type: None, + agent_type: Some("webhook".to_string()), intent: None, ..AgentContext::default() }; @@ -123,6 +180,78 @@ pub async fn handle_webhook( } } +/// Class B admission: HMAC is required and fail-closed. +fn admit_webhook( + state: &ServerState, + tenant: &TenantId, + webhook: &Webhook, + method: &Method, + path_and_query: &str, + headers: &HeaderMap, + body: &Bytes, +) -> Result { + let Some(secret_template) = webhook.hmac_secret.as_deref().filter(|s| !s.is_empty()) else { + tracing::warn!(webhook = %webhook.name, "webhook missing hmac_secret"); + return Err(StatusCode::UNAUTHORIZED); + }; + let Some(header_name) = webhook.hmac_header.as_deref().filter(|s| !s.is_empty()) else { + tracing::warn!(webhook = %webhook.name, "webhook missing hmac_header"); + return Err(StatusCode::UNAUTHORIZED); + }; + let Some(vault) = state.secrets_vault.as_ref() else { + tracing::warn!(webhook = %webhook.name, "webhook HMAC required but no secrets vault"); + return Err(StatusCode::UNAUTHORIZED); + }; + let mut templates = BTreeMap::new(); + templates.insert("hmac".to_string(), secret_template.to_string()); + let resolved = resolve_secret_templates(&templates, vault, tenant.as_str()); + let secret = resolved.get("hmac").cloned().unwrap_or_default(); + if secret.is_empty() || secret.contains("{secret:") { + tracing::warn!(webhook = %webhook.name, "webhook HMAC secret did not resolve"); + return Err(StatusCode::UNAUTHORIZED); + } + let Some(provided) = headers + .get(header_name) + .and_then(|value| value.to_str().ok()) + else { + tracing::warn!(webhook = %webhook.name, header = %header_name, "missing HMAC header"); + return Err(StatusCode::UNAUTHORIZED); + }; + let mut payload = + Vec::with_capacity(method.as_str().len() + path_and_query.len() + body.len() + 2); + payload.extend_from_slice(method.as_str().as_bytes()); + payload.push(b'\n'); + payload.extend_from_slice(path_and_query.as_bytes()); + payload.push(b'\n'); + payload.extend_from_slice(body); + if !hmac_hex_matches(secret.as_bytes(), &payload, provided) { + tracing::warn!(webhook = %webhook.name, "HMAC mismatch"); + return Err(StatusCode::UNAUTHORIZED); + } + Ok(SecurityContext::from_resolved_identity( + &format!("webhook:{}", webhook.name), + "webhook", + None, + )) +} + +fn hmac_hex_matches(secret: &[u8], payload: &[u8], provided: &str) -> bool { + let expected = hex_encode(&hmac_sha256(secret, payload)); + let provided = provided + .strip_prefix("sha256=") + .unwrap_or(provided) + .to_ascii_lowercase(); + if expected.len() != provided.len() { + return false; + } + expected + .as_bytes() + .iter() + .zip(provided.as_bytes()) + .fold(0u8, |acc, (left, right)| acc | (left ^ right)) + == 0 +} + /// Find a webhook matching (tenant, path) in the registry. /// /// Checks the pre-indexed `webhook_routes` map first, then falls back to @@ -158,193 +287,5 @@ fn extract_param(source: &str, query: &BTreeMap) -> Option ServerState { - let csdl = parse_csdl(CSDL_XML).unwrap(); - let system = ActorSystem::new("webhook-test"); - let state = ServerState::new(system, csdl, CSDL_XML.to_string()); - - // Register tenant with webhook-enabled spec. - { - let mut registry = state.registry.write().unwrap(); - let csdl2 = parse_csdl(CSDL_XML).unwrap(); - registry.register_tenant( - "test-tenant", - csdl2, - CSDL_XML.to_string(), - &[("Order", ORDER_IOA_WITH_WEBHOOK)], - ); - } - - state - } - - fn build_test_router() -> axum::Router { - crate::router::build_router(build_test_state()) - } - - #[tokio::test] - async fn webhook_dispatches_action() { - let state = build_test_state(); - let tenant = TenantId::new("test-tenant"); - - // Create entity directly via dispatch. - let _create = state - .get_or_create_tenant_entity( - &tenant, - "Order", - "ent-1", - serde_json::json!({"id": "ent-1"}), - ) - .await - .expect("entity creation should succeed"); - - // Submit to move to "Submitted". - let submit = state - .dispatch_tenant_action( - &tenant, - "Order", - "ent-1", - "SubmitOrder", - serde_json::json!({}), - &AgentContext::default(), - ) - .await - .expect("SubmitOrder should succeed"); - assert!(submit.success, "SubmitOrder should succeed"); - assert_eq!(submit.state.status, "Submitted"); - - // Build router and call webhook. - let app = crate::router::build_router(state); - let response = app - .oneshot( - Request::builder() - .method("GET") - .uri("/webhooks/test-tenant/oauth/callback?state=ent-1&code=abc123") - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - - assert_eq!(response.status(), StatusCode::OK); - let body = axum::body::to_bytes(response.into_body(), 1024 * 1024) - .await - .unwrap(); - let json: serde_json::Value = serde_json::from_slice(&body).unwrap(); - assert!( - json["success"].as_bool().unwrap_or(false), - "HandleOAuthCallback should succeed" - ); - assert_eq!(json["state"]["status"], "Authorized"); - } - - #[tokio::test] - async fn webhook_missing_entity_id_returns_400() { - let app = build_test_router(); - - let response = app - .oneshot( - Request::builder() - .method("GET") - .uri("/webhooks/test-tenant/oauth/callback?code=abc123") - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - - assert_eq!(response.status(), StatusCode::BAD_REQUEST); - } - - #[tokio::test] - async fn webhook_unknown_path_returns_404() { - let app = build_test_router(); - - let response = app - .oneshot( - Request::builder() - .method("GET") - .uri("/webhooks/test-tenant/nonexistent/path?entity_id=ent-1") - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap(); - - assert_eq!(response.status(), StatusCode::NOT_FOUND); - } - - #[tokio::test] - async fn webhook_extracts_query_params() { - let query: BTreeMap = [ - ("code".to_string(), "auth-code-123".to_string()), - ("state".to_string(), "entity-id".to_string()), - ] - .into_iter() - .collect(); - - assert_eq!( - extract_param("query.code", &query), - Some("auth-code-123".to_string()) - ); - assert_eq!( - extract_param("query.state", &query), - Some("entity-id".to_string()) - ); - assert_eq!(extract_param("query.missing", &query), None); - } -} +#[path = "receiver_test.rs"] +mod tests; diff --git a/crates/temper-server/src/webhooks/receiver_test.rs b/crates/temper-server/src/webhooks/receiver_test.rs new file mode 100644 index 000000000..bf8510785 --- /dev/null +++ b/crates/temper-server/src/webhooks/receiver_test.rs @@ -0,0 +1,358 @@ +use super::*; +use crate::aws_sigv4::{hex_encode, hmac_sha256}; +use crate::secrets::SecretsVault; +use axum::body::Body; +use axum::http::Request; +use temper_runtime::ActorSystem; +use temper_spec::csdl::parse_csdl; +use tower::ServiceExt; + +const CSDL_XML: &str = include_str!("../../../../test-fixtures/specs/model.csdl.xml"); + +/// IOA spec with a webhook declaration for OAuth callback. +const ORDER_IOA_WITH_WEBHOOK: &str = r#" +[automaton] +name = "Order" +states = ["Draft", "Submitted", "Confirmed", "Cancelled", "Authorized"] +initial = "Draft" + +[[action]] +name = "SubmitOrder" +kind = "input" +from = ["Draft"] +to = "Submitted" + +[[action]] +name = "ConfirmOrder" +kind = "input" +from = ["Submitted"] +to = "Confirmed" + +[[action]] +name = "CancelOrder" +kind = "input" +from = ["Draft", "Submitted"] +to = "Cancelled" + +[[action]] +name = "HandleOAuthCallback" +kind = "input" +from = ["Submitted"] +to = "Authorized" +params = ["code"] + +[[webhook]] +name = "oauth_callback" +path = "oauth/callback" +method = "GET" +action = "HandleOAuthCallback" +entity_lookup = "query_param" +entity_param = "state" + +[webhook.extract] +code = "query.code" +"#; + +const ORDER_IOA_WITH_HMAC_WEBHOOK: &str = r#" +[automaton] +name = "Order" +states = ["Draft", "Submitted", "Confirmed", "Cancelled", "Authorized"] +initial = "Draft" + +[[action]] +name = "SubmitOrder" +kind = "input" +from = ["Draft"] +to = "Submitted" + +[[action]] +name = "HandleOAuthCallback" +kind = "input" +from = ["Submitted"] +to = "Authorized" +params = ["code"] + +[[webhook]] +name = "oauth_callback" +path = "oauth/callback" +method = "GET" +action = "HandleOAuthCallback" +entity_lookup = "query_param" +entity_param = "state" +hmac_secret = "{secret:WEBHOOK_HMAC}" +hmac_header = "x-webhook-signature" + +[webhook.extract] +code = "query.code" +"#; + +const WEBHOOK_HMAC_SECRET: &str = "webhook-test-secret"; + +fn sign_webhook(method: &str, path_and_query: &str, body: &[u8]) -> String { + let mut payload = Vec::new(); + payload.extend_from_slice(method.as_bytes()); + payload.push(b'\n'); + payload.extend_from_slice(path_and_query.as_bytes()); + payload.push(b'\n'); + payload.extend_from_slice(body); + hex_encode(&hmac_sha256(WEBHOOK_HMAC_SECRET.as_bytes(), &payload)) +} + +fn build_test_state() -> ServerState { + let csdl = parse_csdl(CSDL_XML).unwrap(); + let system = ActorSystem::new("webhook-test"); + let state = ServerState::new(system, csdl, CSDL_XML.to_string()); + + // Register tenant with webhook-enabled spec. + { + let mut registry = state.registry.write().unwrap(); + let csdl2 = parse_csdl(CSDL_XML).unwrap(); + registry.register_tenant( + "test-tenant", + csdl2, + CSDL_XML.to_string(), + &[("Order", ORDER_IOA_WITH_WEBHOOK)], + ); + } + + state +} + +fn build_hmac_test_state() -> ServerState { + let csdl = parse_csdl(CSDL_XML).unwrap(); + let system = ActorSystem::new("webhook-hmac-test"); + let mut state = ServerState::new(system, csdl, CSDL_XML.to_string()); + { + let mut registry = state.registry.write().unwrap(); + let csdl2 = parse_csdl(CSDL_XML).unwrap(); + registry.register_tenant( + "test-tenant", + csdl2, + CSDL_XML.to_string(), + &[("Order", ORDER_IOA_WITH_HMAC_WEBHOOK)], + ); + } + let vault = SecretsVault::new(&[7u8; 32]); + vault + .cache_secret( + "test-tenant", + "WEBHOOK_HMAC", + WEBHOOK_HMAC_SECRET.to_string(), + ) + .expect("cache webhook secret"); + state = state.with_secrets_vault(vault); + state + .authz + .reload_tenant_policies( + "test-tenant", + r#"permit( + principal == Agent::"webhook:oauth_callback", + action == Action::"HandleOAuthCallback", + resource is Order + );"#, + ) + .expect("webhook permit should parse"); + state +} + +fn build_test_router() -> axum::Router { + crate::router::build_router(build_test_state()) +} + +async fn submitted_order(state: &ServerState, entity_id: &str) { + let tenant = TenantId::new("test-tenant"); + state + .get_or_create_tenant_entity( + &tenant, + "Order", + entity_id, + serde_json::json!({"id": entity_id}), + ) + .await + .expect("entity creation should succeed"); + let submit = state + .dispatch_tenant_action( + &tenant, + "Order", + entity_id, + "SubmitOrder", + serde_json::json!({}), + &AgentContext::default(), + ) + .await + .expect("SubmitOrder should succeed"); + assert!(submit.success, "SubmitOrder should succeed"); + assert_eq!(submit.state.status, "Submitted"); +} + +#[tokio::test] +async fn webhook_dispatches_action() { + let state = build_hmac_test_state(); + submitted_order(&state, "ent-1").await; + + let path_and_query = "/webhooks/test-tenant/oauth/callback?state=ent-1&code=abc123"; + let signature = sign_webhook("GET", path_and_query, b""); + let app = crate::router::build_router(state); + let response = app + .oneshot( + Request::builder() + .method("GET") + .uri(path_and_query) + .header("x-webhook-signature", signature) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + let body = axum::body::to_bytes(response.into_body(), 1024 * 1024) + .await + .unwrap(); + let json: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert!( + json["success"].as_bool().unwrap_or(false), + "HandleOAuthCallback should succeed" + ); + assert_eq!(json["state"]["status"], "Authorized"); +} + +#[tokio::test] +async fn webhook_bad_hmac_returns_401() { + let state = build_hmac_test_state(); + submitted_order(&state, "ent-1").await; + let app = crate::router::build_router(state); + let response = app + .oneshot( + Request::builder() + .method("GET") + .uri("/webhooks/test-tenant/oauth/callback?state=ent-1&code=abc123") + .header("x-webhook-signature", "deadbeef") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); +} + +#[tokio::test] +async fn webhook_valid_hmac_without_permit_returns_403() { + let state = build_hmac_test_state(); + state + .authz + .reload_tenant_policies("test-tenant", "") + .expect("clear webhook permit"); + submitted_order(&state, "ent-1").await; + let path_and_query = "/webhooks/test-tenant/oauth/callback?state=ent-1&code=abc123"; + let signature = sign_webhook("GET", path_and_query, b""); + let app = crate::router::build_router(state); + let response = app + .oneshot( + Request::builder() + .method("GET") + .uri(path_and_query) + .header("x-webhook-signature", signature) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::FORBIDDEN); +} + +#[tokio::test] +async fn webhook_without_hmac_config_returns_401() { + let state = build_test_state(); + let tenant = TenantId::new("test-tenant"); + state + .get_or_create_tenant_entity( + &tenant, + "Order", + "ent-1", + serde_json::json!({"id": "ent-1"}), + ) + .await + .expect("entity creation should succeed"); + state + .dispatch_tenant_action( + &tenant, + "Order", + "ent-1", + "SubmitOrder", + serde_json::json!({}), + &AgentContext::default(), + ) + .await + .expect("SubmitOrder should succeed"); + + let app = crate::router::build_router(state); + let response = app + .oneshot( + Request::builder() + .method("GET") + .uri("/webhooks/test-tenant/oauth/callback?state=ent-1&code=abc123") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); +} + +#[tokio::test] +async fn webhook_missing_entity_id_returns_400() { + let app = build_test_router(); + + let response = app + .oneshot( + Request::builder() + .method("GET") + .uri("/webhooks/test-tenant/oauth/callback?code=abc123") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::BAD_REQUEST); +} + +#[tokio::test] +async fn webhook_unknown_path_returns_404() { + let app = build_test_router(); + + let response = app + .oneshot( + Request::builder() + .method("GET") + .uri("/webhooks/test-tenant/nonexistent/path?entity_id=ent-1") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::NOT_FOUND); +} + +#[tokio::test] +async fn webhook_extracts_query_params() { + let query: BTreeMap = [ + ("code".to_string(), "auth-code-123".to_string()), + ("state".to_string(), "entity-id".to_string()), + ] + .into_iter() + .collect(); + + assert_eq!( + extract_param("query.code", &query), + Some("auth-code-123".to_string()) + ); + assert_eq!( + extract_param("query.state", &query), + Some("entity-id".to_string()) + ); + assert_eq!(extract_param("query.missing", &query), None); +} diff --git a/crates/temper-server/tests/dst_concurrency_retry.rs b/crates/temper-server/tests/dst_concurrency_retry.rs index e527ddc60..df8329ed5 100644 --- a/crates/temper-server/tests/dst_concurrency_retry.rs +++ b/crates/temper-server/tests/dst_concurrency_retry.rs @@ -45,6 +45,7 @@ async fn dispatch_action( params, cross_entity_booleans: BTreeMap::new(), idempotency_key: None, + expected_authorization_precondition: None, }, Duration::from_secs(5), ) diff --git a/crates/temper-server/tests/dst_entity_key_index.rs b/crates/temper-server/tests/dst_entity_key_index.rs index 07dc0a66c..325161893 100644 --- a/crates/temper-server/tests/dst_entity_key_index.rs +++ b/crates/temper-server/tests/dst_entity_key_index.rs @@ -42,6 +42,7 @@ async fn dispatch( params, cross_entity_booleans: BTreeMap::new(), idempotency_key: None, + expected_authorization_precondition: None, }, Duration::from_secs(5), ) diff --git a/crates/temper-server/tests/dst_entity_vector_index.rs b/crates/temper-server/tests/dst_entity_vector_index.rs index f26baa214..857896fd5 100644 --- a/crates/temper-server/tests/dst_entity_vector_index.rs +++ b/crates/temper-server/tests/dst_entity_vector_index.rs @@ -40,6 +40,7 @@ async fn dispatch( params, cross_entity_booleans: BTreeMap::new(), idempotency_key: None, + expected_authorization_precondition: None, }, Duration::from_secs(5), ) diff --git a/crates/temper-server/tests/dst_persistence.rs b/crates/temper-server/tests/dst_persistence.rs index 273979d82..33684ecc2 100644 --- a/crates/temper-server/tests/dst_persistence.rs +++ b/crates/temper-server/tests/dst_persistence.rs @@ -41,6 +41,7 @@ async fn dispatch_action( params, cross_entity_booleans: BTreeMap::new(), idempotency_key: None, + expected_authorization_precondition: None, }, Duration::from_secs(5), ) diff --git a/crates/temper-server/tests/e2e_gepa_loop.rs b/crates/temper-server/tests/e2e_gepa_loop.rs index d7f97ed28..f0228a498 100644 --- a/crates/temper-server/tests/e2e_gepa_loop.rs +++ b/crates/temper-server/tests/e2e_gepa_loop.rs @@ -300,6 +300,7 @@ async fn e2e_gepa_sentinel_detects_failure_cluster() { request_body: None, intent: Some("reassign issue to different agent".to_string()), matched_policy_ids: None, + capture_seq: None, }) .collect(); @@ -1048,6 +1049,7 @@ async fn e2e_gepa_full_loop() { request_body: None, intent: None, matched_policy_ids: None, + capture_seq: None, }) .collect(); diff --git a/crates/temper-server/tests/ensure_entity_loaded.rs b/crates/temper-server/tests/ensure_entity_loaded.rs index c3c5cd7c4..a6d37515b 100644 --- a/crates/temper-server/tests/ensure_entity_loaded.rs +++ b/crates/temper-server/tests/ensure_entity_loaded.rs @@ -284,21 +284,13 @@ async fn delete_failure_does_not_remove_live_entity_from_index() { .await .expect("append external race event"); - let response = state + let error = state .delete_tenant_entity(&tenant, entity_type, entity_id) .await - .expect("delete returns response"); + .expect_err("delete should fail when tombstone append hits sequence race"); assert!( - !response.success, - "delete should fail when tombstone append hits sequence race" - ); - assert!( - response - .error - .as_deref() - .is_some_and(|e| e.contains("persistence failed")), - "expected persistence failure error, got: {:?}", - response.error + error.contains("persistence failed"), + "expected persistence failure error, got: {error}" ); assert!( diff --git a/crates/temper-server/tests/field_invariants.rs b/crates/temper-server/tests/field_invariants.rs index a5f764718..24a7feba6 100644 --- a/crates/temper-server/tests/field_invariants.rs +++ b/crates/temper-server/tests/field_invariants.rs @@ -49,6 +49,20 @@ require = { field = "Notes", absent = true } message = "USD orders cannot carry notes" "#; +fn authenticate(mut request: Request) -> Request { + request + .extensions_mut() + .insert(temper_authz::AuthenticatedRequestContext::new( + TenantId::default(), + temper_authz::SecurityContext::from_resolved_identity( + "field-invariant-test", + "test-agent", + None, + ), + )); + request +} + fn build_state_with_field_invariant() -> ServerState { let csdl = parse_csdl(CSDL_XML).expect("CSDL parse"); let mut registry = SpecRegistry::new(); @@ -81,16 +95,22 @@ fn build_state_with_field_invariant() -> ServerState { ); } state + .authz + .reload_tenant_policies("default", "permit(principal, action, resource);") + .expect("field-invariant fixture policy should parse"); + state } /// Send a POST `/tdata/Orders` with the given JSON body and return the /// status + parsed body. async fn post_order(state: &ServerState, body: &str) -> (StatusCode, serde_json::Value) { let router = build_router(state.clone()); - let req = Request::post("/tdata/Orders") - .header("Content-Type", "application/json") - .body(Body::from(body.to_string())) - .unwrap(); + let req = authenticate( + Request::post("/tdata/Orders") + .header("Content-Type", "application/json") + .body(Body::from(body.to_string())) + .unwrap(), + ); let resp = router.oneshot(req).await.unwrap(); let status = resp.status(); let bytes = axum::body::to_bytes(resp.into_body(), 1_000_000) @@ -103,12 +123,14 @@ async fn post_order(state: &ServerState, body: &str) -> (StatusCode, serde_json: /// Send a PATCH `/tdata/Orders('')` with the given JSON body. async fn patch_order(state: &ServerState, id: &str, body: &str) -> (StatusCode, serde_json::Value) { let router = build_router(state.clone()); - let req = Request::builder() - .method(axum::http::Method::PATCH) - .uri(format!("/tdata/Orders('{id}')")) - .header("Content-Type", "application/json") - .body(Body::from(body.to_string())) - .unwrap(); + let req = authenticate( + Request::builder() + .method(axum::http::Method::PATCH) + .uri(format!("/tdata/Orders('{id}')")) + .header("Content-Type", "application/json") + .body(Body::from(body.to_string())) + .unwrap(), + ); let resp = router.oneshot(req).await.unwrap(); let status = resp.status(); let bytes = axum::body::to_bytes(resp.into_body(), 1_000_000) diff --git a/crates/temper-server/tests/file_value_fast_path.rs b/crates/temper-server/tests/file_value_fast_path.rs index 165b29ed6..7e35c8d12 100644 --- a/crates/temper-server/tests/file_value_fast_path.rs +++ b/crates/temper-server/tests/file_value_fast_path.rs @@ -14,6 +14,27 @@ use temper_spec::csdl::parse_csdl; use temper_store_turso::TursoEventStore; use tower::ServiceExt; +async fn authenticate_test_request( + mut request: axum::extract::Request, + next: axum::middleware::Next, +) -> axum::response::Response { + request + .extensions_mut() + .insert(temper_authz::AuthenticatedRequestContext::new( + TenantId::default(), + temper_authz::SecurityContext::from_resolved_identity( + "file-value-test", + "test-agent", + None, + ), + )); + next.run(request).await +} + +fn authenticated_router(state: ServerState) -> axum::Router { + build_router(state).layer(axum::middleware::from_fn(authenticate_test_request)) +} + const FILE_CSDL_XML: &str = r#" @@ -215,6 +236,10 @@ async fn build_turso_file_state(test_name: &str) -> (ServerState, TursoEventStor ); let mut state = ServerState::from_registry(ActorSystem::new(test_name), registry); state.set_storage_stack(StorageStack::from_turso(store.clone())); + state + .authz + .reload_tenant_policies("default", "permit(principal, action, resource);") + .expect("functional file-value tests should install an explicit policy"); (state, store) } @@ -349,7 +374,7 @@ async fn odata_file_value_put_uses_native_path_without_blob_adapter() { .await .expect("create File state"); - let app = build_router(state.clone()); + let app = authenticated_router(state.clone()); let body = b"odata native File value write"; let expected_hash = format!("sha256:{:x}", Sha256::digest(body)); let response = app @@ -399,7 +424,7 @@ async fn odata_file_value_put_applies_cedar_update_policy() { ) .expect("install Cedar policy"); - let response = build_router(state.clone()) + let response = authenticated_router(state.clone()) .oneshot( Request::put("/tdata/Files('fl-write-denied')/$value") .header("content-type", "text/plain") @@ -448,7 +473,7 @@ async fn odata_file_value_get_applies_cedar_read_policy() { ) .expect("install Cedar policy"); - let response = build_router(state) + let response = authenticated_router(state) .oneshot( Request::get("/tdata/Files('fl-read-denied')/$value") .header("x-temper-principal-kind", "customer") @@ -653,7 +678,7 @@ async fn put_value_on_new_file_is_one_atomic_append() { let body = b"brand new file value, one append"; - let response = build_router(state.clone()) + let response = authenticated_router(state.clone()) .oneshot( Request::put("/tdata/Files('fl-new-atomic')/$value") .header("content-type", "text/plain") @@ -718,7 +743,7 @@ async fn new_file_value_put_is_read_after_write_consistent() { let body = b"read after write consistency"; let expected_hash = format!("sha256:{:x}", Sha256::digest(body)); - let response = build_router(state.clone()) + let response = authenticated_router(state.clone()) .oneshot( Request::put("/tdata/Files('fl-raw-consistent')/$value") .header("content-type", "text/markdown") @@ -768,7 +793,7 @@ async fn concurrent_new_file_value_puts_yield_one_204_one_409() { let body_a = b"writer A bytes"; let body_b = b"writer B different bytes"; - let app = build_router(state.clone()); + let app = authenticated_router(state.clone()); let put_a = app.clone().oneshot( Request::put("/tdata/Files('fl-race')/$value") .header("content-type", "text/plain") @@ -824,7 +849,7 @@ async fn existing_file_value_put_routes_through_update_unchanged() { // First write: brand-new File via the atomic create-with-content path. let first = b"version one bytes"; let first_hash = format!("sha256:{:x}", Sha256::digest(first)); - let response = build_router(state.clone()) + let response = authenticated_router(state.clone()) .oneshot( Request::put("/tdata/Files('fl-update')/$value") .header("content-type", "text/plain") @@ -852,7 +877,7 @@ async fn existing_file_value_put_routes_through_update_unchanged() { let second = b"version two bytes, larger payload"; let second_hash = format!("sha256:{:x}", Sha256::digest(second)); assert_ne!(first_hash, second_hash); - let response = build_router(state.clone()) + let response = authenticated_router(state.clone()) .oneshot( Request::put("/tdata/Files('fl-update')/$value") .header("content-type", "text/plain") diff --git a/crates/temper-server/tests/gepa_manual_verification.rs b/crates/temper-server/tests/gepa_manual_verification.rs index 426c1f038..b4ddc8c9f 100644 --- a/crates/temper-server/tests/gepa_manual_verification.rs +++ b/crates/temper-server/tests/gepa_manual_verification.rs @@ -469,6 +469,7 @@ async fn manual_gepa_verification() { request_body: None, intent: None, matched_policy_ids: None, + capture_seq: None, }) .collect(); @@ -514,6 +515,7 @@ async fn manual_gepa_verification() { request_body: None, intent: None, matched_policy_ids: None, + capture_seq: None, }) .collect(); let few_alerts = diff --git a/crates/temper-server/tests/nearest_odata.rs b/crates/temper-server/tests/nearest_odata.rs index 240569bd6..8cfdc6152 100644 --- a/crates/temper-server/tests/nearest_odata.rs +++ b/crates/temper-server/tests/nearest_odata.rs @@ -80,6 +80,14 @@ const CSDL_XML: &str = r#" "#; +const VEC_ITEM_READ_TEST_POLICY: &str = r#" + permit( + principal, + action in [Action::"list", Action::"read"], + resource is VecItem + ); +"#; + fn build_state() -> ServerState { let mut registry = SpecRegistry::new(); let csdl = parse_csdl(CSDL_XML).expect("CSDL parse"); @@ -93,6 +101,10 @@ fn build_state() -> ServerState { let mut state = ServerState::from_registry(system, registry); state.set_storage_stack(StorageStack::from_sim(SimEventStore::no_faults(7), None)); state + .authz + .reload_tenant_policies(TenantId::default().as_str(), VEC_ITEM_READ_TEST_POLICY) + .expect("install VecItem read/list test policy"); + state } async fn create_item( @@ -145,7 +157,16 @@ async fn delete_item(state: &ServerState, tenant: &TenantId, id: &str) { async fn get_json(state: &ServerState, path: &str) -> (StatusCode, serde_json::Value) { let router = build_router(state.clone()); - let req = Request::builder().uri(path).body(Body::empty()).unwrap(); + let mut req = Request::builder().uri(path).body(Body::empty()).unwrap(); + req.extensions_mut() + .insert(temper_authz::AuthenticatedRequestContext::new( + TenantId::default(), + temper_authz::SecurityContext::from_resolved_identity( + "nearest-odata-test", + "test-customer", + None, + ), + )); let resp = router.oneshot(req).await.unwrap(); let status = resp.status(); let bytes = axum::body::to_bytes(resp.into_body(), 1_000_000) diff --git a/crates/temper-server/tests/odata_read.rs b/crates/temper-server/tests/odata_read.rs index ff62320a2..769a4aee6 100644 --- a/crates/temper-server/tests/odata_read.rs +++ b/crates/temper-server/tests/odata_read.rs @@ -21,6 +21,48 @@ use tower::ServiceExt; const CSDL_XML: &str = common::CSDL_XML; const ORDER_IOA: &str = common::ORDER_IOA; +const ORDER_CRUD_TEST_POLICY: &str = r#" + permit( + principal, + action in [ + Action::"list", + Action::"read", + Action::"create", + Action::"update", + Action::"delete" + ], + resource is Order + ); +"#; + +fn install_order_crud_test_policy(state: &ServerState) { + state + .authz + .reload_tenant_policies(TenantId::default().as_str(), ORDER_CRUD_TEST_POLICY) + .expect("install Order CRUD test policy"); +} + +fn authenticate(mut request: Request, principal_id: &str) -> Request { + let security_context = temper_authz::SecurityContext { + principal: temper_authz::Principal { + id: principal_id.to_string(), + kind: temper_authz::PrincipalKind::Customer, + role: None, + acting_for: None, + agent_type: None, + attributes: Default::default(), + }, + context_attrs: Default::default(), + correlation_id: "odata-read-test".to_string(), + }; + request + .extensions_mut() + .insert(temper_authz::AuthenticatedRequestContext::new( + TenantId::default(), + security_context, + )); + request +} /// Send a GET request to the router and return status + parsed JSON body. async fn get_json( @@ -28,7 +70,10 @@ async fn get_json( path: &str, ) -> (StatusCode, serde_json::Value) { let router = build_router(state.clone()); - let req = Request::builder().uri(path).body(Body::empty()).unwrap(); + let req = authenticate( + Request::builder().uri(path).body(Body::empty()).unwrap(), + "test-customer", + ); let resp = router.oneshot(req).await.unwrap(); let status = resp.status(); let bytes = axum::body::to_bytes(resp.into_body(), 1_000_000) @@ -38,16 +83,51 @@ async fn get_json( (status, body) } +fn encoded_wide_filter(comparisons: usize) -> String { + (0..comparisons) + .map(|n| format!("Id eq {n}")) + .collect::>() + .join(" or ") + .replace(' ', "%20") +} + +#[tokio::test] +async fn over_budget_filter_is_rejected_through_tdata_router() { + let (state, _) = build_default_state(176, "odata-filter-budget"); + install_order_crud_test_policy(&state); + let filter = encoded_wide_filter(513); + + let (status, body) = get_json(&state, &format!("/tdata/Orders?$filter={filter}")).await; + + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!(body["error"]["code"].as_str(), Some("InvalidQuery")); +} + +#[tokio::test] +async fn budget_boundary_filter_is_accepted_through_tdata_router() { + let (state, _) = build_default_state(176, "odata-filter-budget-boundary"); + install_order_crud_test_policy(&state); + let filter = encoded_wide_filter(512); + + let (status, body) = get_json(&state, &format!("/tdata/Orders?$filter={filter}")).await; + + assert_eq!(status, StatusCode::OK); + assert_eq!(body["value"].as_array().map(Vec::len), Some(0)); +} + async fn post_json( state: &ServerState, path: &str, body: serde_json::Value, ) -> (StatusCode, serde_json::Value) { let router = build_router(state.clone()); - let req = Request::post(path) - .header("Content-Type", "application/json") - .body(Body::from(body.to_string())) - .unwrap(); + let req = authenticate( + Request::post(path) + .header("Content-Type", "application/json") + .body(Body::from(body.to_string())) + .unwrap(), + "test-customer", + ); let resp = router.oneshot(req).await.unwrap(); let status = resp.status(); let bytes = axum::body::to_bytes(resp.into_body(), 1_000_000) @@ -93,12 +173,15 @@ async fn patch_json( body: serde_json::Value, ) -> (StatusCode, serde_json::Value) { let router = build_router(state.clone()); - let req = Request::builder() - .method(axum::http::Method::PATCH) - .uri(path) - .header("Content-Type", "application/json") - .body(Body::from(body.to_string())) - .unwrap(); + let req = authenticate( + Request::builder() + .method(axum::http::Method::PATCH) + .uri(path) + .header("Content-Type", "application/json") + .body(Body::from(body.to_string())) + .unwrap(), + "test-customer", + ); let resp = router.oneshot(req).await.unwrap(); let status = resp.status(); let bytes = axum::body::to_bytes(resp.into_body(), 1_000_000) @@ -115,17 +198,16 @@ async fn customer_json( body: Option, ) -> (StatusCode, serde_json::Value) { let router = build_router(state.clone()); - let mut request = Request::builder() - .method(method) - .uri(path) - .header("x-temper-principal-kind", "customer") - .header("x-temper-principal-id", "customer-1"); + let mut request = Request::builder().method(method).uri(path); if body.is_some() { request = request.header("Content-Type", "application/json"); } - let req = request - .body(body.map_or_else(Body::empty, |value| Body::from(value.to_string()))) - .unwrap(); + let req = authenticate( + request + .body(body.map_or_else(Body::empty, |value| Body::from(value.to_string()))) + .unwrap(), + "customer-1", + ); let resp = router.oneshot(req).await.unwrap(); let status = resp.status(); let bytes = axum::body::to_bytes(resp.into_body(), 1_000_000) @@ -166,12 +248,14 @@ fn build_turso_state(system_name: &str, store: TursoEventStore) -> ServerState { let mut state = state; state.set_storage_stack(StorageStack::from_turso(store)); + install_order_crud_test_policy(&state); state } #[tokio::test] async fn entity_set_returns_created_entities() { let (state, _sim) = build_default_state(42, "odata-read-set"); + install_order_crud_test_policy(&state); let tenant = TenantId::default(); dispatch( @@ -205,6 +289,7 @@ async fn entity_set_returns_created_entities() { #[tokio::test] async fn entity_get_returns_single_entity_with_actions() { let (state, _sim) = build_default_state(43, "odata-read-entity"); + install_order_crud_test_policy(&state); let tenant = TenantId::default(); dispatch( @@ -652,3 +737,292 @@ async fn crud_routes_apply_cedar_mutation_policies() { let _ = std::fs::remove_file(db_path); } + +#[tokio::test] +async fn patch_and_put_authorize_the_prospective_resource() { + let db_path = std::env::temp_dir().join(format!( + "temper-odata-read-cedar-prospective-{}.db", + uuid::Uuid::new_v4() + )); + let db_url = format!("file:{}", db_path.display()); + let store = TursoEventStore::new(&db_url, None) + .await + .expect("create local turso db"); + let state = build_turso_state("odata-read-cedar-prospective", store); + + let (status, body) = post_json( + &state, + "/tdata/Orders", + serde_json::json!({"id": "ord-prospective", "Currency": "EUR"}), + ) + .await; + assert_eq!(status, StatusCode::CREATED, "seed create failed: {body:?}"); + + state + .authz + .reload_tenant_policies( + TenantId::default().as_str(), + r#" + permit(principal, action in [Action::"read", Action::"list"], resource is Order); + permit(principal, action == Action::"update", resource is Order) + when { resource.Currency == "EUR" }; + "#, + ) + .expect("install Cedar policy"); + + for (method, currency) in [(Method::PATCH, "USD"), (Method::PUT, "GBP")] { + let (status, body) = customer_json( + &state, + method, + "/tdata/Orders('ord-prospective')", + Some(serde_json::json!({"Currency": currency})), + ) + .await; + assert_eq!(status, StatusCode::FORBIDDEN, "unexpected body: {body:?}"); + assert_eq!(body["error"]["code"], "AuthorizationDenied"); + } + + let (status, body) = customer_json( + &state, + Method::GET, + "/tdata/Orders('ord-prospective')", + None, + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["fields"]["Currency"], "EUR"); + + let _ = std::fs::remove_file(db_path); +} + +/// ARN-189. The actor is fail-closed on the journal append, but +/// `update_tenant_entity_fields` returned `Ok(response)` regardless of +/// `response.success`, and the handler only maps `Err` to a non-2xx — so a +/// rejected or unpersisted update was answered `200 OK`. Fail-closed actor, +/// fail-open API, which defeats the point of journaling the update at all. +/// +/// Driven through the real HTTP router rather than the actor, because the defect +/// lived in the seam between them: every actor-level test passed while the client +/// was still told the write had succeeded. +#[tokio::test] +async fn patch_returns_an_error_status_when_the_update_is_refused() { + let (state, sim) = build_default_state(4711, "odata-patch-fail-closed"); + install_order_crud_test_policy(&state); + // HTTP writes are gated on verification status; the sim harness leaves it + // pending, so mark it Completed exactly as `build_turso_state` does. + { + let mut registry = state.registry.write().unwrap(); + registry.set_verification_status( + &TenantId::default(), + "Order", + VerificationStatus::Completed(EntityVerificationResult { + all_passed: true, + levels: vec![EntityLevelSummary { + level: "L0 SMT".to_string(), + passed: true, + summary: "OK".to_string(), + details: None, + }], + verified_at: "2026-04-15T00:00:00Z".to_string(), + }), + ); + } + let tenant = TenantId::default(); + + dispatch( + &state, + &tenant, + "Order", + "ord-fail-closed", + "Create", + serde_json::json!({}), + ) + .await + .expect("create ord-fail-closed"); + + // A PATCH the actor cannot journal. The entity exists and the request is + // well-formed, so it reaches the actor and is refused there. The OData + // handler maps a refused response to an error status itself, so this does not + // exercise `update_tenant_entity_fields`' own propagation — see + // `field_update_failure_is_reported_to_callers_that_do_not_inspect_success` + // for that. What it pins is the end-to-end property: an update that was not + // persisted is never answered 2xx. + sim.inject_concurrency_violations("default:Order:ord-fail-closed", 8); + + let (status, body) = patch_json( + &state, + "/tdata/Orders('ord-fail-closed')", + serde_json::json!({ "Currency": "USD" }), + ) + .await; + assert!( + !status.is_success(), + "an update the actor could not persist must not answer 2xx; got {status} {body:?}" + ); + + // And the entity is unchanged — the refusal was not a partial write. + sim.inject_concurrency_violations("default:Order:ord-fail-closed", 0); + let (status, body) = get_json(&state, "/tdata/Orders('ord-fail-closed')").await; + assert_eq!(status, StatusCode::OK); + assert_ne!( + body["fields"]["Currency"].as_str(), + Some("USD"), + "a refused update must leave the entity untouched: {body:?}" + ); +} + +/// ARN-189 regression guard. `update_tenant_entity_fields` converts a refused +/// actor response into `Err`, which is what callers that do not inspect +/// `response.success` rely on — `os_apps::entity_aliases` calls this and does +/// `.map(|_| ())`, so if `Ok` ever stopped meaning "the update happened", a +/// refused alias repair would be silently treated as done. Journaling field +/// updates adds a new way for them to fail, so this pins that a failed append +/// travels the same path. +#[tokio::test] +async fn field_update_failure_is_reported_to_callers_that_do_not_inspect_success() { + let (state, sim) = build_default_state(4713, "entity-ops-fail-closed"); + let tenant = TenantId::default(); + + dispatch( + &state, + &tenant, + "Order", + "ord-ops-fail-closed", + "Create", + serde_json::json!({}), + ) + .await + .expect("create ord-ops-fail-closed"); + + // Enough conflicts to outlast the retry budget, so the append genuinely fails. + sim.inject_concurrency_violations("default:Order:ord-ops-fail-closed", 8); + + let result = state + .update_tenant_entity_fields( + &tenant, + "Order", + "ord-ops-fail-closed", + serde_json::json!({ "Currency": "USD" }), + false, + ) + .await; + + assert!( + result.is_err(), + "an update that was not persisted must not be returned as Ok: {:?}", + result.map(|r| (r.success, r.error)) + ); +} + +/// ARN-189 live end-to-end: the exact scenario the bug loses data in, driven +/// through the real OData HTTP router over a real Turso file, across a full +/// `ServerState` teardown and rebuild on the same database — a process restart in +/// all but name (fresh actor system, fresh registry, fresh in-memory state; only +/// the durable journal survives). +/// +/// Before the fix, a PATCH mutated actor memory and journaled nothing, so the +/// second `ServerState` — rehydrating purely from the journal — would not see it. +#[tokio::test] +async fn patched_fields_survive_a_server_restart_over_the_http_stack() { + let db_path = + std::env::temp_dir().join(format!("temper-arn189-live-{}.db", uuid::Uuid::new_v4())); + let db_url = format!("file:{}", db_path.display()); + + // --- Run 1: create, then PATCH a field over HTTP. --- + { + let store = TursoEventStore::new(&db_url, None) + .await + .expect("create turso db"); + let state = build_turso_state("arn189-live-run1", store); + + let (status, body) = post_json( + &state, + "/tdata/Orders", + serde_json::json!({ "id": "ord-arn189-live", "Currency": "EUR" }), + ) + .await; + assert_eq!(status, StatusCode::CREATED, "create failed: {body:?}"); + + let (status, body) = patch_json( + &state, + "/tdata/Orders('ord-arn189-live')", + serde_json::json!({ "Currency": "USD", "Notes": "patched before restart" }), + ) + .await; + assert_eq!(status, StatusCode::OK, "patch failed: {body:?}"); + + // Confirm run 1 sees the patch in memory. + let (status, body) = get_json(&state, "/tdata/Orders('ord-arn189-live')").await; + assert_eq!(status, StatusCode::OK); + assert_eq!(body["fields"]["Currency"].as_str(), Some("USD")); + assert_eq!( + body["fields"]["Notes"].as_str(), + Some("patched before restart") + ); + } + + // --- The journal itself must carry the update. --- + // + // The read above is served from the durable query projection, which is a + // read-model *cache* written alongside the update. The source of truth is the + // event journal, and it is what a projection backfill (ARN-216) rebuilds from: + // before this fix the backfill dropped patched fields the live projection + // still had. So the discriminating assertion is that the append happened — + // read the raw journal back and find the field-update event with the new + // value. This is what fails if `commit_field_update` stops journaling. + { + use temper_runtime::persistence::EventStore as _; + let store = TursoEventStore::new(&db_url, None) + .await + .expect("reopen turso db for journal read"); + let envelopes = store + .read_events("default:Order:ord-arn189-live", 0) + .await + .expect("read journal"); + let patched = envelopes.iter().any(|env| { + let is_field_event = + env.event_type == "FieldsUpdated" || env.event_type == "FieldsReplaced"; + is_field_event + && env + .payload + .get("params") + .and_then(|p| p.get("Currency")) + .and_then(|c| c.as_str()) + == Some("USD") + }); + assert!( + patched, + "the PATCH must be durably journaled, not only cached in the projection; \ + journal held {} events: {:?}", + envelopes.len(), + envelopes.iter().map(|e| &e.event_type).collect::>() + ); + } + + // --- Run 2: a brand-new ServerState on the same journal reads it back. --- + { + let store = TursoEventStore::new(&db_url, None) + .await + .expect("reopen turso db"); + let state = build_turso_state("arn189-live-run2", store); + + let (status, body) = get_json(&state, "/tdata/Orders('ord-arn189-live')").await; + assert_eq!( + status, + StatusCode::OK, + "entity missing after restart: {body:?}" + ); + assert_eq!( + body["fields"]["Currency"].as_str(), + Some("USD"), + "the PATCHed field must survive a restart: {body:?}" + ); + assert_eq!( + body["fields"]["Notes"].as_str(), + Some("patched before restart"), + "the PATCHed field must survive a restart: {body:?}" + ); + } + + let _ = std::fs::remove_file(db_path); +} diff --git a/crates/temper-server/tests/policy_authorization.rs b/crates/temper-server/tests/policy_authorization.rs new file mode 100644 index 000000000..5a4d01eca --- /dev/null +++ b/crates/temper-server/tests/policy_authorization.rs @@ -0,0 +1,153 @@ +#![cfg(feature = "observe")] + +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use temper_authz::{AuthenticatedRequestContext, Principal, PrincipalKind, SecurityContext}; +use temper_runtime::ActorSystem; +use temper_runtime::tenant::TenantId; +use temper_server::registry::SpecRegistry; +use temper_server::storage::StorageStack; +use temper_server::{ServerState, build_router}; +use temper_store_turso::TursoEventStore; +use tower::ServiceExt; + +const TENANT: &str = "default"; +const OPERATOR_POLICY: &str = r#" +permit( + principal is Agent, + action == Action::"manage_policies", + resource == PolicySet::"default" +) when { + principal.agent_type == "operator" && + principal.agentTypeVerified == true +}; +"#; + +async fn policy_state() -> (ServerState, TursoEventStore, tempfile::TempDir) { + let temp = tempfile::tempdir().expect("temporary policy database"); + let db_url = format!("file:{}", temp.path().join("policy.db").display()); + let store = TursoEventStore::new(&db_url, None) + .await + .expect("create Turso policy store"); + let mut state = + ServerState::from_registry(ActorSystem::new("policy-auth"), SpecRegistry::new()); + state.set_storage_stack(StorageStack::from_turso(store.clone())); + state + .authz + .reload_tenant_policies(TENANT, OPERATOR_POLICY) + .expect("install operator policy"); + (state, store, temp) +} + +fn request_with_context( + method: &str, + uri: &str, + body: Body, + tenant: &str, + security_context: SecurityContext, +) -> Request { + let mut request = Request::builder() + .method(method) + .uri(uri) + .header("content-type", "application/json") + .body(body) + .expect("policy request should build"); + request + .extensions_mut() + .insert(AuthenticatedRequestContext::new( + TenantId::new(tenant), + security_context, + )); + request +} + +fn admin_context() -> SecurityContext { + SecurityContext { + principal: Principal { + id: "claimed-admin".to_string(), + kind: PrincipalKind::Admin, + role: None, + acting_for: None, + agent_type: None, + attributes: Default::default(), + }, + context_attrs: Default::default(), + correlation_id: "policy-admin-test".to_string(), + } +} + +#[tokio::test] +async fn policy_management_requires_tenant_bound_cedar_authority() { + let (state, store, _temp) = policy_state().await; + let app = build_router(state); + let uri = "/api/tenants/default/policies/create"; + + let forged_header = app + .clone() + .oneshot( + Request::post(uri) + .header("content-type", "application/json") + .header("x-temper-principal-kind", "admin") + .body(Body::from("{}")) + .expect("forged-header request should build"), + ) + .await + .expect("forged-header request should run"); + assert_eq!(forged_header.status(), StatusCode::UNAUTHORIZED); + + let typed_admin = app + .clone() + .oneshot(request_with_context( + "POST", + uri, + Body::from("{}"), + TENANT, + admin_context(), + )) + .await + .expect("typed-admin request should run"); + assert_eq!(typed_admin.status(), StatusCode::FORBIDDEN); + + let wrong_tenant = app + .clone() + .oneshot(request_with_context( + "POST", + uri, + Body::from("{}"), + "other-tenant", + SecurityContext::from_resolved_identity("operator", "operator", None), + )) + .await + .expect("wrong-tenant request should run"); + assert_eq!(wrong_tenant.status(), StatusCode::UNAUTHORIZED); + + let allowed = app + .oneshot(request_with_context( + "POST", + uri, + Body::from( + serde_json::json!({ + "policy_id": "operator-baseline", + "cedar_text": OPERATOR_POLICY, + "created_by": "forged-actor" + }) + .to_string(), + ), + TENANT, + SecurityContext::from_resolved_identity("operator", "operator", None), + )) + .await + .expect("verified-operator request should run"); + assert_eq!(allowed.status(), StatusCode::CREATED); + + let rows = store + .load_policies_for_tenant(TENANT) + .await + .expect("load created policy"); + let row = rows + .iter() + .find(|row| row.policy_id == "operator-baseline") + .expect("created policy should be durable"); + assert_eq!(row.created_by, "operator"); + assert_ne!(row.created_by, "forged-actor"); +} diff --git a/crates/temper-server/tests/reaction_e2e_prod.rs b/crates/temper-server/tests/reaction_e2e_prod.rs index 2c59523e6..5c39e1bc3 100644 --- a/crates/temper-server/tests/reaction_e2e_prod.rs +++ b/crates/temper-server/tests/reaction_e2e_prod.rs @@ -17,6 +17,8 @@ use temper_server::request_context::AgentContext; use temper_server::trigger::registry::parse_reactions; use temper_spec::csdl::parse_csdl; +const REACTION_E2E_SERVICE: &str = "reaction-e2e-prod-test"; + const CSDL_XML: &str = r#" @@ -119,6 +121,31 @@ fn build_state(tenant: &str, reactions_toml: &str) -> ServerState { let system = ActorSystem::new("reaction-e2e-prod"); let state = ServerState::from_registry(system, registry); + state + .authz + .reload_tenant_policies( + tenant, + r#" + permit( + principal == Agent::"service:reaction-e2e-prod-test", + action in [ + Action::"AddItem", + Action::"SubmitOrder", + Action::"ConfirmOrder" + ], + resource is Order + ); + permit( + principal == Agent::"service:reaction-e2e-prod-test", + action in [ + Action::"AuthorizePayment", + Action::"FailPayment" + ], + resource is Payment + ); + "#, + ) + .expect("reaction E2E policy should load"); state.rebuild_reaction_dispatcher(); state } @@ -138,7 +165,7 @@ async fn dispatch( entity_id, action, params, - &AgentContext::default(), + &AgentContext::for_service(REACTION_E2E_SERVICE), ) .await .unwrap_or_else(|e| panic!("dispatch {entity_type}.{action} failed: {e}")) diff --git a/crates/temper-server/tests/resource_authorization.rs b/crates/temper-server/tests/resource_authorization.rs new file mode 100644 index 000000000..e49c51f4e --- /dev/null +++ b/crates/temper-server/tests/resource_authorization.rs @@ -0,0 +1,328 @@ +#![cfg(feature = "observe")] + +use axum::body::Body; +use axum::http::{Method, Request, StatusCode}; +use temper_authz::{AuthenticatedRequestContext, Principal, PrincipalKind, SecurityContext}; +use temper_runtime::ActorSystem; +use temper_runtime::tenant::TenantId; +use temper_server::registry::SpecRegistry; +use temper_server::secrets::vault::SecretsVault; +use temper_server::storage::StorageStack; +use temper_server::{ServerState, build_router}; +use temper_store_turso::TursoEventStore; +use tower::ServiceExt; +use wiremock::matchers::method; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +#[path = "resource_authorization/secret_authorization.rs"] +mod secret_authorization; + +const TENANT: &str = "default"; + +async fn state_with_turso(name: &str) -> (ServerState, TursoEventStore, tempfile::TempDir) { + let temp = tempfile::tempdir().expect("temporary test directory"); + let db_url = format!("file:{}", temp.path().join("metadata.db").display()); + let store = TursoEventStore::new(&db_url, None) + .await + .expect("create Turso test store"); + let mut state = ServerState::from_registry(ActorSystem::new(name), SpecRegistry::new()); + state.data_dir = temp.path().join("data"); + state.set_storage_stack(StorageStack::from_turso(store.clone())); + (state, store, temp) +} + +fn customer_security_context(id: &str) -> SecurityContext { + SecurityContext { + principal: Principal { + id: id.to_string(), + kind: PrincipalKind::Customer, + role: None, + acting_for: None, + agent_type: None, + attributes: Default::default(), + }, + context_attrs: Default::default(), + correlation_id: format!("resource-auth-{id}"), + } +} + +fn json_request( + method: Method, + uri: &str, + body: serde_json::Value, + tenant: &str, + principal_id: &str, +) -> Request { + let mut request = Request::builder() + .method(method) + .uri(uri) + .header("content-type", "application/json") + .body(Body::from(body.to_string())) + .expect("request should build"); + request + .extensions_mut() + .insert(AuthenticatedRequestContext::new( + TenantId::new(tenant), + customer_security_context(principal_id), + )); + request +} + +async fn seed_file( + store: &TursoEventStore, + entity_type: &str, + id: &str, + hash: &str, + body: &[u8], + source_file_id: Option<&str>, +) { + let mut fields = serde_json::json!({ + "content_hash": hash, + "mime_type": "text/markdown", + "has_content": true, + }); + if let Some(source_file_id) = source_file_id { + fields["file_id"] = serde_json::Value::String(source_file_id.to_string()); + } + store + .upsert_query_projection(TENANT, entity_type, id, "Ready", &fields, 1) + .await + .expect("seed file projection"); + store + .put_blob(&format!("temper-fs/{hash}"), body) + .await + .expect("seed file blob"); +} + +#[tokio::test] +async fn batch_file_reads_authorize_every_exact_resource() { + let (state, store, _temp) = state_with_turso("file-resource-auth").await; + state + .authz + .reload_tenant_policies( + TENANT, + r#" +permit( + principal == Customer::"reader", + action == Action::"read", + resource == File::"file-a" +); +permit( + principal == Customer::"reader", + action == Action::"read", + resource == File::"file-large" +); +permit( + principal == Customer::"reader", + action == Action::"read", + resource == FileVersion::"version-a" +); +"#, + ) + .expect("file policy should parse"); + state + .authz + .reload_tenant_policies("other-tenant", "") + .expect("other tenant should default-deny"); + seed_file(&store, "File", "file-a", "sha256:filea", b"file-a", None).await; + let oversized_text = vec![b'x'; 2 * 1024 * 1024 + 1]; + seed_file( + &store, + "File", + "file-large", + "sha256:filelarge", + &oversized_text, + None, + ) + .await; + seed_file( + &store, + "FileVersion", + "version-a", + "sha256:versiona", + b"version-a", + Some("file-a"), + ) + .await; + let app = build_router(state); + + let allowed = app + .clone() + .oneshot(json_request( + Method::POST, + "/api/files/read-text-batch", + serde_json::json!({"file_ids": ["file-a"]}), + TENANT, + "reader", + )) + .await + .expect("allowed request should run"); + assert_eq!(allowed.status(), StatusCode::OK); + + let duplicate = app + .clone() + .oneshot(json_request( + Method::POST, + "/api/files/read-text-batch", + serde_json::json!({"file_ids": ["file-a", "file-a"]}), + TENANT, + "reader", + )) + .await + .expect("duplicate request should be rejected"); + assert_eq!(duplicate.status(), StatusCode::BAD_REQUEST); + + let too_many_ids = (0..101) + .map(|index| format!("file-{index}")) + .collect::>(); + let oversized_batch = app + .clone() + .oneshot(json_request( + Method::POST, + "/api/files/read-text-batch", + serde_json::json!({"file_ids": too_many_ids}), + TENANT, + "reader", + )) + .await + .expect("oversized batch should be rejected"); + assert_eq!(oversized_batch.status(), StatusCode::PAYLOAD_TOO_LARGE); + + let oversized_item = app + .clone() + .oneshot(json_request( + Method::POST, + "/api/files/read-text-batch", + serde_json::json!({"file_ids": ["file-large"]}), + TENANT, + "reader", + )) + .await + .expect("oversized buffered item should be rejected"); + assert_eq!(oversized_item.status(), StatusCode::PAYLOAD_TOO_LARGE); + + let partial_batch = app + .clone() + .oneshot(json_request( + Method::POST, + "/api/files/read-text-batch", + serde_json::json!({"file_ids": ["file-a", "file-b"]}), + TENANT, + "reader", + )) + .await + .expect("partially unauthorized request should run"); + assert_eq!(partial_batch.status(), StatusCode::FORBIDDEN); + + let wrong_principal = app + .clone() + .oneshot(json_request( + Method::POST, + "/api/files/read-text-batch", + serde_json::json!({"file_ids": ["file-a"]}), + TENANT, + "intruder", + )) + .await + .expect("wrong-principal request should run"); + assert_eq!(wrong_principal.status(), StatusCode::FORBIDDEN); + + let wrong_tenant = app + .clone() + .oneshot(json_request( + Method::POST, + "/api/files/read-text-batch", + serde_json::json!({"file_ids": ["file-a"]}), + "other-tenant", + "reader", + )) + .await + .expect("wrong-tenant request should run"); + assert_eq!(wrong_tenant.status(), StatusCode::FORBIDDEN); + + let version_allowed = app + .oneshot(json_request( + Method::POST, + "/api/files/read-version-text-batch", + serde_json::json!({"file_version_ids": ["version-a"]}), + TENANT, + "reader", + )) + .await + .expect("version request should run"); + assert_eq!(version_allowed.status(), StatusCode::OK); +} + +#[tokio::test] +async fn authorize_api_evaluates_the_claimed_resource_id_not_context_spoofing() { + let (state, _store, _temp) = state_with_turso("authorize-exact-resource").await; + state + .authz + .reload_tenant_policies( + TENANT, + r#" +permit( + principal == Customer::"reader", + action == Action::"inspect", + resource == Tool::"target" +); +"#, + ) + .expect("exact-resource policy should parse"); + let app = build_router(state); + + let allowed = app + .clone() + .oneshot(json_request( + Method::POST, + "/api/authorize", + serde_json::json!({ + "agent_id": "reader", + "action": "inspect", + "resource_type": "Tool", + "resource_id": "target", + "context": {"id": "attacker-override", "classification": "public"} + }), + TENANT, + "reader", + )) + .await + .expect("authorization request should run"); + assert_eq!(allowed.status(), StatusCode::OK); + let allowed_body = axum::body::to_bytes(allowed.into_body(), 64 * 1024) + .await + .expect("read allowed response"); + assert_eq!( + serde_json::from_slice::(&allowed_body).expect("allowed response JSON") + ["allowed"], + true + ); + + let denied = app + .oneshot(json_request( + Method::POST, + "/api/authorize", + serde_json::json!({ + "agent_id": "reader", + "action": "inspect", + "resource_type": "Tool", + "resource_id": "other", + "context": {"id": "target"} + }), + TENANT, + "reader", + )) + .await + .expect("authorization request should run"); + assert_eq!(denied.status(), StatusCode::OK); + let denied_body = axum::body::to_bytes(denied.into_body(), 64 * 1024) + .await + .expect("read denied response"); + assert_eq!( + serde_json::from_slice::(&denied_body).expect("denied response JSON")["allowed"], + false + ); +} + +#[path = "resource_authorization/artifact_authorization.rs"] +mod artifact_authorization; diff --git a/crates/temper-server/tests/resource_authorization/artifact_authorization.rs b/crates/temper-server/tests/resource_authorization/artifact_authorization.rs new file mode 100644 index 000000000..5356ca8b7 --- /dev/null +++ b/crates/temper-server/tests/resource_authorization/artifact_authorization.rs @@ -0,0 +1,244 @@ +use super::*; + +fn artifact_vault(endpoint: &str) -> SecretsVault { + let vault = SecretsVault::new(&[11_u8; 32]); + for (key, value) in [ + ("published_blob_public_base_url", "https://public.example"), + ("published_blob_endpoint", endpoint), + ("published_blob_bucket", "published-bucket"), + ] { + vault + .cache_secret(TENANT, key, value.to_string()) + .expect("cache artifact secret"); + } + vault +} + +fn artifact_body(namespace: &str, source_version: &str) -> serde_json::Value { + serde_json::json!({ + "file_id": "file-a", + "label": "latest", + "owner_ref_type": "Document", + "owner_ref_id": "doc-a", + "source_file_version_id": source_version, + "namespace": namespace, + }) +} + +#[tokio::test] +async fn artifact_publish_requires_dedicated_source_permission_and_safe_segments() { + let mock = MockServer::start().await; + Mock::given(method("PUT")) + .respond_with(ResponseTemplate::new(200)) + .expect(1) + .mount(&mock) + .await; + let (mut state, store, _temp) = state_with_turso("artifact-resource-auth").await; + state.secrets_vault = Some(std::sync::Arc::new(artifact_vault(&mock.uri()))); + state + .authz + .reload_tenant_policies( + TENANT, + r#" +permit( + principal == Customer::"publisher", + action == Action::"publish_artifact", + resource == File::"file-a" +); +permit( + principal == Customer::"read-only", + action == Action::"read", + resource == File::"file-a" +); +permit( + principal == Customer::"version-only", + action == Action::"publish_artifact", + resource == FileVersion::"version-a" +); +permit( + principal == Customer::"publisher", + action == Action::"publish_artifact", + resource == FileVersion::"version-mismatch" +); +permit( + principal == Customer::"forged-metadata", + action == Action::"publish_artifact", + resource == File::"file-a" +) +when { + context.owner_ref_id == "doc-a" +}; +"#, + ) + .expect("artifact policy should parse"); + state + .authz + .reload_tenant_policies("other-tenant", "") + .expect("other tenant should default-deny"); + seed_file( + &store, + "File", + "file-a", + "sha256:artifactcontent", + b"artifact body", + None, + ) + .await; + seed_file( + &store, + "FileVersion", + "version-a", + "sha256:artifactversion", + b"artifact version", + Some("file-a"), + ) + .await; + seed_file( + &store, + "FileVersion", + "version-mismatch", + "sha256:artifactmismatch", + b"artifact mismatch", + Some("file-b"), + ) + .await; + let app = build_router(state); + + for (tenant, principal, body, expected) in [ + ( + TENANT, + "intruder", + artifact_body("artifacts", ""), + StatusCode::FORBIDDEN, + ), + ( + TENANT, + "read-only", + artifact_body("artifacts", ""), + StatusCode::FORBIDDEN, + ), + ( + TENANT, + "forged-metadata", + artifact_body("artifacts", ""), + StatusCode::FORBIDDEN, + ), + ( + "other-tenant", + "publisher", + artifact_body("artifacts", ""), + StatusCode::FORBIDDEN, + ), + ( + TENANT, + "publisher", + artifact_body("artifacts", "version-a"), + StatusCode::FORBIDDEN, + ), + ( + TENANT, + "version-only", + artifact_body("artifacts", "version-a"), + StatusCode::FORBIDDEN, + ), + ( + TENANT, + "publisher", + artifact_body("artifacts", "version-mismatch"), + StatusCode::BAD_REQUEST, + ), + ( + TENANT, + "publisher", + artifact_body("../escape", ""), + StatusCode::BAD_REQUEST, + ), + ] { + let response = app + .clone() + .oneshot(json_request( + Method::POST, + "/api/files/publish-artifact", + body, + tenant, + principal, + )) + .await + .expect("rejected publish request should run"); + assert_eq!( + response.status(), + expected, + "unexpected publish status for tenant {tenant} principal {principal}" + ); + } + + let allowed = app + .oneshot(json_request( + Method::POST, + "/api/files/publish-artifact", + artifact_body("artifacts", ""), + TENANT, + "publisher", + )) + .await + .expect("authorized publish request should run"); + assert_eq!(allowed.status(), StatusCode::OK); + mock.verify().await; +} + +#[tokio::test] +async fn local_artifact_publish_uses_direct_tenant_blob_store() { + let (mut state, store, _temp) = state_with_turso("artifact-local-write").await; + state.secrets_vault = Some(std::sync::Arc::new(artifact_vault( + "http://127.0.0.1:9/_internal/blobs", + ))); + state + .authz + .reload_tenant_policies( + TENANT, + r#"permit( + principal == Customer::"publisher", + action == Action::"publish_artifact", + resource == File::"file-a" +);"#, + ) + .expect("artifact policy should parse"); + seed_file( + &store, + "File", + "file-a", + "sha256:localartifact", + b"local artifact body", + None, + ) + .await; + let state_for_read = state.clone(); + let app = build_router(state); + + let response = app + .oneshot(json_request( + Method::POST, + "/api/files/publish-artifact", + artifact_body("artifacts", ""), + TENANT, + "publisher", + )) + .await + .expect("local publish request should run"); + assert_eq!(response.status(), StatusCode::OK); + let body = axum::body::to_bytes(response.into_body(), 1024 * 1024) + .await + .expect("read publish response"); + let json: serde_json::Value = serde_json::from_slice(&body).expect("publish JSON"); + let storage_key = json["artifact"]["public_storage_key"] + .as_str() + .expect("public storage key"); + let stored = state_for_read + .get_blob_with_legacy_fallback( + &TenantId::default(), + &format!("published-bucket/{storage_key}"), + ) + .await + .expect("read local artifact blob"); + assert_eq!(stored.as_deref(), Some(b"local artifact body".as_slice())); +} diff --git a/crates/temper-server/tests/resource_authorization/secret_authorization.rs b/crates/temper-server/tests/resource_authorization/secret_authorization.rs new file mode 100644 index 000000000..8015c422b --- /dev/null +++ b/crates/temper-server/tests/resource_authorization/secret_authorization.rs @@ -0,0 +1,122 @@ +use super::*; + +#[tokio::test] +async fn secret_crud_uses_manage_secrets_for_exact_tenant_and_key() { + let (mut state, _store, _temp) = state_with_turso("secret-resource-auth").await; + state.secrets_vault = Some(std::sync::Arc::new(SecretsVault::new(&[12_u8; 32]))); + state + .authz + .reload_tenant_policies( + TENANT, + r#" +permit( + principal == Customer::"secret-manager", + action == Action::"manage_secrets", + resource +) when { + resource == Secret::"api-key" || resource == Secret::"__keys__" +}; +"#, + ) + .expect("secret policy should parse"); + let state_for_check = state.clone(); + let app = build_router(state); + + let denied = app + .clone() + .oneshot(json_request( + Method::PUT, + "/api/tenants/default/secrets/api-key", + serde_json::json!({"value": "denied"}), + TENANT, + "intruder", + )) + .await + .expect("denied secret request should run"); + assert_eq!(denied.status(), StatusCode::FORBIDDEN); + + let wrong_key = app + .clone() + .oneshot(json_request( + Method::PUT, + "/api/tenants/default/secrets/other-key", + serde_json::json!({"value": "denied"}), + TENANT, + "secret-manager", + )) + .await + .expect("wrong-key secret request should run"); + assert_eq!(wrong_key.status(), StatusCode::FORBIDDEN); + + let wrong_tenant = app + .clone() + .oneshot(json_request( + Method::PUT, + "/api/tenants/default/secrets/api-key", + serde_json::json!({"value": "denied"}), + "other-tenant", + "secret-manager", + )) + .await + .expect("wrong-tenant secret request should run"); + assert_eq!(wrong_tenant.status(), StatusCode::UNAUTHORIZED); + + let allowed = app + .clone() + .oneshot(json_request( + Method::PUT, + "/api/tenants/default/secrets/api-key", + serde_json::json!({"value": "stored-value"}), + TENANT, + "secret-manager", + )) + .await + .expect("allowed secret request should run"); + assert_eq!(allowed.status(), StatusCode::NO_CONTENT); + + let listed = app + .clone() + .oneshot(json_request( + Method::GET, + "/api/tenants/default/secrets", + serde_json::Value::Null, + TENANT, + "secret-manager", + )) + .await + .expect("list secret request should run"); + assert_eq!(listed.status(), StatusCode::OK); + + let denied_delete = app + .clone() + .oneshot(json_request( + Method::DELETE, + "/api/tenants/default/secrets/api-key", + serde_json::Value::Null, + TENANT, + "intruder", + )) + .await + .expect("denied delete request should run"); + assert_eq!(denied_delete.status(), StatusCode::FORBIDDEN); + assert_eq!( + state_for_check + .secrets_vault + .as_ref() + .and_then(|vault| vault.get_secret(TENANT, "api-key")) + .as_deref(), + Some("stored-value") + ); + + let deleted = app + .oneshot(json_request( + Method::DELETE, + "/api/tenants/default/secrets/api-key", + serde_json::Value::Null, + TENANT, + "secret-manager", + )) + .await + .expect("allowed delete request should run"); + assert_eq!(deleted.status(), StatusCode::NO_CONTENT); +} diff --git a/crates/temper-server/tests/spec_validate_endpoint.rs b/crates/temper-server/tests/spec_validate_endpoint.rs index 3880168db..c06f29c83 100644 --- a/crates/temper-server/tests/spec_validate_endpoint.rs +++ b/crates/temper-server/tests/spec_validate_endpoint.rs @@ -1,6 +1,10 @@ +#![cfg(feature = "observe")] + use axum::body::Body; use axum::http::{Request, StatusCode}; +use temper_authz::{AuthenticatedRequestContext, SecurityContext}; use temper_runtime::ActorSystem; +use temper_runtime::tenant::TenantId; use temper_server::{ServerState, SpecRegistry, build_router}; use temper_spec::csdl::parse_csdl; use tower::ServiceExt; @@ -20,19 +24,25 @@ fn test_state_with_registry() -> ServerState { ServerState::from_registry(ActorSystem::new("spec-validate-endpoint-test"), registry) } -fn admin_post(uri: &str, body: &str) -> Request { - Request::post(uri) +fn authenticated_post(uri: &str, body: &str) -> Request { + let mut request = Request::post(uri) .header("Content-Type", "application/json") - .header("X-Temper-Principal-Kind", "admin") .body(Body::from(body.to_string())) - .unwrap() + .unwrap(); + request + .extensions_mut() + .insert(AuthenticatedRequestContext::new( + TenantId::default(), + SecurityContext::system(), + )); + request } #[tokio::test] async fn validate_ioa_runs_server_cascade_without_loading_spec() { let app = build_router(test_state_with_registry()); let response = app - .oneshot(admin_post( + .oneshot(authenticated_post( "/api/specs/validate-ioa", &serde_json::json!({ "ioa_source": ORDER_IOA, @@ -56,7 +66,7 @@ async fn validate_ioa_runs_server_cascade_without_loading_spec() { async fn validate_ioa_rejects_empty_source() { let app = build_router(test_state_with_registry()); let response = app - .oneshot(admin_post( + .oneshot(authenticated_post( "/api/specs/validate-ioa", &serde_json::json!({ "ioa_source": "" }).to_string(), )) diff --git a/crates/temper-server/tests/storage_stack.rs b/crates/temper-server/tests/storage_stack.rs index e49d338b7..dbe8e6629 100644 --- a/crates/temper-server/tests/storage_stack.rs +++ b/crates/temper-server/tests/storage_stack.rs @@ -396,5 +396,6 @@ fn trajectory_entry() -> TrajectoryEntry { request_body: None, intent: None, matched_policy_ids: None, + capture_seq: None, } } diff --git a/crates/temper-server/tests/trajectory_capture.rs b/crates/temper-server/tests/trajectory_capture.rs new file mode 100644 index 000000000..66ade10c9 --- /dev/null +++ b/crates/temper-server/tests/trajectory_capture.rs @@ -0,0 +1,598 @@ +//! End-to-end trajectory capture over the HTTP surface. +//! +//! Covers the two properties a JCS trajectory consumer depends on: +//! +//! 1. A **successful** governed action produces a durable trajectory row that +//! carries its `request_body` — not only failures, which is all the capture +//! path used to record. +//! 2. `X-Session-Id` and `X-Intent` travel from the HTTP request all the way +//! into the persisted row, on both the success and the failure path. + +mod common; + +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use temper_authz::{AuthenticatedRequestContext, SecurityContext}; +use temper_runtime::ActorSystem; +use temper_runtime::tenant::TenantId; +use temper_server::build_router; +use temper_server::registry::{ + EntityLevelSummary, EntityVerificationResult, SpecRegistry, VerificationStatus, +}; +use temper_server::state::TrajectoryEntry; +use temper_server::{ServerState, StorageStack}; +use temper_spec::csdl::parse_csdl; +use temper_store_turso::TursoEventStore; +use tower::ServiceExt; + +const CSDL_XML: &str = common::CSDL_XML; +const ORDER_IOA: &str = common::ORDER_IOA; + +const SESSION_ID: &str = "sess-jcs-e2e"; +const INTENT: &str = "add a line item to the draft order"; + +/// Build a Turso-backed state so trajectory rows land in a real sink. +/// +/// The sim store has no trajectory capability, so a durable backend is the +/// only way to assert on what was actually persisted. +fn build_turso_state(system_name: &str, store: TursoEventStore) -> ServerState { + let mut registry = SpecRegistry::new(); + let csdl = parse_csdl(CSDL_XML).expect("CSDL parse"); + registry.register_tenant( + "default", + csdl, + CSDL_XML.to_string(), + &[("Order", ORDER_IOA)], + ); + + let state = ServerState::from_registry(ActorSystem::new(system_name), registry); + { + let mut registry = state.registry.write().unwrap(); + registry.set_verification_status( + &TenantId::default(), + "Order", + VerificationStatus::Completed(EntityVerificationResult { + all_passed: true, + levels: vec![EntityLevelSummary { + level: "L0 SMT".to_string(), + passed: true, + summary: "OK".to_string(), + details: None, + }], + verified_at: "2026-08-11T00:00:00Z".to_string(), + }), + ); + } + + let mut state = state; + // ARN-170 hardened `from_registry`'s default engine to default-deny + // (`AuthzEngine::empty()`); these capture/telemetry tests exercise the write + // path itself, not authorization, so install a permissive tenant policy — + // the effective posture they were written against. Tests that need a denial + // (e.g. cedar_denied_action) reload a restrictive policy over this. + state + .authz + .reload_tenant_policies( + TenantId::default().as_str(), + r#"permit(principal, action, resource);"#, + ) + .expect("install permissive test policy"); + state.set_storage_stack(StorageStack::from_turso(store)); + state +} + +async fn temp_store(label: &str) -> (TursoEventStore, std::path::PathBuf) { + let db_path = std::env::temp_dir().join(format!("temper-{label}-{}.db", uuid::Uuid::new_v4())); + let db_url = format!("file:{}", db_path.display()); + let store = TursoEventStore::new(&db_url, None) + .await + .expect("create local turso db"); + (store, db_path) +} + +/// Attach the credential context the ingress edge installs in production +/// (ADR-0157). These fixtures carry no principal headers, so the anonymous +/// Customer is exactly what the pre-edge header path produced; the +/// `X-Session-Id`/`X-Intent` correlation headers still travel on the request +/// and are read by the odata dispatch path. +fn with_test_auth(mut request: Request) -> Request { + request + .extensions_mut() + .insert(AuthenticatedRequestContext::new( + TenantId::default(), + SecurityContext::anonymous(), + )); + request +} + +/// POST with the observability headers under test. +async fn post_observed( + state: &ServerState, + path: &str, + body: serde_json::Value, +) -> (StatusCode, serde_json::Value) { + let router = build_router(state.clone()); + let req = with_test_auth( + Request::post(path) + .header("Content-Type", "application/json") + .header("X-Session-Id", SESSION_ID) + .header("X-Intent", INTENT) + .body(Body::from(body.to_string())) + .unwrap(), + ); + let resp = router.oneshot(req).await.unwrap(); + let status = resp.status(); + let bytes = axum::body::to_bytes(resp.into_body(), 1_000_000) + .await + .unwrap(); + let parsed = serde_json::from_slice(&bytes).unwrap_or(serde_json::Value::Null); + (status, parsed) +} + +/// Wait for a persisted trajectory row matching `predicate`. +/// +/// Trajectory persistence is a background outbox (ADR-0067), so the row lands +/// after the HTTP response returns. +async fn await_trajectory( + state: &ServerState, + label: &str, + predicate: impl Fn(&TrajectoryEntry) -> bool, +) -> TrajectoryEntry { + for _ in 0..200 { + if let Some(found) = state + .load_trajectory_entries(TenantId::default().as_str(), 200) + .await + .into_iter() + .find(&predicate) + { + return found; + } + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + } + let seen: Vec = state + .load_trajectory_entries(TenantId::default().as_str(), 200) + .await + .into_iter() + .map(|e| format!("{}.{} success={}", e.entity_type, e.action, e.success)) + .collect(); + panic!("no trajectory row matched '{label}'; rows seen: {seen:?}"); +} + +#[tokio::test] +async fn successful_governed_action_persists_request_body_session_and_intent() { + let (store, db_path) = temp_store("trajectory-success").await; + let state = build_turso_state("trajectory-capture-success", store); + + let (status, body) = post_observed( + &state, + "/tdata/Orders", + serde_json::json!({"id": "ord-jcs-1", "Currency": "USD"}), + ) + .await; + assert_eq!(status, StatusCode::CREATED, "seed create failed: {body:?}"); + + let (status, body) = post_observed( + &state, + "/tdata/Orders('ord-jcs-1')/Temper.AddItem", + serde_json::json!({"ProductId": "prod-9", "Quantity": 3}), + ) + .await; + assert_eq!(status, StatusCode::OK, "AddItem failed: {body:?}"); + + let entry = await_trajectory(&state, "successful AddItem", |entry| { + entry.action == "AddItem" && entry.entity_id == "ord-jcs-1" && entry.success + }) + .await; + + assert!(entry.success, "the captured row is the successful action"); + assert_eq!( + entry.session_id.as_deref(), + Some(SESSION_ID), + "X-Session-Id must reach the persisted trajectory row" + ); + assert_eq!( + entry.intent.as_deref(), + Some(INTENT), + "X-Intent must reach the persisted trajectory row" + ); + + let request_body = entry + .request_body + .as_ref() + .expect("successful actions must persist their request body"); + assert_eq!(request_body["ProductId"], serde_json::json!("prod-9")); + assert_eq!(request_body["Quantity"], serde_json::json!(3)); + + let _ = std::fs::remove_file(db_path); +} + +#[tokio::test] +async fn failed_governed_action_still_persists_request_body_session_and_intent() { + let (store, db_path) = temp_store("trajectory-failure").await; + let state = build_turso_state("trajectory-capture-failure", store); + + let (status, body) = post_observed( + &state, + "/tdata/Orders", + serde_json::json!({"id": "ord-jcs-2", "Currency": "USD"}), + ) + .await; + assert_eq!(status, StatusCode::CREATED, "seed create failed: {body:?}"); + + // SubmitOrder guards on `items > 0`; the fresh order has none, so the + // guard rejects and the dispatch records a failed intent. + let (status, _body) = post_observed( + &state, + "/tdata/Orders('ord-jcs-2')/Temper.SubmitOrder", + serde_json::json!({"ShippingAddressId": "addr-1", "PaymentMethod": "card"}), + ) + .await; + assert_eq!( + status, + StatusCode::CONFLICT, + "SubmitOrder without items must fail the guard" + ); + + let entry = await_trajectory(&state, "failed SubmitOrder", |entry| { + entry.action == "SubmitOrder" && entry.entity_id == "ord-jcs-2" && !entry.success + }) + .await; + + assert_eq!(entry.session_id.as_deref(), Some(SESSION_ID)); + assert_eq!(entry.intent.as_deref(), Some(INTENT)); + let request_body = entry + .request_body + .as_ref() + .expect("failed actions keep persisting their request body"); + assert_eq!( + request_body["ShippingAddressId"], + serde_json::json!("addr-1") + ); + assert!(entry.error.is_some(), "the failure reason is recorded"); + + let _ = std::fs::remove_file(db_path); +} + +#[tokio::test] +async fn observe_prefixed_headers_are_honoured_as_session_and_intent() { + // The canonical spellings are the `X-Temper-Observe-*` headers; the short + // `X-Session-Id`/`X-Intent` forms are aliases. Both must land identically. + let (store, db_path) = temp_store("trajectory-observe-headers").await; + let state = build_turso_state("trajectory-capture-observe-headers", store); + + let router = build_router(state.clone()); + let resp = router + .oneshot(with_test_auth( + Request::post("/tdata/Orders") + .header("Content-Type", "application/json") + .body(Body::from( + serde_json::json!({"id": "ord-jcs-3", "Currency": "USD"}).to_string(), + )) + .unwrap(), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::CREATED); + + let router = build_router(state.clone()); + let resp = router + .oneshot(with_test_auth( + Request::post("/tdata/Orders('ord-jcs-3')/Temper.AddItem") + .header("Content-Type", "application/json") + .header("X-Temper-Observe-Session-Id", "sess-observe-prefixed") + .header("X-Temper-Observe-Intent", "observe-prefixed intent") + .body(Body::from( + serde_json::json!({"ProductId": "prod-3", "Quantity": 1}).to_string(), + )) + .unwrap(), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + + let entry = await_trajectory(&state, "observe-prefixed AddItem", |entry| { + entry.action == "AddItem" && entry.entity_id == "ord-jcs-3" && entry.success + }) + .await; + + assert_eq!( + entry.session_id.as_deref(), + Some("sess-observe-prefixed"), + "X-Temper-Observe-Session-Id must reach the persisted row" + ); + assert_eq!( + entry.intent.as_deref(), + Some("observe-prefixed intent"), + "X-Temper-Observe-Intent must reach the persisted row" + ); + + let _ = std::fs::remove_file(db_path); +} + +#[tokio::test] +async fn cedar_denied_action_persists_intent_and_evaluated_attributes() { + // An authorization denial is the row the Evolution Engine reasons over. + // Without the caller's intent and the attributes Cedar actually saw, the + // denial says what was blocked but not what the agent was attempting. + let (store, db_path) = temp_store("trajectory-denial").await; + let state = build_turso_state("trajectory-capture-denial", store); + + let (status, body) = post_observed( + &state, + "/tdata/Orders", + serde_json::json!({"id": "ord-jcs-4", "Currency": "USD"}), + ) + .await; + assert_eq!(status, StatusCode::CREATED, "seed create failed: {body:?}"); + + state + .authz + .reload_tenant_policies( + TenantId::default().as_str(), + r#"permit(principal, action in [Action::"list", Action::"read"], resource is Order);"#, + ) + .expect("install Cedar policy"); + + let (status, _body) = post_observed( + &state, + "/tdata/Orders('ord-jcs-4')/Temper.AddItem", + serde_json::json!({"ProductId": "prod-denied", "Quantity": 1}), + ) + .await; + assert_eq!( + status, + StatusCode::FORBIDDEN, + "AddItem must be denied under a read-only policy set" + ); + + let entry = await_trajectory(&state, "denied AddItem", |entry| { + entry.action == "AddItem" + && entry.entity_id == "ord-jcs-4" + && entry.authz_denied == Some(true) + }) + .await; + + assert_eq!(entry.intent.as_deref(), Some(INTENT)); + let request_body = entry + .request_body + .as_ref() + .expect("denials must persist the attributes Cedar evaluated"); + assert_eq!( + request_body["id"], + serde_json::json!("ord-jcs-4"), + "the evaluated resource attributes are recorded" + ); + + let _ = std::fs::remove_file(db_path); +} + +#[tokio::test] +async fn oversized_request_body_is_bounded_before_persistence() { + // Capturing every successful action makes large bodies routine rather than + // rare. The stored row must stay bounded and stay parseable JSON — a + // byte-sliced prefix would read back as no body at all. + let (store, db_path) = temp_store("trajectory-oversized").await; + let state = build_turso_state("trajectory-capture-oversized", store); + + let (status, body) = post_observed( + &state, + "/tdata/Orders", + serde_json::json!({"id": "ord-jcs-5", "Currency": "USD"}), + ) + .await; + assert_eq!(status, StatusCode::CREATED, "seed create failed: {body:?}"); + + let (status, body) = post_observed( + &state, + "/tdata/Orders('ord-jcs-5')/Temper.AddItem", + serde_json::json!({"ProductId": "prod-big", "Quantity": 1, "Notes": "x".repeat(50_000)}), + ) + .await; + assert_eq!(status, StatusCode::OK, "AddItem failed: {body:?}"); + + let entry = await_trajectory(&state, "oversized AddItem", |entry| { + entry.action == "AddItem" && entry.entity_id == "ord-jcs-5" && entry.success + }) + .await; + + let request_body = entry + .request_body + .as_ref() + .expect("an oversized body must still round-trip as parseable JSON, not vanish"); + assert_eq!( + request_body["_truncated"], + serde_json::json!(true), + "the stored row declares that it was truncated: {request_body}" + ); + assert!( + request_body["_original_bytes"] + .as_u64() + .expect("original bytes") + > 50_000, + "the envelope records the pre-truncation size" + ); + assert!( + request_body.to_string().len() <= 4096, + "the stored body respects the cap" + ); + + let _ = std::fs::remove_file(db_path); +} + +#[tokio::test] +async fn secret_named_parameters_are_redacted_before_persistence() { + // Every successful action now records its arguments, so an action that + // takes a credential would put it in a durable row that trajectory + // observation and training exports both read. + let (store, db_path) = temp_store("trajectory-secrets").await; + let state = build_turso_state("trajectory-capture-secrets", store); + + let (status, body) = post_observed( + &state, + "/tdata/Orders", + serde_json::json!({"id": "ord-jcs-6", "Currency": "USD"}), + ) + .await; + assert_eq!(status, StatusCode::CREATED, "seed create failed: {body:?}"); + + let (status, body) = post_observed( + &state, + "/tdata/Orders('ord-jcs-6')/Temper.AddItem", + serde_json::json!({ + "ProductId": "prod-secret", + "Quantity": 1, + "api_token": "sk-live-must-not-be-stored", + "payment": {"card_number": "4111111111111111", "cvv": "123"} + }), + ) + .await; + assert_eq!(status, StatusCode::OK, "AddItem failed: {body:?}"); + + let entry = await_trajectory(&state, "AddItem with secrets", |entry| { + entry.action == "AddItem" && entry.entity_id == "ord-jcs-6" && entry.success + }) + .await; + + let request_body = entry.request_body.as_ref().expect("request body persisted"); + let rendered = request_body.to_string(); + assert!( + !rendered.contains("sk-live-must-not-be-stored"), + "the token must not survive into storage: {rendered}" + ); + assert!( + !rendered.contains("4111111111111111") && !rendered.contains("\"123\""), + "nested payment details must not survive into storage: {rendered}" + ); + assert_eq!( + request_body["ProductId"], + serde_json::json!("prod-secret"), + "ordinary arguments are still captured" + ); + + let _ = std::fs::remove_file(db_path); +} + +#[tokio::test] +async fn a_rejected_retry_records_the_state_it_was_attempted_from() { + // A guard failure appends no event, so reading the newest event would + // report the source state of the *previous* successful transition — the + // state where the action was legal — and hide the illegal retry. + let (store, db_path) = temp_store("trajectory-retry-source").await; + let state = build_turso_state("trajectory-capture-retry-source", store); + + let (status, body) = post_observed( + &state, + "/tdata/Orders", + serde_json::json!({"id": "ord-jcs-7", "Currency": "USD"}), + ) + .await; + assert_eq!(status, StatusCode::CREATED, "seed create failed: {body:?}"); + + let (status, body) = post_observed( + &state, + "/tdata/Orders('ord-jcs-7')/Temper.AddItem", + serde_json::json!({"ProductId": "prod-1", "Quantity": 1}), + ) + .await; + assert_eq!(status, StatusCode::OK, "AddItem failed: {body:?}"); + + let (status, body) = post_observed( + &state, + "/tdata/Orders('ord-jcs-7')/Temper.SubmitOrder", + serde_json::json!({"ShippingAddressId": "addr-1", "PaymentMethod": "card"}), + ) + .await; + assert_eq!(status, StatusCode::OK, "SubmitOrder failed: {body:?}"); + + // Second SubmitOrder: the order is already Submitted, where the action is + // illegal, so the guard rejects it. + let (status, _body) = post_observed( + &state, + "/tdata/Orders('ord-jcs-7')/Temper.SubmitOrder", + serde_json::json!({"ShippingAddressId": "addr-1", "PaymentMethod": "card"}), + ) + .await; + assert_eq!( + status, + StatusCode::CONFLICT, + "SubmitOrder from Submitted must be rejected" + ); + + let entry = await_trajectory(&state, "rejected SubmitOrder retry", |entry| { + entry.action == "SubmitOrder" && entry.entity_id == "ord-jcs-7" && !entry.success + }) + .await; + + assert_eq!( + entry.from_status.as_deref(), + Some("Submitted"), + "the rejected attempt was made from Submitted, not from the Draft the \ + previous successful transition started in" + ); + assert_eq!(entry.to_status.as_deref(), Some("Submitted")); + + let _ = std::fs::remove_file(db_path); +} + +#[tokio::test] +async fn a_session_reads_back_in_capture_order() { + // Rows are persisted by independently spawned tasks, so the storage id is + // the order the writes landed. The session read must reproduce the order + // the kernel captured, which is what the conformance walk replays. + let (store, db_path) = temp_store("trajectory-capture-order").await; + let state = build_turso_state("trajectory-capture-order", store.clone()); + + let (status, body) = post_observed( + &state, + "/tdata/Orders", + serde_json::json!({"id": "ord-jcs-8", "Currency": "USD"}), + ) + .await; + assert_eq!(status, StatusCode::CREATED, "seed create failed: {body:?}"); + + for quantity in 1..=6 { + let (status, body) = post_observed( + &state, + "/tdata/Orders('ord-jcs-8')/Temper.AddItem", + serde_json::json!({"ProductId": format!("prod-{quantity}"), "Quantity": quantity}), + ) + .await; + assert_eq!(status, StatusCode::OK, "AddItem failed: {body:?}"); + } + let (status, body) = post_observed( + &state, + "/tdata/Orders('ord-jcs-8')/Temper.SubmitOrder", + serde_json::json!({"ShippingAddressId": "addr-1", "PaymentMethod": "card"}), + ) + .await; + assert_eq!(status, StatusCode::OK, "SubmitOrder failed: {body:?}"); + + // Wait for the last captured row to land, then read the session back. + let _ = await_trajectory(&state, "SubmitOrder", |entry| { + entry.action == "SubmitOrder" && entry.entity_id == "ord-jcs-8" && entry.success + }) + .await; + + let rows = store + .query_trajectories_by_session(SESSION_ID, Some("default"), Some("Order"), 100) + .await + .expect("read session"); + let sequences: Vec = rows.iter().filter_map(|row| row.capture_seq).collect(); + assert_eq!( + sequences.len(), + rows.len(), + "every captured row carries its capture order" + ); + assert!( + sequences.windows(2).all(|pair| pair[0] < pair[1]), + "the session must read back in capture order, got {sequences:?}" + ); + let actions: Vec<&str> = rows.iter().map(|row| row.action.as_str()).collect(); + assert_eq!( + actions.last(), + Some(&"SubmitOrder"), + "the last captured action is last in the read: {actions:?}" + ); + + let _ = std::fs::remove_file(db_path); +} diff --git a/crates/temper-server/tests/wasm_dispatch.rs b/crates/temper-server/tests/wasm_dispatch.rs index 50c4b29ff..e412e3f93 100644 --- a/crates/temper-server/tests/wasm_dispatch.rs +++ b/crates/temper-server/tests/wasm_dispatch.rs @@ -151,6 +151,23 @@ permit( ); "#; +const ECHO_HTTP_POLICY: &str = r#" +permit( + principal is Agent, + action == Action::"http_call", + resource is HttpEndpoint +) when { + context.module == "echo_integration" +}; +"#; + +fn install_echo_http_policy(state: &ServerState) { + state + .authz + .reload_tenant_policies(TenantId::default().as_str(), ECHO_HTTP_POLICY) + .expect("policy should parse"); +} + fn install_non_wasm_policy(state: &ServerState) { state .authz @@ -192,7 +209,7 @@ async fn assert_wasm_authz_denial_artifacts(state: &ServerState, entity_id: &str let mut authz_traj = None; for _ in 0..100 { let trajectories = turso - .load_recent_trajectories(1000) + .load_recent_trajectories("default", 1000) .await .expect("query trajectories from Turso"); authz_traj = trajectories @@ -246,6 +263,7 @@ async fn assert_wasm_authz_denial_artifacts(state: &ServerState, entity_id: &str async fn wasm_integration_dispatches_callback() { let state = build_echo_test_state(); let tenant = TenantId::default(); + install_echo_http_policy(&state); // Register the WASM module in the engine and module registry. let hash = state @@ -268,7 +286,7 @@ async fn wasm_integration_dispatches_callback() { "echo-1", "TriggerEcho", serde_json::json!({}), - &AgentContext::default(), + &AgentContext::system(), ) .await .expect("TriggerEcho should succeed"); @@ -309,6 +327,7 @@ async fn wasm_integration_dispatches_callback() { async fn persisted_wasm_modules_are_lazy_compiled_on_first_invoke() { let state = build_echo_test_state_with_turso().await; let tenant = TenantId::default(); + install_echo_http_policy(&state); let hash = temper_wasm::WasmEngine::hash_module(ECHO_WASM); state @@ -342,7 +361,7 @@ async fn persisted_wasm_modules_are_lazy_compiled_on_first_invoke() { "echo-lazy-1", "TriggerEcho", serde_json::json!({}), - &AgentContext::default(), + &AgentContext::system(), ) .await .expect("TriggerEcho should succeed"); @@ -354,7 +373,7 @@ async fn persisted_wasm_modules_are_lazy_compiled_on_first_invoke() { "EchoTest", "echo-lazy-1", &["Done", "Failed"], - Duration::from_secs(5), + Duration::from_secs(45), ) .await; assert_eq!(final_status, "Done"); @@ -368,6 +387,7 @@ async fn persisted_wasm_modules_are_lazy_compiled_on_first_invoke() { async fn persisted_wasm_modules_with_legacy_db_blob_fallback_execute_after_startup_restore() { let state = build_echo_test_state_with_turso().await; let tenant = TenantId::default(); + install_echo_http_policy(&state); let turso = state .platform_turso_store() .expect("turso backend required"); @@ -411,7 +431,7 @@ async fn persisted_wasm_modules_with_legacy_db_blob_fallback_execute_after_start "echo-legacy-hash", "TriggerEcho", serde_json::json!({}), - &AgentContext::default(), + &AgentContext::system(), ) .await .expect("TriggerEcho should succeed"); @@ -423,7 +443,7 @@ async fn persisted_wasm_modules_with_legacy_db_blob_fallback_execute_after_start "EchoTest", "echo-legacy-hash", &["Done", "Failed"], - Duration::from_secs(5), + Duration::from_secs(45), ) .await; assert_eq!(final_status, "Done"); diff --git a/crates/temper-server/tests/wasm_dispatch_observability_contract.rs b/crates/temper-server/tests/wasm_dispatch_observability_contract.rs index 521afdb76..e10b297a0 100644 --- a/crates/temper-server/tests/wasm_dispatch_observability_contract.rs +++ b/crates/temper-server/tests/wasm_dispatch_observability_contract.rs @@ -2,6 +2,51 @@ const WASM_DISPATCH_SOURCE: &str = include_str!("../src/state/dispatch/wasm.rs"); +/// Blank out `//` line comments so a commented-out call cannot satisfy a source +/// contract. Byte offsets are preserved (comment bodies become spaces) so every +/// ordering assertion still compares positions in the original file. String +/// literals are tracked, so a `"http://..."` in code is not mistaken for a +/// comment and does not blank the rest of its line. +/// +/// This is a heuristic, not a lexer: char literals (`'/'`) and raw strings +/// (`r#"..."#`) are not modelled. Both scanned files are free of them today, and +/// every assertion built on this asserts *presence*, so a mis-scan produces a +/// loud failure rather than a silent pass. +fn strip_line_comments(src: &str) -> String { + let mut out = String::with_capacity(src.len()); + for line in src.split_inclusive('\n') { + let bytes = line.as_bytes(); + let mut in_string = false; + let mut escaped = false; + let mut comment_at = None; + for (i, &b) in bytes.iter().enumerate() { + if escaped { + escaped = false; + continue; + } + match b { + b'\\' if in_string => escaped = true, + b'"' => in_string = !in_string, + b'/' if !in_string && bytes.get(i + 1) == Some(&b'/') => { + comment_at = Some(i); + break; + } + _ => {} + } + } + match comment_at { + Some(at) => { + out.push_str(&line[..at]); + for ch in line[at..].chars() { + out.push(if ch == '\n' { '\n' } else { ' ' }); + } + } + None => out.push_str(line), + } + } + out +} + #[test] fn wasm_dispatch_emits_integration_envelope_phase_spans() { let required_phases = [ @@ -27,3 +72,123 @@ fn wasm_dispatch_emits_integration_envelope_phase_spans() { ); } } + +/// ARN-243 wiring contract: per-tenant LLM content redaction must run before +/// every telemetry sink reads the callback params. A refactor that moved the +/// strip after any sink would leak prompts/completions for non-opted-in +/// tenants — this guards the ordering that unit tests on the helper cannot. +/// See ADR-0166. +#[test] +fn llm_content_redaction_precedes_every_dispatch_sink() { + // Production code only. The unit tests now live in `wasm/wasm_test.rs`, so + // today this trims nothing — it is kept because an inlined `#[cfg(test)] mod + // tests` would otherwise call the helper and name these sinks, inflating the + // call count and offering false ordering evidence. + let prod = WASM_DISPATCH_SOURCE + .split_once("#[cfg(test)]\nmod tests") + .map_or(WASM_DISPATCH_SOURCE, |(prod, _)| prod); + let src = &strip_line_comments(prod); + + // Anchor on the *call*, not the comment above it: a comment can be reworded + // or deleted while the guard still runs, and — worse — can survive after the + // call it describes is removed, leaving this test green over a leak. + let calls = src.matches("redact_llm_content_params(").count() + - src.matches("fn redact_llm_content_params(").count(); + assert_eq!( + calls, 1, + "exactly one redaction call site is expected in dispatch; a second would \ + need its own ordering proof rather than riding on this one" + ); + let strip = src + .match_indices("redact_llm_content_params(") + .map(|(at, _)| at) + .find(|at| !src[..*at].ends_with("fn ")) + .expect("dispatch must redact LLM content before recording telemetry"); + + // Every sink below reads content from `result.callback_params`; *every* + // occurrence of each must come after the strip so it observes the redacted + // map. Iterating all occurrences (not just the first) catches a future + // dispatch branch that reads the params before the strip runs. + let sinks = [ + "let callback_params = &result.callback_params;", + "llm_call_wide_event(", + "submit_llmobs_llm_span(", + "submit_llmobs_tool_spans(", + ]; + for sink in sinks { + let mut from = 0; + let mut found = false; + while let Some(rel) = src[from..].find(sink) { + let at = from + rel; + found = true; + assert!( + strip < at, + "LLM content redaction (byte {strip}) must precede every read of sink \ + `{sink}`; found an occurrence at byte {at} before the strip" + ); + from = at + sink.len(); + } + assert!(found, "expected dispatch telemetry sink `{sink}`"); + } +} + +/// Companion to the ordering test: pin the *argument*, not just the call. A site +/// that redacts with a hardcoded `true`, or a host built with +/// `.with_llm_content_export(true)`, satisfies every ordering and unit test while +/// exporting content for every tenant. +#[test] +fn dispatch_redaction_and_host_wiring_use_the_per_tenant_policy() { + let prod = WASM_DISPATCH_SOURCE + .split_once("#[cfg(test)]\nmod tests") + .map_or(WASM_DISPATCH_SOURCE, |(prod, _)| prod); + let src = &strip_line_comments(prod); + + let redact_at = src + .match_indices("redact_llm_content_params(") + .map(|(at, _)| at) + .find(|at| !src[..*at].ends_with("fn ")) + .expect("dispatch must redact LLM content"); + let call = &src[redact_at..src.len().min(redact_at + 220)]; + assert!( + call.contains("self.export_llm_content("), + "the dispatch redaction must be driven by the per-tenant policy, not a \ + constant; found: {call:?}" + ); + + // Every host handed to the engine must take its export flag from the policy. + // Checked per site rather than by counting markers, so a multi-line + // `.with_llm_content_export(\n true,\n)` cannot pass as policy-driven. + let marker = ".with_llm_content_export("; + let mut sites = 0; + for (at, _) in src.match_indices(marker) { + sites += 1; + let arg_start = at + marker.len(); + let arg = &src[arg_start..src.len().min(arg_start + 120)]; + let arg = arg.split_once(')').map_or(arg, |(head, _)| head); + assert!( + arg.contains("export_llm_content("), + "`.with_llm_content_export` at byte {at} must be passed the per-tenant \ + policy, not a constant; found argument {arg:?}" + ); + } + assert!( + sites >= 3, + "expected the known host construction sites, found {sites}" + ); +} + +#[test] +fn comment_stripper_ignores_slashes_inside_string_literals() { + let src = "let url = \"http://example.com/a\"; // real comment\nlet keep = 1;\n"; + let stripped = strip_line_comments(src); + assert_eq!(stripped.len(), src.len(), "offsets must be preserved"); + assert!( + stripped.contains("http://example.com/a"), + "a URL in a string literal must survive: {stripped:?}" + ); + assert!( + !stripped.contains("real comment"), + "the comment must be blanked" + ); + assert!(stripped.contains("let keep = 1;")); +} diff --git a/crates/temper-spec/src/automaton/parser.rs b/crates/temper-spec/src/automaton/parser.rs index 9db406227..6b9fdc42c 100644 --- a/crates/temper-spec/src/automaton/parser.rs +++ b/crates/temper-spec/src/automaton/parser.rs @@ -539,7 +539,29 @@ fn validate(automaton: &Automaton) -> Result<(), AutomatonParseError> { ))); } - // 2. All `from` and `to` states in actions must be declared states. + // 2. Runtime-owned fields cannot also be mutable spec variables or action + // params. That would create a second identity/lifecycle/context truth. + for state_var in &automaton.state { + if super::types::is_server_derived_field_name(&state_var.name) { + return Err(AutomatonParseError::Validation(format!( + "state variable '{}' uses a runtime-owned field name", + state_var.name + ))); + } + } + for action in &automaton.actions { + for param in &action.params { + if super::types::is_server_derived_field_name(param.name()) { + return Err(AutomatonParseError::Validation(format!( + "action '{}' parameter '{}' uses a runtime-owned field name", + action.name, + param.name() + ))); + } + } + } + + // 3. All `from` and `to` states in actions must be declared states. for action in &automaton.actions { for from in &action.from { if !automaton.automaton.states.contains(from) { @@ -559,7 +581,7 @@ fn validate(automaton: &Automaton) -> Result<(), AutomatonParseError> { } } - // 3. Validate WASM integrations. + // 4. Validate WASM integrations. let action_names: Vec<&str> = automaton.actions.iter().map(|a| a.name.as_str()).collect(); for ig in &automaton.integrations { if ig.integration_type == "wasm" { @@ -588,7 +610,7 @@ fn validate(automaton: &Automaton) -> Result<(), AutomatonParseError> { } } - // 4. Validate [[state_timeout]] declarations (ADR-0049). + // 5. Validate [[state_timeout]] declarations (ADR-0049). // - `state` must be a declared state. // - `on_timeout` must be a declared action. // - each `reset_on` entry must be a declared action. @@ -637,7 +659,7 @@ fn validate(automaton: &Automaton) -> Result<(), AutomatonParseError> { } } - // 5. Validate allow_indefinite_states entries are declared states + // 6. Validate allow_indefinite_states entries are declared states // (ADR-0050 support). for state in &automaton.automaton.allow_indefinite_states { if !automaton.automaton.states.contains(state) { @@ -647,7 +669,7 @@ fn validate(automaton: &Automaton) -> Result<(), AutomatonParseError> { } } - // 6. Validate [[action.triggers]] declarations (ADR-0046). + // 7. Validate [[action.triggers]] declarations (ADR-0046). validate_action_triggers(automaton, &action_names)?; // 7. Validate [[vector]] access-path declarations (ADR-0155). diff --git a/crates/temper-spec/src/automaton/parser_core_test.rs b/crates/temper-spec/src/automaton/parser_core_test.rs index b8ae332b5..b7ef33ee2 100644 --- a/crates/temper-spec/src/automaton/parser_core_test.rs +++ b/crates/temper-spec/src/automaton/parser_core_test.rs @@ -146,3 +146,41 @@ to = "B" let result = parse_automaton(toml); assert!(result.is_err()); } + +#[test] +fn runtime_owned_state_field_is_rejected() { + let toml = r#" +[automaton] +name = "Bad" +states = ["Draft"] +initial = "Draft" + +[[state]] +name = "Status" +type = "string" +initial = "forged" +"#; + let error = parse_automaton(toml) + .expect_err("runtime-owned status cannot also be a mutable state variable"); + assert!(error.to_string().contains("runtime-owned field name")); +} + +#[test] +fn runtime_owned_action_param_is_rejected() { + let toml = r#" +[automaton] +name = "Bad" +states = ["Draft"] +initial = "Draft" + +[[action]] +name = "Create" +kind = "input" +from = ["Draft"] +to = "Draft" +params = ["Id", "ctx_owner_status"] +"#; + let error = parse_automaton(toml) + .expect_err("runtime-owned fields cannot be declared as action params"); + assert!(error.to_string().contains("parameter 'Id'")); +} diff --git a/crates/temper-spec/src/automaton/types.rs b/crates/temper-spec/src/automaton/types.rs index 34cf56049..966460bac 100644 --- a/crates/temper-spec/src/automaton/types.rs +++ b/crates/temper-spec/src/automaton/types.rs @@ -9,6 +9,23 @@ use std::collections::BTreeMap; use super::field_invariant::FieldInvariant; +/// Return whether a field name is owned by the runtime rather than an action. +/// +/// Entity identity, lifecycle status, spec-governance metadata, and declared +/// context statuses are derived from server-proven state. Specs and callers +/// must not create a second mutable representation of these values. +pub fn is_server_derived_field_name(name: &str) -> bool { + matches!( + name, + "Id" | "id" | "Status" | "status" | "has_spec" | "HasSpec" + ) || is_server_derived_context_status_name(name) +} + +/// Return whether a field is in the server-derived context-status namespace. +pub fn is_server_derived_context_status_name(name: &str) -> bool { + name.starts_with("ctx_") && name.ends_with("_status") +} + /// A complete I/O Automaton specification for a single entity type. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Automaton { diff --git a/crates/temper-spec/src/csdl/emit.rs b/crates/temper-spec/src/csdl/emit.rs index 1ea494e9e..a5a2d69ae 100644 --- a/crates/temper-spec/src/csdl/emit.rs +++ b/crates/temper-spec/src/csdl/emit.rs @@ -10,7 +10,7 @@ pub fn emit_csdl_xml(doc: &CsdlDocument) -> String { let mut out = String::from("\n"); out.push_str(&format!( "\n", - doc.version + xml_escape(&doc.version) )); out.push_str(" \n"); @@ -26,7 +26,7 @@ pub fn emit_csdl_xml(doc: &CsdlDocument) -> String { fn emit_schema(out: &mut String, schema: &Schema) { out.push_str(&format!( " \n", - schema.namespace + xml_escape(&schema.namespace) )); for term in &schema.terms { @@ -54,10 +54,11 @@ fn emit_schema(out: &mut String, schema: &Schema) { fn emit_term(out: &mut String, term: &Term) { out.push_str(&format!( " \n", et.name)); + out.push_str(&format!( + " \n", + xml_escape(&et.name) + )); for member in &et.members { if let Some(val) = member.value { out.push_str(&format!( " \n", - member.name, val + xml_escape(&member.name), + val )); } else { - out.push_str(&format!(" \n", member.name)); + out.push_str(&format!( + " \n", + xml_escape(&member.name) + )); } } out.push_str(" \n"); @@ -84,17 +92,23 @@ fn emit_entity_type(out: &mut String, et: &EntityType) { if et.has_stream { out.push_str(&format!( " \n", - et.name + xml_escape(&et.name) )); } else { - out.push_str(&format!(" \n", et.name)); + out.push_str(&format!( + " \n", + xml_escape(&et.name) + )); } // Key if !et.key_properties.is_empty() { out.push_str(" \n"); for key in &et.key_properties { - out.push_str(&format!(" \n")); + out.push_str(&format!( + " \n", + xml_escape(key) + )); } out.push_str(" \n"); } @@ -120,13 +134,14 @@ fn emit_entity_type(out: &mut String, et: &EntityType) { fn emit_property(out: &mut String, prop: &Property) { out.push_str(&format!( " \n", - rc.property, rc.referenced_property + xml_escape(&rc.property), + xml_escape(&rc.referenced_property) )); } out.push_str(" \n"); @@ -170,7 +187,10 @@ fn emit_action(out: &mut String, action: &Action) { || action.return_type.is_some() || !action.annotations.is_empty(); - out.push_str(&format!(" \n"); } fn emit_return_type(out: &mut String, rt: &ReturnType) { - out.push_str(&format!(" \n", - ai.name, ai.action + xml_escape(&ai.name), + xml_escape(&ai.action) )); } for fi in &container.function_imports { out.push_str(&format!( " \n", - fi.name, fi.function + xml_escape(&fi.name), + xml_escape(&fi.function) )); } out.push_str(" \n"); @@ -283,17 +312,20 @@ fn emit_entity_set(out: &mut String, es: &EntitySet) { if es.navigation_bindings.is_empty() { out.push_str(&format!( " \n", - es.name, es.entity_type + xml_escape(&es.name), + xml_escape(&es.entity_type) )); } else { out.push_str(&format!( " \n", - es.name, es.entity_type + xml_escape(&es.name), + xml_escape(&es.entity_type) )); for nb in &es.navigation_bindings { out.push_str(&format!( " \n", - nb.path, nb.target + xml_escape(&nb.path), + xml_escape(&nb.target) )); } out.push_str(" \n"); @@ -302,34 +334,35 @@ fn emit_entity_set(out: &mut String, es: &EntitySet) { fn emit_annotation(out: &mut String, ann: &Annotation, indent: usize) { let pad: String = " ".repeat(indent); + let term = xml_escape(&ann.term); match &ann.value { AnnotationValue::String(s) => { out.push_str(&format!( "{pad}\n", - ann.term, + term, xml_escape(s) )); } AnnotationValue::Float(f) => { out.push_str(&format!( "{pad}\n", - ann.term + term )); } AnnotationValue::Bool(b) => { out.push_str(&format!( "{pad}\n", - ann.term + term )); } AnnotationValue::Int(i) => { out.push_str(&format!( "{pad}\n", - ann.term + term )); } AnnotationValue::Collection(items) => { - out.push_str(&format!("{pad}\n", ann.term)); + out.push_str(&format!("{pad}\n")); out.push_str(&format!("{pad} \n")); for item in items { out.push_str(&format!("{pad} {}\n", xml_escape(item))); @@ -338,11 +371,12 @@ fn emit_annotation(out: &mut String, ann: &Annotation, indent: usize) { out.push_str(&format!("{pad}\n")); } AnnotationValue::Record(map) => { - out.push_str(&format!("{pad}\n", ann.term)); + out.push_str(&format!("{pad}\n")); out.push_str(&format!("{pad} \n")); for (k, v) in map { out.push_str(&format!( - "{pad} \n", + "{pad} \n", + xml_escape(k), xml_escape(v) )); } @@ -353,113 +387,33 @@ fn emit_annotation(out: &mut String, ann: &Annotation, indent: usize) { } /// Escape XML special characters in attribute/text values. +/// +/// Every value interpolated into emitted CSDL must pass through this function — +/// identifiers included. Names, types, and references are agent- or +/// user-influenced, so an unescaped `"` there closes the attribute and lets the +/// value inject arbitrary markup. +/// +/// Tab, newline, and carriage return are escaped as character references +/// because XML attribute-value normalization would otherwise replace them with +/// spaces, silently changing the value on the way back in. fn xml_escape(s: &str) -> String { - s.replace('&', "&") - .replace('<', "<") - .replace('>', ">") - .replace('"', """) - .replace('\'', "'") -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::csdl::parse_csdl; - - #[test] - fn emit_round_trips_minimal_csdl() { - let xml = r#" - - - - - - - - - - - - - - "#; - - let doc = parse_csdl(xml).unwrap(); - let emitted = emit_csdl_xml(&doc); - - // Parse the emitted XML back and verify structure is preserved. - let doc2 = parse_csdl(&emitted).expect("emitted XML should re-parse"); - assert_eq!(doc2.version, "4.0"); - assert_eq!(doc2.schemas.len(), 1); - let schema = &doc2.schemas[0]; - assert_eq!(schema.namespace, "Test"); - assert_eq!(schema.entity_types.len(), 1); - assert_eq!(schema.entity_types[0].name, "Widget"); - assert_eq!(schema.entity_types[0].key_properties, vec!["Id"]); - assert_eq!(schema.entity_types[0].properties.len(), 2); - assert_eq!(schema.entity_containers.len(), 1); - assert_eq!(schema.entity_containers[0].entity_sets.len(), 1); - assert_eq!( - schema.entity_containers[0].entity_sets[0].entity_type, - "Test.Widget" - ); - } - - #[test] - fn emit_round_trips_has_stream() { - let xml = r#" - - - - - - - - - - - - - - - "#; - - let doc = parse_csdl(xml).unwrap(); - let schema = &doc.schemas[0]; - - let media = schema.entity_type("MediaFile").unwrap(); - assert!(media.has_stream, "MediaFile should have has_stream=true"); - - let regular = schema.entity_type("RegularEntity").unwrap(); - assert!( - !regular.has_stream, - "RegularEntity should have has_stream=false" - ); - - // Round-trip - let emitted = emit_csdl_xml(&doc); - let doc2 = parse_csdl(&emitted).unwrap(); - let schema2 = &doc2.schemas[0]; - - assert!(schema2.entity_type("MediaFile").unwrap().has_stream); - assert!(!schema2.entity_type("RegularEntity").unwrap().has_stream); - } - - #[test] - fn emit_round_trips_reference_csdl() { - let xml = include_str!("../../../../test-fixtures/specs/model.csdl.xml"); - let doc = parse_csdl(xml).unwrap(); - let emitted = emit_csdl_xml(&doc); - - let doc2 = parse_csdl(&emitted).expect("emitted reference CSDL should re-parse"); - assert_eq!(doc2.schemas.len(), doc.schemas.len()); - - // Verify entity types are preserved. - for (s1, s2) in doc.schemas.iter().zip(doc2.schemas.iter()) { - assert_eq!(s1.namespace, s2.namespace); - assert_eq!(s1.entity_types.len(), s2.entity_types.len()); - assert_eq!(s1.actions.len(), s2.actions.len()); - assert_eq!(s1.entity_containers.len(), s2.entity_containers.len()); + let mut out = String::with_capacity(s.len()); + for ch in s.chars() { + match ch { + '&' => out.push_str("&"), + '<' => out.push_str("<"), + '>' => out.push_str(">"), + '"' => out.push_str("""), + '\'' => out.push_str("'"), + '\t' => out.push_str(" "), + '\n' => out.push_str(" "), + '\r' => out.push_str(" "), + _ => out.push(ch), } } + out } + +#[cfg(test)] +#[path = "emit_test.rs"] +mod tests; diff --git a/crates/temper-spec/src/csdl/emit_test.rs b/crates/temper-spec/src/csdl/emit_test.rs new file mode 100644 index 000000000..1880fd3a2 --- /dev/null +++ b/crates/temper-spec/src/csdl/emit_test.rs @@ -0,0 +1,261 @@ +use super::*; +use crate::csdl::parse_csdl; + +#[test] +fn emit_round_trips_minimal_csdl() { + let xml = r#" + + + + + + + + + + + + + + "#; + + let doc = parse_csdl(xml).unwrap(); + let emitted = emit_csdl_xml(&doc); + + // Parse the emitted XML back and verify structure is preserved. + let doc2 = parse_csdl(&emitted).expect("emitted XML should re-parse"); + assert_eq!(doc2.version, "4.0"); + assert_eq!(doc2.schemas.len(), 1); + let schema = &doc2.schemas[0]; + assert_eq!(schema.namespace, "Test"); + assert_eq!(schema.entity_types.len(), 1); + assert_eq!(schema.entity_types[0].name, "Widget"); + assert_eq!(schema.entity_types[0].key_properties, vec!["Id"]); + assert_eq!(schema.entity_types[0].properties.len(), 2); + assert_eq!(schema.entity_containers.len(), 1); + assert_eq!(schema.entity_containers[0].entity_sets.len(), 1); + assert_eq!( + schema.entity_containers[0].entity_sets[0].entity_type, + "Test.Widget" + ); +} + +#[test] +fn emit_round_trips_has_stream() { + let xml = r#" + + + + + + + + + + + + + + + "#; + + let doc = parse_csdl(xml).unwrap(); + let schema = &doc.schemas[0]; + + let media = schema.entity_type("MediaFile").unwrap(); + assert!(media.has_stream, "MediaFile should have has_stream=true"); + + let regular = schema.entity_type("RegularEntity").unwrap(); + assert!( + !regular.has_stream, + "RegularEntity should have has_stream=false" + ); + + // Round-trip + let emitted = emit_csdl_xml(&doc); + let doc2 = parse_csdl(&emitted).unwrap(); + let schema2 = &doc2.schemas[0]; + + assert!(schema2.entity_type("MediaFile").unwrap().has_stream); + assert!(!schema2.entity_type("RegularEntity").unwrap().has_stream); +} + +/// A property name carrying a quote must not be able to close the attribute +/// and inject markup of its own. +#[test] +fn adversarial_identifiers_do_not_inject_markup() { + let doc = CsdlDocument { + version: "4.0".to_string(), + schemas: vec![Schema { + namespace: "Ns\">".to_string()], + properties: vec![Property { + name: "Name\"/>".to_string()), + }], + has_stream: false, + }], + enum_types: Vec::new(), + actions: Vec::new(), + functions: Vec::new(), + entity_containers: vec![EntityContainer { + name: "Svc\">".to_string(), + entity_type: "Test.Widget\">".to_string(), + navigation_bindings: vec![NavigationBinding { + path: "Path\">".to_string(), + target: "Target\">".to_string(), + }], + }], + action_imports: vec![ActionImport { + name: "DoIt\">".to_string(), + action: "Test.DoIt\">".to_string(), + }], + function_imports: vec![FunctionImport { + name: "GetIt\">".to_string(), + function: "Test.GetIt\">".to_string(), + }], + }], + terms: Vec::new(), + }], + }; + + let emitted = emit_csdl_xml(&doc); + // The adversarial substrings may legitimately appear *escaped* inside an + // attribute value; what must never appear is live markup. + assert!( + !emitted.contains(""), + "namespace escaped its attribute:\n{emitted}" + ); + assert!( + !emitted.contains(" { - let text = reader.read_text(element.name()).unwrap_or_default(); + let text = reader + .read_text(element.name()) + .ok() + .and_then(|t| t.xml10_content().ok().map(|s| s.into_owned())) + .unwrap_or_default(); let text = text.trim().to_string(); if !text.is_empty() { collection_items.push(text); diff --git a/crates/temper-spec/src/csdl/parser/xml.rs b/crates/temper-spec/src/csdl/parser/xml.rs index b8dc630f4..9d123610e 100644 --- a/crates/temper-spec/src/csdl/parser/xml.rs +++ b/crates/temper-spec/src/csdl/parser/xml.rs @@ -38,12 +38,22 @@ pub(super) fn local_name_end(element: &BytesEnd) -> String { full.rsplit(':').next().unwrap_or(full).to_string() } +/// Read an attribute, decoding XML entity and character references. +/// +/// `Attribute::value` is the raw, still-escaped bytes. Returning those directly +/// would surface `&` and ` ` as literal text and make parse/emit/parse +/// double-escape, so the value is unescaped here. pub(super) fn attr_str(element: &BytesStart, name: &str) -> Option { element .attributes() .flatten() .find(|attribute| std::str::from_utf8(attribute.key.as_ref()).unwrap_or("") == name) - .and_then(|attribute| String::from_utf8(attribute.value.to_vec()).ok()) + .and_then(|attribute| { + attribute + .normalized_value(quick_xml::XmlVersion::Implicit1_0) + .ok() + }) + .map(|value| value.into_owned()) } pub(super) fn required_attr(element: &BytesStart, name: &str) -> Result { diff --git a/crates/temper-store-postgres/migrations/0012_evolution_tenant_ownership.sql b/crates/temper-store-postgres/migrations/0012_evolution_tenant_ownership.sql new file mode 100644 index 000000000..fced43e83 --- /dev/null +++ b/crates/temper-store-postgres/migrations/0012_evolution_tenant_ownership.sql @@ -0,0 +1,17 @@ +-- Evolution data is tenant-owned at the storage boundary. Existing rows +-- predate explicit ownership and remain assigned to the historical default +-- tenant; every new read and write supplies a tenant predicate. +ALTER TABLE feature_requests + ADD COLUMN IF NOT EXISTS tenant TEXT NOT NULL DEFAULT 'default'; + +ALTER TABLE evolution_records + ADD COLUMN IF NOT EXISTS tenant TEXT NOT NULL DEFAULT 'default'; + +CREATE INDEX IF NOT EXISTS idx_feature_requests_tenant_disposition + ON feature_requests (tenant, disposition, frequency DESC, created_at DESC); + +CREATE INDEX IF NOT EXISTS idx_evolution_records_tenant_type_status + ON evolution_records (tenant, record_type, status, timestamp DESC); + +CREATE INDEX IF NOT EXISTS idx_evolution_records_tenant_parent + ON evolution_records (tenant, derived_from); diff --git a/crates/temper-store-postgres/migrations/0013_trajectory_session_index.sql b/crates/temper-store-postgres/migrations/0013_trajectory_session_index.sql new file mode 100644 index 000000000..562e7ba42 --- /dev/null +++ b/crates/temper-store-postgres/migrations/0013_trajectory_session_index.sql @@ -0,0 +1,6 @@ +-- Session-scoped trajectory replay (conformance checking). +-- +-- Conformance reads one session's rows in the order the kernel wrote them, +-- so the index covers both the filter column and the ordering columns. +CREATE INDEX IF NOT EXISTS idx_trajectories_session + ON trajectories (session_id, created_at, id); diff --git a/crates/temper-store-postgres/migrations/0014_trajectory_capture_seq.sql b/crates/temper-store-postgres/migrations/0014_trajectory_capture_seq.sql new file mode 100644 index 000000000..3ea03a2ee --- /dev/null +++ b/crates/temper-store-postgres/migrations/0014_trajectory_capture_seq.sql @@ -0,0 +1,17 @@ +-- Capture order within the writing process. +-- +-- Rows are written by independently spawned persistence tasks, so the +-- BIGSERIAL id records the order the writes landed, not the order the kernel +-- captured them. Conformance checking replays a session as a state-machine +-- walk and needs capture order, so the capturing process stamps a monotonic +-- sequence on the entry before it is queued and the session read orders by it. +-- Null on rows written before this column existed. +ALTER TABLE trajectories + ADD COLUMN IF NOT EXISTS capture_seq BIGINT; + +-- Session-scoped trajectory replay, covering every ordering column so the +-- read is an index scan. Replaces idx_trajectories_session from 0013, which +-- ordered by id. +CREATE INDEX IF NOT EXISTS idx_trajectories_session_capture + ON trajectories (session_id, created_at, capture_seq, id); +DROP INDEX IF EXISTS idx_trajectories_session; diff --git a/crates/temper-store-postgres/migrations/0015_ots_trajectory_tenant_identity.sql b/crates/temper-store-postgres/migrations/0015_ots_trajectory_tenant_identity.sql new file mode 100644 index 000000000..51fdc117e --- /dev/null +++ b/crates/temper-store-postgres/migrations/0015_ots_trajectory_tenant_identity.sql @@ -0,0 +1,27 @@ +-- Tenant-scoped OTS trajectory identity. +-- +-- The table was keyed on trajectory_id alone. That id is chosen by the +-- uploading harness and one database holds every tenant's rows, so two tenants +-- could collide on one id — and the upsert resolved the collision by rewriting +-- the existing row, tenant column included, handing one tenant's trajectory to +-- another. Reads were already tenant-scoped; the identity has to be too. +-- +-- No row can violate the new key: the old one was global, so (tenant, +-- trajectory_id) is unique wherever trajectory_id was. + +ALTER TABLE ots_trajectories + DROP CONSTRAINT IF EXISTS ots_trajectories_pkey; + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'ots_trajectories_tenant_identity' + AND conrelid = 'ots_trajectories'::regclass + ) THEN + ALTER TABLE ots_trajectories + ADD CONSTRAINT ots_trajectories_tenant_identity + PRIMARY KEY (tenant, trajectory_id); + END IF; +END +$$; diff --git a/crates/temper-store-postgres/src/lib.rs b/crates/temper-store-postgres/src/lib.rs index af25937ce..15e9f3033 100644 --- a/crates/temper-store-postgres/src/lib.rs +++ b/crates/temper-store-postgres/src/lib.rs @@ -27,8 +27,9 @@ pub mod store; pub use metrics::init_metrics; pub use platform::{ PostgresActionStats, PostgresAgentSummary, PostgresDesignTimeEventRow, - PostgresEvolutionRecordRow, PostgresFeatureRequestRow, PostgresInstalledAppRow, - PostgresOtsTrajectoryParams, PostgresOtsTrajectoryRow, PostgresPolicyDenialPatternRow, + PostgresEvolutionRecordInsert, PostgresEvolutionRecordRow, PostgresFeatureRequestRow, + PostgresInstalledAppRow, PostgresOtsTrajectoryDocument, PostgresOtsTrajectoryParams, + PostgresOtsTrajectoryRow, PostgresPolicyApprovalCommit, PostgresPolicyDenialPatternRow, PostgresPolicyRow, PostgresProjectedEntityFieldsRow, PostgresPublishedArtifactRow, PostgresPublishedArtifactUpsert, PostgresQueuedOtsTrajectoryRow, PostgresSecretRow, PostgresSpecRow, PostgresSpecVerificationUpdate, PostgresTrajectoryInsert, diff --git a/crates/temper-store-postgres/src/migration.rs b/crates/temper-store-postgres/src/migration.rs index 5a40c7493..05f4fec98 100644 --- a/crates/temper-store-postgres/src/migration.rs +++ b/crates/temper-store-postgres/src/migration.rs @@ -37,6 +37,10 @@ mod tests { include_str!("../migrations/0006_segmented_event_history.sql"), include_str!("../migrations/0007_installed_app_follow_policy.sql"), include_str!("../migrations/0008_ots_trajectory_outbox_status.sql"), + include_str!("../migrations/0009_entity_key_index.sql"), + include_str!("../migrations/0010_key_index_backfill_watermark.sql"), + include_str!("../migrations/0011_key_index_watermark_key_set.sql"), + include_str!("../migrations/0012_evolution_tenant_ownership.sql"), ] .join("\n") .to_lowercase(); @@ -103,6 +107,51 @@ mod tests { ); } + #[test] + fn capture_order_column_exists_on_both_schema_paths() { + // A fresh bootstrap reads schema.rs and an existing database reads the + // migration; a session read that orders by `capture_seq` fails on + // whichever path forgets it. + let migration = include_str!("../migrations/0014_trajectory_capture_seq.sql"); + assert!( + migration.contains("ADD COLUMN IF NOT EXISTS capture_seq"), + "migration 0014 must add the capture-order column idempotently" + ); + assert!( + migration.contains("idx_trajectories_session_capture"), + "migration 0014 must cover the session read's ordering columns" + ); + assert!( + schema::CREATE_TRAJECTORIES_TABLE.contains("capture_seq"), + "a freshly bootstrapped trajectories table must carry the capture-order column" + ); + } + + #[test] + fn ots_trajectory_identity_is_tenant_scoped_on_both_schema_paths() { + // A fresh bootstrap reads schema.rs and an existing database reads the + // migration. Whichever path keeps the global key lets one tenant's + // upload overwrite another tenant's trajectory. + let migration = include_str!("../migrations/0015_ots_trajectory_tenant_identity.sql"); + assert!( + migration.contains("DROP CONSTRAINT IF EXISTS ots_trajectories_pkey"), + "migration 0015 must drop the globally keyed primary key idempotently" + ); + assert!( + migration.contains("PRIMARY KEY (tenant, trajectory_id)"), + "migration 0015 must rekey the table by (tenant, trajectory_id)" + ); + assert!( + schema::CREATE_OTS_TRAJECTORIES_TABLE.contains("PRIMARY KEY (tenant, trajectory_id)"), + "a freshly bootstrapped ots_trajectories table must be keyed by tenant and id" + ); + assert!( + !schema::CREATE_OTS_TRAJECTORIES_TABLE + .contains("trajectory_id TEXT PRIMARY KEY"), + "the globally keyed column definition must be gone, not shadowed" + ); + } + #[test] fn migration_sql_is_idempotent() { // Both schemas must use IF NOT EXISTS so repeated execution is safe. diff --git a/crates/temper-store-postgres/src/platform.rs b/crates/temper-store-postgres/src/platform.rs index 322e61fe9..432c0b415 100644 --- a/crates/temper-store-postgres/src/platform.rs +++ b/crates/temper-store-postgres/src/platform.rs @@ -14,7 +14,11 @@ use crate::metrics::{ record_postgres_transaction_begin_duration, record_postgres_transaction_commit_duration, }; +mod evolution; +mod inputs; +mod policy_approval; mod rows; +pub use inputs::{PostgresEvolutionRecordInsert, PostgresPolicyApprovalCommit}; use rows::*; const DISTINCT_RESOURCE_IDS_BUDGET: usize = 100; @@ -307,6 +311,9 @@ pub struct PostgresTrajectoryRow { pub request_body: Option, pub intent: Option, pub matched_policy_ids: Option>, + /// Monotonic capture order stamped by the process that recorded the row. + /// Null on rows written before the column existed. + pub capture_seq: Option, } #[derive(Debug, Clone, serde::Serialize)] @@ -350,6 +357,7 @@ pub struct PostgresUnmetIntentAggRow { #[derive(Debug, Clone, serde::Serialize)] pub struct PostgresFeatureRequestRow { pub id: String, + pub tenant: String, pub category: String, pub description: String, pub frequency: i64, @@ -363,6 +371,7 @@ pub struct PostgresFeatureRequestRow { #[derive(Debug, Clone, serde::Serialize)] pub struct PostgresEvolutionRecordRow { pub id: String, + pub tenant: String, pub record_type: String, pub status: String, pub created_by: String, @@ -400,6 +409,18 @@ pub struct PostgresOtsTrajectoryRow { pub updated_at: String, } +/// A stored OTS trajectory document together with the run identity recorded +/// alongside it. +#[derive(Debug, Clone, serde::Serialize)] +pub struct PostgresOtsTrajectoryDocument { + pub trajectory_id: String, + pub tenant: String, + pub agent_id: String, + pub session_id: String, + pub outcome: String, + pub data: String, +} + #[derive(Debug, Clone, serde::Serialize)] pub struct PostgresQueuedOtsTrajectoryRow { pub trajectory_id: String, @@ -478,6 +499,10 @@ pub struct PostgresTrajectoryInsert<'a> { pub request_body: Option<&'a str>, pub intent: Option<&'a str>, pub matched_policy_ids: Option<&'a str>, + /// Monotonic capture order stamped by the recording process, so a session + /// reads back in the order the kernel captured it rather than the order + /// independent persistence tasks happened to land. + pub capture_seq: Option, } impl PostgresEventStore { @@ -492,8 +517,8 @@ impl PostgresEventStore { "INSERT INTO trajectories \ (tenant, entity_type, entity_id, action, success, from_status, to_status, error, \ agent_id, session_id, authz_denied, denied_resource, denied_module, source, \ - spec_governed, created_at, request_body, intent, matched_policy_ids) \ - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19)", + spec_governed, created_at, request_body, intent, matched_policy_ids, capture_seq) \ + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20)", ) .bind(entry.tenant) .bind(entry.entity_type) @@ -514,6 +539,7 @@ impl PostgresEventStore { .bind(request_body) .bind(entry.intent) .bind(matched_policy_ids) + .bind(entry.capture_seq) .execute(self.pool()) .await .map_err(storage_error)?; @@ -1468,11 +1494,12 @@ impl PostgresEventStore { data: &str, ) -> Result<(), PersistenceError> { let data = parse_json(data)?; - crate::dbm::postgres_query!( + let result = crate::dbm::postgres_query!( "INSERT INTO pending_decisions (id, tenant, status, data, created_at, updated_at) \ VALUES ($1, $2, $3, $4, now(), now()) \ - ON CONFLICT (id) DO UPDATE SET tenant = EXCLUDED.tenant, status = EXCLUDED.status, \ - data = EXCLUDED.data, updated_at = now()", + ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, \ + data = EXCLUDED.data, updated_at = now() \ + WHERE pending_decisions.tenant = EXCLUDED.tenant", ) .bind(id) .bind(tenant) @@ -1481,6 +1508,11 @@ impl PostgresEventStore { .execute(self.pool()) .await .map_err(storage_error)?; + if result.rows_affected() != 1 { + return Err(PersistenceError::Storage(format!( + "pending decision '{id}' is owned by another tenant" + ))); + } Ok(()) } @@ -1498,6 +1530,30 @@ impl PostgresEventStore { Ok(rows.into_iter().map(|v| v.to_string()).collect()) } + /// Load approved decisions whose scope names `session_id`, for one tenant. + /// + /// Backs session-grant validation (ADR-0157): a caller-asserted session id + /// becomes a Cedar input only when an approved decision binds that session, + /// so the lookup filters on the approved scope's session, not the denial's. + pub async fn load_approved_session_decisions( + &self, + tenant: &str, + session_id: &str, + ) -> Result, PersistenceError> { + let rows: Vec = crate::dbm::postgres_query_scalar!( + "SELECT data FROM pending_decisions \ + WHERE tenant = $1 \ + AND status = 'approved' \ + AND data->'approved_scope'->>'session_id' = $2", + ) + .bind(tenant) + .bind(session_id) + .fetch_all(self.pool()) + .await + .map_err(storage_error)?; + Ok(rows.into_iter().map(|v| v.to_string()).collect()) + } + pub async fn load_all_wasm_modules( &self, tenant: &str, @@ -1580,16 +1636,19 @@ impl PostgresEventStore { impl PostgresEventStore { pub async fn load_recent_trajectories( &self, + tenant: &str, limit: i64, ) -> Result, PersistenceError> { let rows = crate::dbm::postgres_query!( "SELECT tenant, entity_type, entity_id, action, success, from_status, to_status, error, \ agent_id, session_id, authz_denied, denied_resource, denied_module, source, spec_governed, \ - created_at, request_body, intent, matched_policy_ids \ + created_at, request_body, intent, matched_policy_ids, capture_seq \ FROM trajectories \ + WHERE tenant = $1 \ ORDER BY created_at DESC \ - LIMIT $1", + LIMIT $2", ) + .bind(tenant) .bind(limit) .fetch_all(self.pool()) .await @@ -1599,16 +1658,19 @@ impl PostgresEventStore { pub async fn load_unmet_intent_rows( &self, + tenant: &str, ) -> Result, PersistenceError> { let rows = crate::dbm::postgres_query!( "SELECT entity_type, MAX(action) AS action, error, COUNT(*)::bigint AS cnt, \ MIN(created_at) AS first_seen, MAX(created_at) AS last_seen \ FROM trajectories \ - WHERE success = false AND (authz_denied IS NULL OR authz_denied = false) \ + WHERE tenant = $1 \ + AND success = false AND (authz_denied IS NULL OR authz_denied = false) \ GROUP BY entity_type, error \ ORDER BY cnt DESC \ LIMIT 100", ) + .bind(tenant) .fetch_all(self.pool()) .await .map_err(storage_error)?; @@ -1617,13 +1679,15 @@ impl PostgresEventStore { pub async fn load_submit_spec_timestamps( &self, + tenant: &str, ) -> Result, PersistenceError> { let rows: Vec<(String, chrono::DateTime)> = crate::dbm::postgres_query_as!( "SELECT entity_type, MAX(created_at) AS latest_at \ FROM trajectories \ - WHERE success = true AND action = 'SubmitSpec' \ + WHERE tenant = $1 AND success = true AND action = 'SubmitSpec' \ GROUP BY entity_type", ) + .bind(tenant) .fetch_all(self.pool()) .await .map_err(storage_error)?; @@ -1648,8 +1712,14 @@ impl PostgresEventStore { .collect()) } + /// Trajectory statistics for one tenant. + /// + /// Scoped to `tenant` in SQL: the failed-intent list returns whole rows — + /// error strings, entity ids — so an unscoped read would hand one tenant + /// another's operational detail (ADR-0157). pub async fn query_trajectory_stats( &self, + tenant: &str, entity_type: Option<&str>, action: Option<&str>, success_filter: Option, @@ -1659,10 +1729,12 @@ impl PostgresEventStore { "SELECT COUNT(*)::bigint AS total, \ COALESCE(SUM(CASE WHEN success = true THEN 1 ELSE 0 END), 0)::bigint AS success_count \ FROM trajectories \ - WHERE ($1::text IS NULL OR entity_type = $1) \ - AND ($2::text IS NULL OR action = $2) \ - AND ($3::boolean IS NULL OR success = $3)", + WHERE tenant = $1 \ + AND ($2::text IS NULL OR entity_type = $2) \ + AND ($3::text IS NULL OR action = $3) \ + AND ($4::boolean IS NULL OR success = $4)", ) + .bind(tenant) .bind(entity_type) .bind(action) .bind(success_filter) @@ -1677,8 +1749,10 @@ impl PostgresEventStore { COALESCE(SUM(CASE WHEN success = true THEN 1 ELSE 0 END), 0)::bigint AS success, \ COALESCE(SUM(CASE WHEN success = false THEN 1 ELSE 0 END), 0)::bigint AS error \ FROM trajectories \ + WHERE tenant = $1 \ GROUP BY action", ) + .bind(tenant) .fetch_all(self.pool()) .await .map_err(storage_error)?; @@ -1699,12 +1773,14 @@ impl PostgresEventStore { let failed_rows = crate::dbm::postgres_query!( "SELECT tenant, entity_type, entity_id, action, success, from_status, to_status, error, \ agent_id, session_id, authz_denied, denied_resource, denied_module, source, spec_governed, \ - created_at, request_body, intent, matched_policy_ids \ + created_at, request_body, intent, matched_policy_ids, capture_seq \ FROM trajectories \ - WHERE success = false \ + WHERE tenant = $1 \ + AND success = false \ ORDER BY created_at DESC \ - LIMIT $1", + LIMIT $2", ) + .bind(tenant) .bind(failed_limit) .fetch_all(self.pool()) .await @@ -1735,7 +1811,7 @@ impl PostgresEventStore { let rows = crate::dbm::postgres_query!( "SELECT tenant, entity_type, entity_id, action, success, from_status, to_status, error, \ agent_id, session_id, authz_denied, denied_resource, denied_module, source, spec_governed, \ - created_at, request_body, intent, matched_policy_ids \ + created_at, request_body, intent, matched_policy_ids, capture_seq \ FROM trajectories \ WHERE agent_id = $1 \ AND ($2::text IS NULL OR tenant = $2) \ @@ -1753,6 +1829,48 @@ impl PostgresEventStore { Ok(rows.into_iter().map(row_to_trajectory).collect()) } + /// Query one session's trajectory rows in the order the kernel wrote them. + /// + /// Ordered ascending — oldest first — because the conformance checker + /// replays a session as a state-machine run and a newest-first read would + /// hand it the run backwards. + /// + /// Ties inside one `created_at` tick are broken by `capture_seq`, the + /// order the capturing process stamped on the entry, and only then by + /// `id`. `id` alone is the order the writes landed: rows are persisted by + /// independently spawned tasks, so two entries captured in one tick can be + /// inserted in either order and a denial/retry pair would be replayed + /// backwards. `COALESCE` sorts rows written before the column existed + /// first and identically on both backends, rather than leaving it to each + /// engine's NULL ordering. + pub async fn query_trajectories_by_session( + &self, + session_id: &str, + tenant: Option<&str>, + entity_type: Option<&str>, + limit: i64, + ) -> Result, PersistenceError> { + let rows = crate::dbm::postgres_query!( + "SELECT tenant, entity_type, entity_id, action, success, from_status, to_status, error, \ + agent_id, session_id, authz_denied, denied_resource, denied_module, source, spec_governed, \ + created_at, request_body, intent, matched_policy_ids, capture_seq \ + FROM trajectories \ + WHERE session_id = $1 \ + AND ($2::text IS NULL OR tenant = $2) \ + AND ($3::text IS NULL OR entity_type = $3) \ + ORDER BY created_at ASC, COALESCE(capture_seq, 0) ASC, id ASC \ + LIMIT $4", + ) + .bind(session_id) + .bind(tenant) + .bind(entity_type) + .bind(limit) + .fetch_all(self.pool()) + .await + .map_err(storage_error)?; + Ok(rows.into_iter().map(row_to_trajectory).collect()) + } + pub async fn query_agent_summaries( &self, tenant: Option<&str>, @@ -1775,157 +1893,6 @@ impl PostgresEventStore { Ok(rows.into_iter().map(row_to_agent_summary).collect()) } - #[allow(clippy::too_many_arguments)] - pub async fn upsert_feature_request( - &self, - id: &str, - category: &str, - description: &str, - frequency: i64, - trajectory_refs_json: &str, - disposition: &str, - developer_notes: Option<&str>, - ) -> Result<(), PersistenceError> { - let trajectory_refs = parse_json(trajectory_refs_json)?; - crate::dbm::postgres_query!( - "INSERT INTO feature_requests \ - (id, category, description, frequency, trajectory_refs, disposition, developer_notes, updated_at) \ - VALUES ($1, $2, $3, $4, $5, $6, $7, now()) \ - ON CONFLICT (id) DO UPDATE SET \ - category = EXCLUDED.category, description = EXCLUDED.description, frequency = EXCLUDED.frequency, \ - trajectory_refs = EXCLUDED.trajectory_refs, disposition = EXCLUDED.disposition, \ - developer_notes = EXCLUDED.developer_notes, updated_at = now()", - ) - .bind(id) - .bind(category) - .bind(description) - .bind(frequency) - .bind(trajectory_refs) - .bind(disposition) - .bind(developer_notes) - .execute(self.pool()) - .await - .map_err(storage_error)?; - Ok(()) - } - - pub async fn list_feature_requests( - &self, - disposition: Option<&str>, - ) -> Result, PersistenceError> { - let rows = crate::dbm::postgres_query!( - "SELECT id, category, description, frequency, trajectory_refs, disposition, developer_notes, created_at, updated_at \ - FROM feature_requests \ - WHERE ($1::text IS NULL OR disposition = $1) \ - ORDER BY frequency DESC, created_at DESC", - ) - .bind(disposition) - .fetch_all(self.pool()) - .await - .map_err(storage_error)?; - Ok(rows.into_iter().map(row_to_feature_request).collect()) - } - - pub async fn update_feature_request( - &self, - id: &str, - disposition: &str, - developer_notes: Option<&str>, - ) -> Result { - let result = crate::dbm::postgres_query!( - "UPDATE feature_requests SET disposition = $2, developer_notes = $3, updated_at = now() \ - WHERE id = $1", - ) - .bind(id) - .bind(disposition) - .bind(developer_notes) - .execute(self.pool()) - .await - .map_err(storage_error)?; - Ok(result.rows_affected() > 0) - } - - pub async fn insert_evolution_record( - &self, - id: &str, - record_type: &str, - status: &str, - created_by: &str, - derived_from: Option<&str>, - data_json: &str, - ) -> Result<(), PersistenceError> { - let payload = parse_json(data_json)?; - crate::dbm::postgres_query!( - "INSERT INTO evolution_records (id, record_type, status, created_by, derived_from, payload, timestamp) \ - VALUES ($1, $2, $3, $4, $5, $6, now())", - ) - .bind(id) - .bind(record_type) - .bind(status) - .bind(created_by) - .bind(derived_from) - .bind(payload) - .execute(self.pool()) - .await - .map_err(storage_error)?; - Ok(()) - } - - pub async fn get_evolution_record( - &self, - id: &str, - ) -> Result, PersistenceError> { - let row = crate::dbm::postgres_query!( - "SELECT id, record_type, status, created_by, derived_from, payload, timestamp \ - FROM evolution_records WHERE id = $1", - ) - .bind(id) - .fetch_optional(self.pool()) - .await - .map_err(storage_error)?; - Ok(row.map(row_to_evolution_record)) - } - - pub async fn list_evolution_records( - &self, - record_type: Option<&str>, - status: Option<&str>, - ) -> Result, PersistenceError> { - let rows = crate::dbm::postgres_query!( - "SELECT id, record_type, status, created_by, derived_from, payload, timestamp \ - FROM evolution_records \ - WHERE ($1::text IS NULL OR record_type = $1) \ - AND ($2::text IS NULL OR status = $2) \ - ORDER BY timestamp DESC", - ) - .bind(record_type) - .bind(status) - .fetch_all(self.pool()) - .await - .map_err(storage_error)?; - Ok(rows.into_iter().map(row_to_evolution_record).collect()) - } - - pub async fn list_ranked_insights( - &self, - ) -> Result, PersistenceError> { - let mut rows = self.list_evolution_records(Some("Insight"), None).await?; - rows.sort_by(|a, b| { - let score_a = serde_json::from_str::(&a.data) - .ok() - .and_then(|v| v.get("priority_score").and_then(|s| s.as_f64())) - .unwrap_or(0.0); - let score_b = serde_json::from_str::(&b.data) - .ok() - .and_then(|v| v.get("priority_score").and_then(|s| s.as_f64())) - .unwrap_or(0.0); - score_b - .partial_cmp(&score_a) - .unwrap_or(std::cmp::Ordering::Equal) - }); - Ok(rows) - } - #[allow(clippy::too_many_arguments)] pub async fn insert_design_time_event( &self, @@ -1977,6 +1944,12 @@ impl PostgresEventStore { Ok(rows.into_iter().map(row_to_design_time_event).collect()) } + /// Persist a full OTS trajectory JSON blob. + /// + /// Identity is `(tenant, trajectory_id)`: the id comes from the uploading + /// harness and one database holds every tenant's rows, so keying on the id + /// alone would let one tenant's upload replace another's row — tenant + /// column included. pub async fn persist_ots_trajectory( &self, p: &PostgresOtsTrajectoryParams<'_>, @@ -1986,8 +1959,8 @@ impl PostgresEventStore { "INSERT INTO ots_trajectories \ (trajectory_id, tenant, agent_id, session_id, outcome, turn_count, data, persistence_status, persist_attempts, last_error, created_at, updated_at) \ VALUES ($1, $2, $3, $4, $5, $6, $7, 'persisted', 0, NULL, now(), now()) \ - ON CONFLICT (trajectory_id) DO UPDATE SET \ - tenant = EXCLUDED.tenant, agent_id = EXCLUDED.agent_id, session_id = EXCLUDED.session_id, \ + ON CONFLICT (tenant, trajectory_id) DO UPDATE SET \ + agent_id = EXCLUDED.agent_id, session_id = EXCLUDED.session_id, \ outcome = EXCLUDED.outcome, turn_count = EXCLUDED.turn_count, data = EXCLUDED.data, \ persistence_status = 'persisted', persist_attempts = 0, last_error = NULL, updated_at = now()", ) @@ -2013,8 +1986,8 @@ impl PostgresEventStore { "INSERT INTO ots_trajectories \ (trajectory_id, tenant, agent_id, session_id, outcome, turn_count, data, persistence_status, persist_attempts, last_error, created_at, updated_at) \ VALUES ($1, $2, $3, $4, $5, $6, $7, 'queued', 0, NULL, now(), now()) \ - ON CONFLICT (trajectory_id) DO UPDATE SET \ - tenant = EXCLUDED.tenant, agent_id = EXCLUDED.agent_id, session_id = EXCLUDED.session_id, \ + ON CONFLICT (tenant, trajectory_id) DO UPDATE SET \ + agent_id = EXCLUDED.agent_id, session_id = EXCLUDED.session_id, \ outcome = EXCLUDED.outcome, turn_count = EXCLUDED.turn_count, data = EXCLUDED.data, \ persistence_status = 'queued', last_error = NULL, updated_at = now()", ) @@ -2031,15 +2004,22 @@ impl PostgresEventStore { Ok(()) } + /// Mark a queued OTS trajectory as persisted. + /// + /// Addressed by the same `(tenant, trajectory_id)` identity the row is + /// keyed by: two tenants may hold the same id, and an unscoped update would + /// declare both of them persisted. pub async fn mark_ots_trajectory_persisted( &self, + tenant: &str, trajectory_id: &str, ) -> Result<(), PersistenceError> { crate::dbm::postgres_query!( "UPDATE ots_trajectories \ SET persistence_status = 'persisted', last_error = NULL, updated_at = now() \ - WHERE trajectory_id = $1", + WHERE tenant = $1 AND trajectory_id = $2", ) + .bind(tenant) .bind(trajectory_id) .execute(self.pool()) .await @@ -2047,16 +2027,19 @@ impl PostgresEventStore { Ok(()) } + /// Mark a queued OTS trajectory as failed after retries exhaust. pub async fn mark_ots_trajectory_failed( &self, + tenant: &str, trajectory_id: &str, error: &str, ) -> Result<(), PersistenceError> { crate::dbm::postgres_query!( "UPDATE ots_trajectories \ - SET persistence_status = 'failed', persist_attempts = persist_attempts + 1, last_error = $2, updated_at = now() \ - WHERE trajectory_id = $1", + SET persistence_status = 'failed', persist_attempts = persist_attempts + 1, last_error = $3, updated_at = now() \ + WHERE tenant = $1 AND trajectory_id = $2", ) + .bind(tenant) .bind(trajectory_id) .bind(error) .execute(self.pool()) @@ -2109,18 +2092,26 @@ impl PostgresEventStore { Ok(rows.into_iter().map(row_to_ots_trajectory).collect()) } + /// Load full OTS trajectory data by tenant and ID. + /// + /// The tenant is part of the lookup rather than a post-filter: one store + /// holds every tenant's rows, so a caller that takes the trajectory id + /// from a request path would otherwise read across tenants. pub async fn get_ots_trajectory( &self, + tenant: &str, trajectory_id: &str, - ) -> Result, PersistenceError> { - let row: Option = crate::dbm::postgres_query_scalar!( - "SELECT data FROM ots_trajectories WHERE trajectory_id = $1" + ) -> Result, PersistenceError> { + let row = crate::dbm::postgres_query!( + "SELECT agent_id, COALESCE(session_id, '') AS session_id, outcome, data \ + FROM ots_trajectories WHERE tenant = $1 AND trajectory_id = $2" ) + .bind(tenant) .bind(trajectory_id) .fetch_optional(self.pool()) .await .map_err(storage_error)?; - Ok(row.map(|value| value.to_string())) + Ok(row.map(|row| row_to_ots_document(row, tenant.to_string(), trajectory_id.to_string()))) } pub async fn put_blob(&self, key: &str, data: &[u8]) -> Result<(), String> { @@ -2173,6 +2164,25 @@ impl PostgresEventStore { .map_err(|e| format!("blob get failed: {e}")) } + /// Retrieve a blob only when its durable size metadata is within the + /// caller's allocation budget. `None` means missing or over budget. + pub async fn get_blob_if_size_at_most( + &self, + key: &str, + max_bytes: usize, + ) -> Result>, String> { + let max_bytes = i64::try_from(max_bytes) + .map_err(|_| "legacy blob read budget exceeds i64".to_string())?; + crate::dbm::postgres_query_scalar!( + "SELECT data FROM blobs WHERE blob_key = $1 AND octet_length(data) <= $2" + ) + .bind(key) + .bind(max_bytes) + .fetch_optional(self.pool()) + .await + .map_err(|e| format!("bounded blob get failed: {e}")) + } + #[tracing::instrument(skip_all, fields( otel.name = "postgres.upsert_published_artifact", tenant = %artifact.tenant, @@ -2430,13 +2440,19 @@ impl PostgresEventStore { Ok(rows.into_iter().map(|value| value.to_string()).collect()) } - pub async fn get_pending_decision(&self, id: &str) -> Result, PersistenceError> { - let row: Option = - crate::dbm::postgres_query_scalar!("SELECT data FROM pending_decisions WHERE id = $1") - .bind(id) - .fetch_optional(self.pool()) - .await - .map_err(storage_error)?; + pub async fn get_pending_decision( + &self, + tenant: &str, + id: &str, + ) -> Result, PersistenceError> { + let row: Option = crate::dbm::postgres_query_scalar!( + "SELECT data FROM pending_decisions WHERE tenant = $1 AND id = $2" + ) + .bind(tenant) + .bind(id) + .fetch_optional(self.pool()) + .await + .map_err(storage_error)?; Ok(row.map(|value| value.to_string())) } diff --git a/crates/temper-store-postgres/src/platform/evolution.rs b/crates/temper-store-postgres/src/platform/evolution.rs new file mode 100644 index 000000000..7bd7c1477 --- /dev/null +++ b/crates/temper-store-postgres/src/platform/evolution.rs @@ -0,0 +1,185 @@ +//! Tenant-scoped feature-request and evolution-record persistence. + +use super::*; + +impl PostgresEventStore { + #[allow(clippy::too_many_arguments)] + pub async fn upsert_feature_request( + &self, + tenant: &str, + id: &str, + category: &str, + description: &str, + frequency: i64, + trajectory_refs_json: &str, + disposition: &str, + developer_notes: Option<&str>, + ) -> Result<(), PersistenceError> { + let trajectory_refs = parse_json(trajectory_refs_json)?; + crate::dbm::postgres_query!( + "INSERT INTO feature_requests \ + (id, tenant, category, description, frequency, trajectory_refs, disposition, developer_notes, updated_at) \ + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now()) \ + ON CONFLICT (id) DO UPDATE SET \ + category = EXCLUDED.category, description = EXCLUDED.description, frequency = EXCLUDED.frequency, \ + trajectory_refs = EXCLUDED.trajectory_refs, disposition = EXCLUDED.disposition, \ + developer_notes = EXCLUDED.developer_notes, updated_at = now() \ + WHERE feature_requests.tenant = EXCLUDED.tenant", + ) + .bind(id) + .bind(tenant) + .bind(category) + .bind(description) + .bind(frequency) + .bind(trajectory_refs) + .bind(disposition) + .bind(developer_notes) + .execute(self.pool()) + .await + .map_err(storage_error) + .and_then(|result| { + if result.rows_affected() == 0 { + Err(PersistenceError::Storage(format!( + "feature request '{id}' is owned by another tenant" + ))) + } else { + Ok(result) + } + })?; + Ok(()) + } + + pub async fn list_feature_requests( + &self, + tenant: &str, + disposition: Option<&str>, + ) -> Result, PersistenceError> { + let rows = crate::dbm::postgres_query!( + "SELECT id, tenant, category, description, frequency, trajectory_refs, disposition, developer_notes, created_at, updated_at \ + FROM feature_requests \ + WHERE tenant = $1 AND ($2::text IS NULL OR disposition = $2) \ + ORDER BY frequency DESC, created_at DESC", + ) + .bind(tenant) + .bind(disposition) + .fetch_all(self.pool()) + .await + .map_err(storage_error)?; + Ok(rows.into_iter().map(row_to_feature_request).collect()) + } + + pub async fn update_feature_request( + &self, + tenant: &str, + id: &str, + disposition: &str, + developer_notes: Option<&str>, + ) -> Result { + let result = crate::dbm::postgres_query!( + "UPDATE feature_requests SET disposition = $3, developer_notes = $4, updated_at = now() \ + WHERE tenant = $1 AND id = $2", + ) + .bind(tenant) + .bind(id) + .bind(disposition) + .bind(developer_notes) + .execute(self.pool()) + .await + .map_err(storage_error)?; + Ok(result.rows_affected() > 0) + } + + pub async fn insert_evolution_record( + &self, + record: PostgresEvolutionRecordInsert<'_>, + ) -> Result<(), PersistenceError> { + let PostgresEvolutionRecordInsert { + tenant, + id, + record_type, + status, + created_by, + derived_from, + data_json, + } = record; + let payload = parse_json(data_json)?; + crate::dbm::postgres_query!( + "INSERT INTO evolution_records (id, tenant, record_type, status, created_by, derived_from, payload, timestamp) \ + VALUES ($1, $2, $3, $4, $5, $6, $7, now())", + ) + .bind(id) + .bind(tenant) + .bind(record_type) + .bind(status) + .bind(created_by) + .bind(derived_from) + .bind(payload) + .execute(self.pool()) + .await + .map_err(storage_error)?; + Ok(()) + } + + pub async fn get_evolution_record( + &self, + tenant: &str, + id: &str, + ) -> Result, PersistenceError> { + let row = crate::dbm::postgres_query!( + "SELECT id, tenant, record_type, status, created_by, derived_from, payload, timestamp \ + FROM evolution_records WHERE tenant = $1 AND id = $2", + ) + .bind(tenant) + .bind(id) + .fetch_optional(self.pool()) + .await + .map_err(storage_error)?; + Ok(row.map(row_to_evolution_record)) + } + + pub async fn list_evolution_records( + &self, + tenant: &str, + record_type: Option<&str>, + status: Option<&str>, + ) -> Result, PersistenceError> { + let rows = crate::dbm::postgres_query!( + "SELECT id, tenant, record_type, status, created_by, derived_from, payload, timestamp \ + FROM evolution_records \ + WHERE tenant = $1 \ + AND ($2::text IS NULL OR record_type = $2) \ + AND ($3::text IS NULL OR status = $3) \ + ORDER BY timestamp DESC", + ) + .bind(tenant) + .bind(record_type) + .bind(status) + .fetch_all(self.pool()) + .await + .map_err(storage_error)?; + Ok(rows.into_iter().map(row_to_evolution_record).collect()) + } + + pub async fn list_ranked_insights( + &self, + tenant: &str, + ) -> Result, PersistenceError> { + let mut rows = self + .list_evolution_records(tenant, Some("Insight"), None) + .await?; + rows.sort_by(|a, b| { + let score_a = serde_json::from_str::(&a.data) + .ok() + .and_then(|value| value.get("priority_score").and_then(|score| score.as_f64())) + .unwrap_or(0.0); + let score_b = serde_json::from_str::(&b.data) + .ok() + .and_then(|value| value.get("priority_score").and_then(|score| score.as_f64())) + .unwrap_or(0.0); + score_b + .partial_cmp(&score_a) + .unwrap_or(std::cmp::Ordering::Equal) + }); + Ok(rows) + } +} diff --git a/crates/temper-store-postgres/src/platform/inputs.rs b/crates/temper-store-postgres/src/platform/inputs.rs new file mode 100644 index 000000000..3fc58f9b5 --- /dev/null +++ b/crates/temper-store-postgres/src/platform/inputs.rs @@ -0,0 +1,37 @@ +//! Typed write inputs for atomic platform-store operations. + +/// Values written atomically as one evolution record. +#[derive(Clone, Copy, Debug)] +pub struct PostgresEvolutionRecordInsert<'a> { + /// Tenant that owns the record. + pub tenant: &'a str, + /// Stable evolution record identifier. + pub id: &'a str, + /// Evolution record kind. + pub record_type: &'a str, + /// Current record status. + pub status: &'a str, + /// Principal that created the record. + pub created_by: &'a str, + /// Optional predecessor record identifier. + pub derived_from: Option<&'a str>, + /// Serialized record payload. + pub data_json: &'a str, +} + +/// Exact decision and policy values committed by one approval transaction. +#[derive(Clone, Copy, Debug)] +pub struct PostgresPolicyApprovalCommit<'a> { + /// Tenant that owns both rows. + pub tenant: &'a str, + /// Pending decision to transition. + pub decision_id: &'a str, + /// Serialized approved decision. + pub approved_decision_json: &'a str, + /// Policy row created by the decision. + pub policy_id: &'a str, + /// Approved Cedar source. + pub cedar_text: &'a str, + /// Principal that approved the decision. + pub created_by: &'a str, +} diff --git a/crates/temper-store-postgres/src/platform/policy_approval.rs b/crates/temper-store-postgres/src/platform/policy_approval.rs new file mode 100644 index 000000000..0dd84122c --- /dev/null +++ b/crates/temper-store-postgres/src/platform/policy_approval.rs @@ -0,0 +1,105 @@ +//! Transactional persistence for Cedar policy approvals. + +use temper_runtime::persistence::{PersistenceError, storage_error}; + +use super::compute_policy_hash; +use crate::{PostgresEventStore, PostgresPolicyApprovalCommit}; + +impl PostgresEventStore { + /// Atomically insert an approved policy and transition its decision. + pub async fn commit_policy_approval( + &self, + commit: PostgresPolicyApprovalCommit<'_>, + ) -> Result<(), PersistenceError> { + let PostgresPolicyApprovalCommit { + tenant, + decision_id, + approved_decision_json, + policy_id, + cedar_text, + created_by, + } = commit; + let approved_data: serde_json::Value = serde_json::from_str(approved_decision_json) + .map_err(|error| PersistenceError::Serialization(error.to_string()))?; + let policy_hash = compute_policy_hash(cedar_text); + let mut transaction = self.pool().begin().await.map_err(storage_error)?; + + let policy_result = crate::dbm::postgres_query!( + "INSERT INTO policies \ + (tenant, policy_id, cedar_text, policy_hash, created_at, created_by, enabled) \ + VALUES ($1, $2, $3, $4, now(), $5, true) \ + ON CONFLICT (tenant, policy_id) DO NOTHING", + ) + .bind(tenant) + .bind(policy_id) + .bind(cedar_text) + .bind(policy_hash) + .bind(created_by) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + if policy_result.rows_affected() != 1 { + return Err(PersistenceError::Storage(format!( + "policy approval '{policy_id}' already exists" + ))); + } + + let decision_result = crate::dbm::postgres_query!( + "UPDATE pending_decisions \ + SET status = 'approved', data = $3, updated_at = now() \ + WHERE tenant = $1 AND id = $2 AND status = 'pending'", + ) + .bind(tenant) + .bind(decision_id) + .bind(approved_data) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + if decision_result.rows_affected() != 1 { + return Err(PersistenceError::Storage(format!( + "pending decision '{decision_id}' was not available for approval" + ))); + } + + transaction.commit().await.map_err(storage_error) + } + + /// Compensate a committed approval when runtime activation fails. + pub async fn rollback_policy_approval( + &self, + tenant: &str, + decision_id: &str, + pending_decision_json: &str, + policy_id: &str, + ) -> Result<(), PersistenceError> { + let pending_data: serde_json::Value = serde_json::from_str(pending_decision_json) + .map_err(|error| PersistenceError::Serialization(error.to_string()))?; + let mut transaction = self.pool().begin().await.map_err(storage_error)?; + + crate::dbm::postgres_query!("DELETE FROM policies WHERE tenant = $1 AND policy_id = $2",) + .bind(tenant) + .bind(policy_id) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + + let decision_result = crate::dbm::postgres_query!( + "UPDATE pending_decisions \ + SET status = 'pending', data = $3, updated_at = now() \ + WHERE tenant = $1 AND id = $2 AND status = 'approved'", + ) + .bind(tenant) + .bind(decision_id) + .bind(pending_data) + .execute(&mut *transaction) + .await + .map_err(storage_error)?; + if decision_result.rows_affected() != 1 { + return Err(PersistenceError::Storage(format!( + "approved decision '{decision_id}' was not available for rollback" + ))); + } + + transaction.commit().await.map_err(storage_error) + } +} diff --git a/crates/temper-store-postgres/src/platform/rows.rs b/crates/temper-store-postgres/src/platform/rows.rs index 309bfa9f1..1c1d6e8b6 100644 --- a/crates/temper-store-postgres/src/platform/rows.rs +++ b/crates/temper-store-postgres/src/platform/rows.rs @@ -149,6 +149,7 @@ pub(super) fn row_to_trajectory(row: sqlx::postgres::PgRow) -> PostgresTrajector intent: row.get("intent"), matched_policy_ids: matched_policy_ids .and_then(|value| serde_json::from_value::>(value).ok()), + capture_seq: row.try_get("capture_seq").ok().flatten(), } } @@ -191,6 +192,7 @@ pub(super) fn row_to_feature_request(row: sqlx::postgres::PgRow) -> PostgresFeat let updated_at: chrono::DateTime = row.get("updated_at"); PostgresFeatureRequestRow { id: row.get("id"), + tenant: row.get("tenant"), category: row.get("category"), description: row.get("description"), frequency: row.get("frequency"), @@ -207,6 +209,7 @@ pub(super) fn row_to_evolution_record(row: sqlx::postgres::PgRow) -> PostgresEvo let timestamp: chrono::DateTime = row.get("timestamp"); PostgresEvolutionRecordRow { id: row.get("id"), + tenant: row.get("tenant"), record_type: row.get("record_type"), status: row.get("status"), created_by: row.get("created_by"), @@ -252,6 +255,22 @@ pub(super) fn row_to_ots_trajectory(row: sqlx::postgres::PgRow) -> PostgresOtsTr } } +pub(super) fn row_to_ots_document( + row: sqlx::postgres::PgRow, + tenant: String, + trajectory_id: String, +) -> PostgresOtsTrajectoryDocument { + let data: serde_json::Value = row.get("data"); + PostgresOtsTrajectoryDocument { + trajectory_id, + tenant, + agent_id: row.get("agent_id"), + session_id: row.get("session_id"), + outcome: row.get("outcome"), + data: data.to_string(), + } +} + pub(super) fn row_to_queued_ots_trajectory( row: sqlx::postgres::PgRow, ) -> PostgresQueuedOtsTrajectoryRow { diff --git a/crates/temper-store-postgres/src/schema.rs b/crates/temper-store-postgres/src/schema.rs index 865a036ab..a6d6eb423 100644 --- a/crates/temper-store-postgres/src/schema.rs +++ b/crates/temper-store-postgres/src/schema.rs @@ -105,6 +105,7 @@ CREATE TABLE IF NOT EXISTS trajectories ( request_body JSONB, intent TEXT, matched_policy_ids JSONB, + capture_seq BIGINT, agent_type TEXT, created_at TIMESTAMPTZ NOT NULL DEFAULT now() );"; @@ -117,6 +118,13 @@ CREATE INDEX IF NOT EXISTS idx_trajectories_success ON trajectories (success, cr pub const CREATE_TRAJECTORIES_ENTITY_INDEX: &str = "\ CREATE INDEX IF NOT EXISTS idx_trajectories_entity ON trajectories (entity_type, action);"; +/// CREATE INDEX statement for session-scoped trajectory replay. +/// +/// Conformance checking reads one session's rows in write order, so the index +/// covers the ordering columns and the read stays a range scan. +pub const CREATE_TRAJECTORIES_SESSION_INDEX: &str = "\ +CREATE INDEX IF NOT EXISTS idx_trajectories_session ON trajectories (session_id, created_at, id);"; + /// CREATE TABLE statement for persisted design-time workflow events. pub const CREATE_DESIGN_TIME_EVENTS_TABLE: &str = "\ CREATE TABLE IF NOT EXISTS design_time_events ( @@ -405,9 +413,15 @@ CREATE INDEX IF NOT EXISTS idx_evolution_records_derived_from ON evolution_records (derived_from);"; /// Full OTS trajectory storage for agent execution traces. +/// +/// Keyed by `(tenant, trajectory_id)`. One database holds every tenant's rows +/// and the id is chosen by the uploading harness, so a global key lets one +/// tenant's upload land on an id another tenant already used — and the upsert +/// would rewrite that row's tenant along with its data. Migration 0015 rekeys +/// databases created before this. pub const CREATE_OTS_TRAJECTORIES_TABLE: &str = "\ CREATE TABLE IF NOT EXISTS ots_trajectories ( - trajectory_id TEXT PRIMARY KEY, + trajectory_id TEXT NOT NULL, tenant TEXT NOT NULL, agent_id TEXT NOT NULL, session_id TEXT, @@ -419,7 +433,8 @@ CREATE TABLE IF NOT EXISTS ots_trajectories ( persist_attempts BIGINT NOT NULL DEFAULT 0, last_error TEXT, created_at TIMESTAMPTZ NOT NULL DEFAULT now(), - updated_at TIMESTAMPTZ NOT NULL DEFAULT now() + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT ots_trajectories_tenant_identity PRIMARY KEY (tenant, trajectory_id) );"; /// Add durable outbox status to existing OTS trajectory tables. diff --git a/crates/temper-store-postgres/src/store.rs b/crates/temper-store-postgres/src/store.rs index 21aa5c717..a04ecfc44 100644 --- a/crates/temper-store-postgres/src/store.rs +++ b/crates/temper-store-postgres/src/store.rs @@ -1257,6 +1257,7 @@ mod tests { request_body: Some("{\"ok\":true}"), intent: Some("test"), matched_policy_ids: Some("[\"policy:test\"]"), + capture_seq: Some(1), }); let _ = store.save_policy( "tenant", @@ -1287,6 +1288,7 @@ mod tests { let _ = PostgresEventStore::count_trajectories_by_tenant; let _ = PostgresEventStore::query_trajectory_stats; let _ = PostgresEventStore::query_trajectories_by_agent; + let _ = PostgresEventStore::query_trajectories_by_session; let _ = PostgresEventStore::query_agent_summaries; let _ = PostgresEventStore::upsert_feature_request; diff --git a/crates/temper-store-postgres/src/store_projection_test.rs b/crates/temper-store-postgres/src/store_projection_test.rs index 22f3e2df1..36587e32b 100644 --- a/crates/temper-store-postgres/src/store_projection_test.rs +++ b/crates/temper-store-postgres/src/store_projection_test.rs @@ -1144,3 +1144,80 @@ fn upsert_query_projection_removes_index_row_when_value_becomes_too_long() { .unwrap(); }); } + +/// The session read replays a run in capture order on Postgres too. +/// +/// Rows are persisted by independently spawned tasks, so inside one +/// `created_at` tick the BIGSERIAL id is the order the writes landed rather +/// than the order the kernel captured them. The Turso path proves the same +/// property in `temper-store-turso`; this is the Postgres half, so the two +/// backends cannot drift into different replay orders. +/// +/// Gated on DATABASE_URL (skips otherwise); isolated by a unique tenant. +#[test] +fn session_read_follows_capture_order_not_insert_order() { + let database_url = match std::env::var("DATABASE_URL") { + Ok(url) => url, + Err(_) => return, + }; + + sqlx::test_block_on(async { + let pool = PgPool::connect(&database_url).await.unwrap(); + run_migrations(&pool).await.unwrap(); + let store = PostgresEventStore::new(pool.clone()); + let tenant = format!("tenant-capture-order-{}", uuid::Uuid::new_v4()); + let session = format!("session-{}", uuid::Uuid::new_v4()); + + // Captured first, inserted second. + for (action, capture_seq) in [("SubmitOrder", 2i64), ("AddItem", 1i64)] { + store + .persist_trajectory(crate::PostgresTrajectoryInsert { + tenant: &tenant, + entity_type: "Order", + entity_id: "order-1", + action, + success: true, + from_status: Some("Draft"), + to_status: Some("Draft"), + error: None, + agent_id: Some("agent-1"), + session_id: Some(&session), + authz_denied: None, + denied_resource: None, + denied_module: None, + source: Some("Entity"), + spec_governed: Some(true), + created_at: "2026-01-01T00:00:00Z", + request_body: None, + intent: None, + matched_policy_ids: None, + capture_seq: Some(capture_seq), + }) + .await + .unwrap(); + } + + let rows = store + .query_trajectories_by_session(&session, Some(&tenant), None, 100) + .await + .unwrap(); + let actions: Vec<&str> = rows.iter().map(|row| row.action.as_str()).collect(); + assert_eq!( + actions, + vec!["AddItem", "SubmitOrder"], + "the read must follow capture order, not the order the writes landed" + ); + assert_eq!( + rows.iter() + .filter_map(|row| row.capture_seq) + .collect::>(), + vec![1, 2] + ); + + crate::dbm::postgres_query!("DELETE FROM trajectories WHERE tenant = $1") + .bind(&tenant) + .execute(&pool) + .await + .unwrap(); + }); +} diff --git a/crates/temper-store-turso/src/lib.rs b/crates/temper-store-turso/src/lib.rs index 231b54209..53a5372cc 100644 --- a/crates/temper-store-turso/src/lib.rs +++ b/crates/temper-store-turso/src/lib.rs @@ -53,6 +53,10 @@ pub struct TursoTrajectoryInsert<'a> { pub request_body: Option<&'a str>, pub intent: Option<&'a str>, pub matched_policy_ids: Option<&'a str>, + /// Monotonic capture order stamped by the recording process, so a session + /// reads back in the order the kernel captured it rather than the order + /// independent persistence tasks happened to land. + pub capture_seq: Option, } #[derive(Clone, Copy, Debug)] @@ -69,6 +73,42 @@ pub struct TursoWasmInvocationInsert<'a> { pub created_at: &'a str, } +/// Values written atomically as one evolution record. +#[derive(Clone, Copy, Debug)] +pub struct TursoEvolutionRecordInsert<'a> { + /// Tenant that owns the record. + pub tenant: &'a str, + /// Stable evolution record identifier. + pub id: &'a str, + /// Evolution record kind. + pub record_type: &'a str, + /// Current record status. + pub status: &'a str, + /// Principal that created the record. + pub created_by: &'a str, + /// Optional predecessor record identifier. + pub derived_from: Option<&'a str>, + /// Serialized record payload. + pub data_json: &'a str, +} + +/// Exact decision and policy values committed by one approval transaction. +#[derive(Clone, Copy, Debug)] +pub struct TursoPolicyApprovalCommit<'a> { + /// Tenant that owns both rows. + pub tenant: &'a str, + /// Pending decision to transition. + pub decision_id: &'a str, + /// Serialized approved decision. + pub approved_decision_json: &'a str, + /// Policy row created by the decision. + pub policy_id: &'a str, + /// Approved Cedar source. + pub cedar_text: &'a str, + /// Principal that approved the decision. + pub created_by: &'a str, +} + pub use metrics::init_metrics; pub use router::{TenantRegistryRow, TenantStoreRouter, TenantUserRow}; pub use store::{ @@ -77,5 +117,5 @@ pub use store::{ QueryProjectionUpsert, TursoBlobRow, TursoEventStore, TursoInstalledAppRow, TursoQueryProjectionRow, TursoSpecRow, TursoTenantConstraintRow, TursoTrajectoryRow, TursoWasmInvocationRow, TursoWasmModuleMetadataRow, TursoWasmModuleRow, UnmetIntentAggRow, - ots::{OtsQueuedTrajectoryRow, OtsTrajectoryParams, OtsTrajectoryRow}, + ots::{OtsQueuedTrajectoryRow, OtsTrajectoryDocument, OtsTrajectoryParams, OtsTrajectoryRow}, }; diff --git a/crates/temper-store-turso/src/schema.rs b/crates/temper-store-turso/src/schema.rs index 1012b2e03..9bd2ec550 100644 --- a/crates/temper-store-turso/src/schema.rs +++ b/crates/temper-store-turso/src/schema.rs @@ -300,16 +300,36 @@ pub const ALTER_TRAJECTORIES_ADD_REQUEST_BODY: &str = pub const ALTER_TRAJECTORIES_ADD_INTENT: &str = "ALTER TABLE trajectories ADD COLUMN intent TEXT"; pub const ALTER_TRAJECTORIES_ADD_MATCHED_POLICY_IDS: &str = "ALTER TABLE trajectories ADD COLUMN matched_policy_ids TEXT"; +/// Capture order within the writing process. +/// +/// Rows are written by independently spawned persistence tasks, so the +/// autoincrement `id` records the order the writes *landed*, not the order the +/// kernel *captured* them. The conformance checker replays a session as a +/// state-machine walk and needs capture order, so the capturing process stamps +/// a monotonic sequence on the entry before it is queued and the read orders +/// by it. Null on rows written before this column existed. +pub const ALTER_TRAJECTORIES_ADD_CAPTURE_SEQ: &str = + "ALTER TABLE trajectories ADD COLUMN capture_seq INTEGER"; /// Index on agent_id for agent-scoped trajectory queries. pub const CREATE_TRAJECTORIES_AGENT_INDEX: &str = "\ CREATE INDEX IF NOT EXISTS idx_trajectories_agent ON trajectories(agent_id);"; +/// Index on session_id for session-scoped trajectory replay. +/// +/// Conformance checking reads one session's rows in capture order, so the +/// index covers every ordering column and the read is a range scan rather +/// than a table scan. +pub const CREATE_TRAJECTORIES_SESSION_INDEX: &str = "\ +CREATE INDEX IF NOT EXISTS idx_trajectories_session_capture + ON trajectories(session_id, created_at, capture_seq, id);"; + /// Feature request records generated from trajectory analysis. pub const CREATE_FEATURE_REQUESTS_TABLE: &str = "\ CREATE TABLE IF NOT EXISTS feature_requests ( id TEXT PRIMARY KEY, + tenant TEXT NOT NULL DEFAULT 'default', category TEXT NOT NULL, description TEXT NOT NULL, frequency INTEGER NOT NULL DEFAULT 0, @@ -324,6 +344,7 @@ CREATE TABLE IF NOT EXISTS feature_requests ( pub const CREATE_EVOLUTION_RECORDS_TABLE: &str = "\ CREATE TABLE IF NOT EXISTS evolution_records ( id TEXT PRIMARY KEY, + tenant TEXT NOT NULL DEFAULT 'default', record_type TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'Open', created_by TEXT NOT NULL, @@ -332,6 +353,26 @@ CREATE TABLE IF NOT EXISTS evolution_records ( timestamp TEXT NOT NULL DEFAULT (datetime('now')) );"; +/// Idempotent-at-runner migration for feature-request tenant ownership. +pub const ALTER_FEATURE_REQUESTS_ADD_TENANT: &str = + "ALTER TABLE feature_requests ADD COLUMN tenant TEXT NOT NULL DEFAULT 'default'"; + +/// Idempotent-at-runner migration for evolution-record tenant ownership. +pub const ALTER_EVOLUTION_RECORDS_ADD_TENANT: &str = + "ALTER TABLE evolution_records ADD COLUMN tenant TEXT NOT NULL DEFAULT 'default'"; + +pub const CREATE_FEATURE_REQUESTS_TENANT_INDEX: &str = "\ +CREATE INDEX IF NOT EXISTS idx_feature_requests_tenant_disposition + ON feature_requests(tenant, disposition, frequency DESC, created_at DESC);"; + +pub const CREATE_EVOLUTION_RECORDS_TENANT_INDEX: &str = "\ +CREATE INDEX IF NOT EXISTS idx_evolution_records_tenant_type_status + ON evolution_records(tenant, record_type, status, timestamp DESC);"; + +pub const CREATE_EVOLUTION_RECORDS_TENANT_PARENT_INDEX: &str = "\ +CREATE INDEX IF NOT EXISTS idx_evolution_records_tenant_parent + ON evolution_records(tenant, derived_from);"; + pub const CREATE_EVOLUTION_RECORDS_TYPE_INDEX: &str = "\ CREATE INDEX IF NOT EXISTS idx_evolution_records_type ON evolution_records(record_type);"; @@ -439,59 +480,8 @@ ALTER TABLE blobs ADD COLUMN expires_at TEXT;"; pub const CREATE_BLOBS_EXPIRES_AT_INDEX: &str = "\ CREATE INDEX IF NOT EXISTS idx_blobs_expires_at ON blobs(expires_at) WHERE expires_at IS NOT NULL;"; -// --------------------------------------------------------------------------- -// OTS trajectory storage (full agent execution traces) -// --------------------------------------------------------------------------- - -/// Full OTS trajectory storage for GEPA self-improvement loop. -/// -/// Stores complete agent execution traces (tool calls, decisions, reasoning) -/// captured by the MCP server during agent sessions. The `data` column holds -/// the full OTS JSON blob; indexed columns enable efficient filtering. -pub const CREATE_OTS_TRAJECTORIES_TABLE: &str = "\ -CREATE TABLE IF NOT EXISTS ots_trajectories ( - trajectory_id TEXT PRIMARY KEY, - tenant TEXT NOT NULL, - agent_id TEXT NOT NULL, - session_id TEXT, - outcome TEXT NOT NULL DEFAULT 'unknown', - entity_type TEXT, - turn_count INTEGER NOT NULL DEFAULT 0, - data TEXT NOT NULL, - persistence_status TEXT NOT NULL DEFAULT 'persisted', - persist_attempts INTEGER NOT NULL DEFAULT 0, - last_error TEXT, - created_at TEXT NOT NULL DEFAULT (datetime('now')), - updated_at TEXT NOT NULL DEFAULT (datetime('now')) -);"; - -pub const ALTER_OTS_TRAJECTORIES_ADD_PERSISTENCE_STATUS: &str = "\ -ALTER TABLE ots_trajectories ADD COLUMN persistence_status TEXT NOT NULL DEFAULT 'persisted';"; - -pub const ALTER_OTS_TRAJECTORIES_ADD_PERSIST_ATTEMPTS: &str = "\ -ALTER TABLE ots_trajectories ADD COLUMN persist_attempts INTEGER NOT NULL DEFAULT 0;"; - -pub const ALTER_OTS_TRAJECTORIES_ADD_LAST_ERROR: &str = "\ -ALTER TABLE ots_trajectories ADD COLUMN last_error TEXT;"; - -pub const ALTER_OTS_TRAJECTORIES_ADD_UPDATED_AT: &str = "\ -ALTER TABLE ots_trajectories ADD COLUMN updated_at TEXT NOT NULL DEFAULT (datetime('now'));"; - -pub const CREATE_OTS_TRAJECTORIES_AGENT_INDEX: &str = "\ -CREATE INDEX IF NOT EXISTS idx_ots_trajectories_agent - ON ots_trajectories(agent_id);"; - -pub const CREATE_OTS_TRAJECTORIES_TENANT_INDEX: &str = "\ -CREATE INDEX IF NOT EXISTS idx_ots_trajectories_tenant - ON ots_trajectories(tenant);"; - -pub const CREATE_OTS_TRAJECTORIES_OUTCOME_INDEX: &str = "\ -CREATE INDEX IF NOT EXISTS idx_ots_trajectories_outcome - ON ots_trajectories(outcome);"; - -pub const CREATE_OTS_TRAJECTORIES_STATUS_INDEX: &str = "\ -CREATE INDEX IF NOT EXISTS idx_ots_trajectories_status - ON ots_trajectories(persistence_status, updated_at);"; +mod ots; +pub use ots::*; #[cfg(test)] #[path = "schema_test.rs"] diff --git a/crates/temper-store-turso/src/schema/ots.rs b/crates/temper-store-turso/src/schema/ots.rs new file mode 100644 index 000000000..ab8a175cd --- /dev/null +++ b/crates/temper-store-turso/src/schema/ots.rs @@ -0,0 +1,122 @@ +//! OTS trajectory schema used by the GEPA self-improvement loop. +//! +//! Rows are keyed by `(tenant, trajectory_id)`. A global id let one tenant's +//! upload land on an id another tenant already used, and the upsert would +//! rewrite that row's tenant along with its data; the migration constants +//! below rebuild legacy tables onto the tenant-scoped key. + +/// Full OTS trajectory storage for GEPA self-improvement loop. +/// +/// Stores complete agent execution traces (tool calls, decisions, reasoning) +/// captured by the MCP server during agent sessions. The `data` column holds +/// the full OTS JSON blob; indexed columns enable efficient filtering. +/// +/// Keyed by `(tenant, trajectory_id)`. One store holds every tenant's rows and +/// the id is chosen by the uploading harness, so a global key lets one tenant's +/// upload land on an id another tenant already used — and the upsert would +/// rewrite that row's tenant along with its data. Reads are already +/// tenant-scoped; the identity has to be too. +pub const CREATE_OTS_TRAJECTORIES_TABLE: &str = "\ +CREATE TABLE IF NOT EXISTS ots_trajectories ( + trajectory_id TEXT NOT NULL, + tenant TEXT NOT NULL, + agent_id TEXT NOT NULL, + session_id TEXT, + outcome TEXT NOT NULL DEFAULT 'unknown', + entity_type TEXT, + turn_count INTEGER NOT NULL DEFAULT 0, + data TEXT NOT NULL, + persistence_status TEXT NOT NULL DEFAULT 'persisted', + persist_attempts INTEGER NOT NULL DEFAULT 0, + last_error TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')), + PRIMARY KEY(tenant, trajectory_id) +);"; + +/// Whether an existing `ots_trajectories` table is already tenant-scoped. +/// +/// SQLite cannot alter a primary key, so the migration below rebuilds the +/// table. This reads the stored DDL to decide whether it has to. +pub const SELECT_OTS_TRAJECTORIES_DDL: &str = "\ +SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'ots_trajectories';"; + +/// Marker the tenant-scoped DDL contains and the legacy DDL does not. +pub const OTS_TRAJECTORIES_TENANT_IDENTITY_MARKER: &str = "PRIMARY KEY(tenant, trajectory_id)"; + +/// Migration step 1: the tenant-scoped table, under a staging name. +pub const CREATE_OTS_TRAJECTORIES_REBUILD_TABLE: &str = "\ +CREATE TABLE IF NOT EXISTS ots_trajectories_rebuild ( + trajectory_id TEXT NOT NULL, + tenant TEXT NOT NULL, + agent_id TEXT NOT NULL, + session_id TEXT, + outcome TEXT NOT NULL DEFAULT 'unknown', + entity_type TEXT, + turn_count INTEGER NOT NULL DEFAULT 0, + data TEXT NOT NULL, + persistence_status TEXT NOT NULL DEFAULT 'persisted', + persist_attempts INTEGER NOT NULL DEFAULT 0, + last_error TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')), + PRIMARY KEY(tenant, trajectory_id) +);"; + +/// Migration step 2: carry every stored row across. +/// +/// The old key was global, so no two rows can collide on the new one; the +/// column list is explicit so a table with extra legacy columns still copies. +pub const COPY_OTS_TRAJECTORIES_INTO_REBUILD: &str = "\ +INSERT INTO ots_trajectories_rebuild + (trajectory_id, tenant, agent_id, session_id, outcome, entity_type, turn_count, data, + persistence_status, persist_attempts, last_error, created_at, updated_at) +SELECT trajectory_id, tenant, agent_id, session_id, outcome, entity_type, turn_count, data, + persistence_status, persist_attempts, last_error, created_at, updated_at +FROM ots_trajectories;"; + +/// Migration step 3: drop the globally keyed table. +pub const DROP_OTS_TRAJECTORIES_LEGACY_TABLE: &str = "DROP TABLE ots_trajectories;"; + +/// Migration step 4: put the rebuilt table in its place. +/// +/// The indexes are recreated after this by the same bootstrap that runs the +/// migration, because dropping the old table dropped them with it. +pub const RENAME_OTS_TRAJECTORIES_REBUILD: &str = "\ +ALTER TABLE ots_trajectories_rebuild RENAME TO ots_trajectories;"; + +/// Add durable outbox persistence status to legacy OTS tables. +pub const ALTER_OTS_TRAJECTORIES_ADD_PERSISTENCE_STATUS: &str = "\ +ALTER TABLE ots_trajectories ADD COLUMN persistence_status TEXT NOT NULL DEFAULT 'persisted';"; + +/// Add durable outbox attempt accounting to legacy OTS tables. +pub const ALTER_OTS_TRAJECTORIES_ADD_PERSIST_ATTEMPTS: &str = "\ +ALTER TABLE ots_trajectories ADD COLUMN persist_attempts INTEGER NOT NULL DEFAULT 0;"; + +/// Add the last persistence error to legacy OTS tables. +pub const ALTER_OTS_TRAJECTORIES_ADD_LAST_ERROR: &str = "\ +ALTER TABLE ots_trajectories ADD COLUMN last_error TEXT;"; + +/// Add the outbox update timestamp to legacy OTS tables. +pub const ALTER_OTS_TRAJECTORIES_ADD_UPDATED_AT: &str = "\ +ALTER TABLE ots_trajectories ADD COLUMN updated_at TEXT NOT NULL DEFAULT (datetime('now'));"; + +/// Index OTS trajectories by agent. +pub const CREATE_OTS_TRAJECTORIES_AGENT_INDEX: &str = "\ +CREATE INDEX IF NOT EXISTS idx_ots_trajectories_agent + ON ots_trajectories(agent_id);"; + +/// Index OTS trajectories by tenant. +pub const CREATE_OTS_TRAJECTORIES_TENANT_INDEX: &str = "\ +CREATE INDEX IF NOT EXISTS idx_ots_trajectories_tenant + ON ots_trajectories(tenant);"; + +/// Index OTS trajectories by outcome. +pub const CREATE_OTS_TRAJECTORIES_OUTCOME_INDEX: &str = "\ +CREATE INDEX IF NOT EXISTS idx_ots_trajectories_outcome + ON ots_trajectories(outcome);"; + +/// Index the durable OTS outbox scan by status and update time. +pub const CREATE_OTS_TRAJECTORIES_STATUS_INDEX: &str = "\ +CREATE INDEX IF NOT EXISTS idx_ots_trajectories_status + ON ots_trajectories(persistence_status, updated_at);"; diff --git a/crates/temper-store-turso/src/store/authz.rs b/crates/temper-store-turso/src/store/authz.rs index 7714db95f..406dbd9a0 100644 --- a/crates/temper-store-turso/src/store/authz.rs +++ b/crates/temper-store-turso/src/store/authz.rs @@ -61,13 +61,17 @@ impl TursoEventStore { /// Get a single pending decision by ID, returning the full JSON data. #[instrument(skip_all, fields(id, otel.name = "turso.get_pending_decision"))] - pub async fn get_pending_decision(&self, id: &str) -> Result, PersistenceError> { + pub async fn get_pending_decision( + &self, + tenant: &str, + id: &str, + ) -> Result, PersistenceError> { let _query_timer = TursoQueryTimer::start("turso.get_pending_decision"); let conn = self.configured_connection().await?; let mut rows = conn .query( - "SELECT data FROM pending_decisions WHERE id = ?1", - params![id], + "SELECT data FROM pending_decisions WHERE tenant = ?1 AND id = ?2", + params![tenant, id], ) .await .map_err(storage_error)?; @@ -89,14 +93,21 @@ impl TursoEventStore { ) -> Result<(), PersistenceError> { let _query_timer = TursoQueryTimer::start("turso.upsert_pending_decision"); let conn = self.configured_connection().await?; - conn.execute( - "INSERT INTO pending_decisions (id, tenant, status, data, updated_at) \ + let affected = conn + .execute( + "INSERT INTO pending_decisions (id, tenant, status, data, updated_at) \ VALUES (?1, ?2, ?3, ?4, datetime('now')) \ - ON CONFLICT(id) DO UPDATE SET status = ?3, data = ?4, updated_at = datetime('now')", - params![id, tenant, status, data_json], - ) - .await - .map_err(storage_error)?; + ON CONFLICT(id) DO UPDATE SET status = ?3, data = ?4, updated_at = datetime('now') \ + WHERE pending_decisions.tenant = excluded.tenant", + params![id, tenant, status, data_json], + ) + .await + .map_err(storage_error)?; + if affected != 1 { + return Err(PersistenceError::Storage(format!( + "pending decision '{id}' is owned by another tenant" + ))); + } Ok(()) } @@ -123,6 +134,37 @@ impl TursoEventStore { Ok(out) } + /// Load approved decisions whose scope names `session_id`, for one tenant. + /// + /// Backs session-grant validation (ADR-0157): a caller-asserted session id + /// becomes a Cedar input only when an approved decision binds that session, + /// so the lookup filters on the approved scope's session, not the denial's. + #[instrument(skip_all, fields(tenant, otel.name = "turso.load_approved_session_decisions"))] + pub async fn load_approved_session_decisions( + &self, + tenant: &str, + session_id: &str, + ) -> Result, PersistenceError> { + let _query_timer = TursoQueryTimer::start("turso.load_approved_session_decisions"); + let conn = self.configured_connection().await?; + let mut rows = conn + .query( + "SELECT data FROM pending_decisions \ + WHERE tenant = ?1 \ + AND status = 'approved' \ + AND json_extract(data, '$.approved_scope.session_id') = ?2", + params![tenant, session_id], + ) + .await + .map_err(storage_error)?; + + let mut out = Vec::new(); + while let Some(row) = rows.next().await.map_err(storage_error)? { + out.push(row.get::(0).map_err(storage_error)?); + } + Ok(out) + } + /// Upsert Cedar policy text for a tenant. #[instrument(skip_all, fields(tenant, otel.name = "turso.upsert_tenant_policy"))] pub async fn upsert_tenant_policy( diff --git a/crates/temper-store-turso/src/store/blobs.rs b/crates/temper-store-turso/src/store/blobs.rs index 019457d08..ecaeac402 100644 --- a/crates/temper-store-turso/src/store/blobs.rs +++ b/crates/temper-store-turso/src/store/blobs.rs @@ -136,15 +136,46 @@ impl TursoEventStore { /// Retrieve a blob by key. Returns `None` if not found. pub async fn get_blob(&self, key: &str) -> Result>, String> { + self.get_blob_with_size_predicate(key, None).await + } + + /// Retrieve a blob only when the durable size metadata fits the caller's + /// allocation budget. `None` means missing or larger than `max_bytes`. + pub async fn get_blob_if_size_at_most( + &self, + key: &str, + max_bytes: usize, + ) -> Result>, String> { + let max_bytes = i64::try_from(max_bytes) + .map_err(|_| "legacy blob read budget exceeds i64".to_string())?; + self.get_blob_with_size_predicate(key, Some(max_bytes)) + .await + } + + async fn get_blob_with_size_predicate( + &self, + key: &str, + max_bytes: Option, + ) -> Result>, String> { for attempt in 1..=BLOB_STORE_ATTEMPTS { let conn = self .configured_connection() .await .map_err(|e| e.to_string())?; - let mut rows = match conn - .query("SELECT data FROM blobs WHERE blob_key = ?1", params![key]) - .await - { + let query = match max_bytes { + Some(max_bytes) => { + conn.query( + "SELECT data FROM blobs WHERE blob_key = ?1 AND length(data) <= ?2", + params![key, max_bytes], + ) + .await + } + None => { + conn.query("SELECT data FROM blobs WHERE blob_key = ?1", params![key]) + .await + } + }; + let mut rows = match query { Ok(rows) => rows, Err(error) => { let message = error.to_string(); diff --git a/crates/temper-store-turso/src/store/evolution.rs b/crates/temper-store-turso/src/store/evolution.rs index ee220eb1c..736955614 100644 --- a/crates/temper-store-turso/src/store/evolution.rs +++ b/crates/temper-store-turso/src/store/evolution.rs @@ -5,6 +5,7 @@ use temper_runtime::persistence::{PersistenceError, storage_error}; use tracing::instrument; use super::{DesignTimeEventRow, EvolutionRecordRow, FeatureRequestRow, TursoEventStore}; +use crate::TursoEvolutionRecordInsert; use crate::metrics::TursoQueryTimer; // ----------------------------------------------------------------------- @@ -14,9 +15,10 @@ use crate::metrics::TursoQueryTimer; impl TursoEventStore { /// Upsert a feature request. #[allow(clippy::too_many_arguments)] - #[instrument(skip_all, fields(id, otel.name = "turso.upsert_feature_request"))] + #[instrument(skip_all, fields(tenant, id, otel.name = "turso.upsert_feature_request"))] pub async fn upsert_feature_request( &self, + tenant: &str, id: &str, category: &str, description: &str, @@ -27,34 +29,43 @@ impl TursoEventStore { ) -> Result<(), PersistenceError> { let _query_timer = TursoQueryTimer::start("turso.upsert_feature_request"); let conn = self.configured_connection().await?; - conn.execute( - "INSERT INTO feature_requests (id, category, description, frequency, trajectory_refs, disposition, developer_notes, updated_at) \ - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, datetime('now')) \ + let affected = conn.execute( + "INSERT INTO feature_requests (id, tenant, category, description, frequency, trajectory_refs, disposition, developer_notes, updated_at) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, datetime('now')) \ ON CONFLICT(id) DO UPDATE SET \ - category = ?2, description = ?3, frequency = ?4, trajectory_refs = ?5, \ - disposition = ?6, developer_notes = ?7, updated_at = datetime('now')", - params![id, category, description, frequency, trajectory_refs_json, disposition, developer_notes], + category = excluded.category, description = excluded.description, \ + frequency = excluded.frequency, trajectory_refs = excluded.trajectory_refs, \ + disposition = excluded.disposition, developer_notes = excluded.developer_notes, \ + updated_at = datetime('now') \ + WHERE feature_requests.tenant = excluded.tenant", + params![id, tenant, category, description, frequency, trajectory_refs_json, disposition, developer_notes], ) .await .map_err(storage_error)?; + if affected == 0 { + return Err(PersistenceError::Storage(format!( + "feature request '{id}' is owned by another tenant" + ))); + } Ok(()) } /// List feature requests with optional disposition filter. - #[instrument(skip_all, fields(otel.name = "turso.list_feature_requests"))] + #[instrument(skip_all, fields(tenant, otel.name = "turso.list_feature_requests"))] pub async fn list_feature_requests( &self, + tenant: &str, disposition: Option<&str>, ) -> Result, PersistenceError> { let _query_timer = TursoQueryTimer::start("turso.list_feature_requests"); let conn = self.configured_connection().await?; let mut rows = conn .query( - "SELECT id, category, description, frequency, trajectory_refs, disposition, developer_notes, created_at, updated_at \ + "SELECT id, tenant, category, description, frequency, trajectory_refs, disposition, developer_notes, created_at, updated_at \ FROM feature_requests \ - WHERE (?1 IS NULL OR disposition = ?1) \ + WHERE tenant = ?1 AND (?2 IS NULL OR disposition = ?2) \ ORDER BY frequency DESC, created_at DESC", - params![disposition], + params![tenant, disposition], ) .await .map_err(storage_error)?; @@ -63,23 +74,25 @@ impl TursoEventStore { while let Some(row) = rows.next().await.map_err(storage_error)? { out.push(FeatureRequestRow { id: row.get::(0).map_err(storage_error)?, - category: row.get::(1).map_err(storage_error)?, - description: row.get::(2).map_err(storage_error)?, - frequency: row.get::(3).map_err(storage_error)?, - trajectory_refs: row.get::(4).map_err(storage_error)?, - disposition: row.get::(5).map_err(storage_error)?, - developer_notes: row.get::>(6).map_err(storage_error)?, - created_at: row.get::(7).map_err(storage_error)?, - updated_at: row.get::(8).map_err(storage_error)?, + tenant: row.get::(1).map_err(storage_error)?, + category: row.get::(2).map_err(storage_error)?, + description: row.get::(3).map_err(storage_error)?, + frequency: row.get::(4).map_err(storage_error)?, + trajectory_refs: row.get::(5).map_err(storage_error)?, + disposition: row.get::(6).map_err(storage_error)?, + developer_notes: row.get::>(7).map_err(storage_error)?, + created_at: row.get::(8).map_err(storage_error)?, + updated_at: row.get::(9).map_err(storage_error)?, }); } Ok(out) } /// Update a feature request's disposition and developer notes. - #[instrument(skip_all, fields(id, otel.name = "turso.update_feature_request"))] + #[instrument(skip_all, fields(tenant, id, otel.name = "turso.update_feature_request"))] pub async fn update_feature_request( &self, + tenant: &str, id: &str, disposition: &str, developer_notes: Option<&str>, @@ -88,9 +101,9 @@ impl TursoEventStore { let conn = self.configured_connection().await?; let affected = conn .execute( - "UPDATE feature_requests SET disposition = ?2, developer_notes = ?3, updated_at = datetime('now') \ - WHERE id = ?1", - params![id, disposition, developer_notes], + "UPDATE feature_requests SET disposition = ?3, developer_notes = ?4, updated_at = datetime('now') \ + WHERE tenant = ?1 AND id = ?2", + params![tenant, id, disposition, developer_notes], ) .await .map_err(storage_error)?; @@ -102,23 +115,35 @@ impl TursoEventStore { // ----------------------------------------------------------------------- /// Insert an evolution record. - #[instrument(skip_all, fields(id, record_type, otel.name = "turso.insert_evolution_record"))] + #[instrument( + skip_all, + fields( + tenant = record.tenant, + id = record.id, + record_type = record.record_type, + otel.name = "turso.insert_evolution_record" + ) + )] pub async fn insert_evolution_record( &self, - id: &str, - record_type: &str, - status: &str, - created_by: &str, - derived_from: Option<&str>, - data_json: &str, + record: TursoEvolutionRecordInsert<'_>, ) -> Result<(), PersistenceError> { + let TursoEvolutionRecordInsert { + tenant, + id, + record_type, + status, + created_by, + derived_from, + data_json, + } = record; let _query_timer = TursoQueryTimer::start("turso.insert_evolution_record"); let conn = self.configured_connection().await?; let execute_res = conn .execute( - "INSERT INTO evolution_records (id, record_type, status, created_by, derived_from, data, timestamp) \ - VALUES (?1, ?2, ?3, ?4, ?5, ?6, datetime('now'))", - params![id, record_type, status, created_by, derived_from, data_json], + "INSERT INTO evolution_records (id, tenant, record_type, status, created_by, derived_from, data, timestamp) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, datetime('now'))", + params![id, tenant, record_type, status, created_by, derived_from, data_json], ) .await .map_err(storage_error); @@ -146,18 +171,19 @@ impl TursoEventStore { } /// Get a single evolution record by ID. - #[instrument(skip_all, fields(id, otel.name = "turso.get_evolution_record"))] + #[instrument(skip_all, fields(tenant, id, otel.name = "turso.get_evolution_record"))] pub async fn get_evolution_record( &self, + tenant: &str, id: &str, ) -> Result, PersistenceError> { let _query_timer = TursoQueryTimer::start("turso.get_evolution_record"); let conn = self.configured_connection().await?; let mut rows = conn .query( - "SELECT id, record_type, status, created_by, derived_from, data, timestamp \ - FROM evolution_records WHERE id = ?1", - params![id], + "SELECT id, tenant, record_type, status, created_by, derived_from, data, timestamp \ + FROM evolution_records WHERE tenant = ?1 AND id = ?2", + params![tenant, id], ) .await .map_err(storage_error)?; @@ -178,9 +204,10 @@ impl TursoEventStore { } /// List evolution records with optional type and status filters. - #[instrument(skip_all, fields(otel.name = "turso.list_evolution_records"))] + #[instrument(skip_all, fields(tenant, otel.name = "turso.list_evolution_records"))] pub async fn list_evolution_records( &self, + tenant: &str, record_type: Option<&str>, status: Option<&str>, ) -> Result, PersistenceError> { @@ -188,12 +215,13 @@ impl TursoEventStore { let conn = self.configured_connection().await?; let mut rows = conn .query( - "SELECT id, record_type, status, created_by, derived_from, data, timestamp \ + "SELECT id, tenant, record_type, status, created_by, derived_from, data, timestamp \ FROM evolution_records \ - WHERE (?1 IS NULL OR record_type = ?1) \ - AND (?2 IS NULL OR status = ?2) \ + WHERE tenant = ?1 \ + AND (?2 IS NULL OR record_type = ?2) \ + AND (?3 IS NULL OR status = ?3) \ ORDER BY timestamp DESC", - params![record_type, status], + params![tenant, record_type, status], ) .await .map_err(storage_error)?; @@ -212,25 +240,15 @@ impl TursoEventStore { } /// List ranked insights (Insight type, sorted by priority_score in data). - #[instrument(skip_all, fields(otel.name = "turso.list_ranked_insights"))] - pub async fn list_ranked_insights(&self) -> Result, PersistenceError> { + #[instrument(skip_all, fields(tenant, otel.name = "turso.list_ranked_insights"))] + pub async fn list_ranked_insights( + &self, + tenant: &str, + ) -> Result, PersistenceError> { let _query_timer = TursoQueryTimer::start("turso.list_ranked_insights"); - let conn = self.configured_connection().await?; - let mut rows = conn - .query( - "SELECT id, record_type, status, created_by, derived_from, data, timestamp \ - FROM evolution_records \ - WHERE record_type = 'Insight' \ - ORDER BY timestamp DESC", - (), - ) - .await - .map_err(storage_error)?; - - let mut out = Vec::new(); - while let Some(row) = rows.next().await.map_err(storage_error)? { - out.push(Self::row_to_evolution_record(&row)?); - } + let mut out = self + .list_evolution_records(tenant, Some("Insight"), None) + .await?; // Sort by priority_score descending (extracted from JSON data). out.sort_by(|a, b| { let score_a = serde_json::from_str::(&a.data) @@ -255,12 +273,13 @@ impl TursoEventStore { ) -> Result { Ok(EvolutionRecordRow { id: row.get::(0).map_err(storage_error)?, - record_type: row.get::(1).map_err(storage_error)?, - status: row.get::(2).map_err(storage_error)?, - created_by: row.get::(3).map_err(storage_error)?, - derived_from: row.get::>(4).map_err(storage_error)?, - data: row.get::(5).map_err(storage_error)?, - timestamp: row.get::(6).map_err(storage_error)?, + tenant: row.get::(1).map_err(storage_error)?, + record_type: row.get::(2).map_err(storage_error)?, + status: row.get::(3).map_err(storage_error)?, + created_by: row.get::(4).map_err(storage_error)?, + derived_from: row.get::>(5).map_err(storage_error)?, + data: row.get::(6).map_err(storage_error)?, + timestamp: row.get::(7).map_err(storage_error)?, }) } diff --git a/crates/temper-store-turso/src/store/migration_support.rs b/crates/temper-store-turso/src/store/migration_support.rs new file mode 100644 index 000000000..a0f4044ed --- /dev/null +++ b/crates/temper-store-turso/src/store/migration_support.rs @@ -0,0 +1,21 @@ +use temper_runtime::persistence::{PersistenceError, storage_error}; + +use super::instrumentation::InstrumentedConnection; + +/// Execute a SQLite `ADD COLUMN` migration while rejecting every error except +/// the duplicate-column result expected on an already-migrated database. +pub(super) async fn add_column_if_missing( + connection: &InstrumentedConnection, + statement: &str, +) -> Result<(), PersistenceError> { + if let Err(error) = connection.execute(statement, ()).await { + let message = error.to_string().to_ascii_lowercase(); + if !message.contains("duplicate column") + && !message.contains("already exists") + && !message.contains("already has") + { + return Err(storage_error(error)); + } + } + Ok(()) +} diff --git a/crates/temper-store-turso/src/store/mod.rs b/crates/temper-store-turso/src/store/mod.rs index fba0d839d..fce01e292 100644 --- a/crates/temper-store-turso/src/store/mod.rs +++ b/crates/temper-store-turso/src/store/mod.rs @@ -2,7 +2,7 @@ //! //! Split into domain-focused sub-modules for cohesion: //! - [`specs`]: Spec CRUD (upsert, verification, load) -//! - [`trajectory`]: Trajectory persistence and queries +//! - [`trajectory`] / [`trajectory_queries`]: Trajectory writes and reads //! - [`evolution`]: Feature requests, evolution records, design-time events //! - [`authz`]: Authorization decisions and Cedar policies //! - [`wasm`]: WASM module storage and invocation logs @@ -27,8 +27,10 @@ mod event_store; mod evolution; pub mod field_index; mod instrumentation; +mod migration_support; pub mod ots; mod policy; +mod policy_approval; mod published_artifacts; mod query_page; mod secrets; @@ -36,6 +38,7 @@ mod specs; #[cfg(test)] mod tests; mod trajectory; +mod trajectory_queries; mod wasm; mod write_gate; @@ -264,6 +267,19 @@ impl TursoEventStore { conn.execute(schema::CREATE_EVOLUTION_RECORDS_TABLE, ()) .await .map_err(storage_error)?; + migration_support::add_column_if_missing(&conn, schema::ALTER_FEATURE_REQUESTS_ADD_TENANT) + .await?; + migration_support::add_column_if_missing(&conn, schema::ALTER_EVOLUTION_RECORDS_ADD_TENANT) + .await?; + conn.execute(schema::CREATE_FEATURE_REQUESTS_TENANT_INDEX, ()) + .await + .map_err(storage_error)?; + conn.execute(schema::CREATE_EVOLUTION_RECORDS_TENANT_INDEX, ()) + .await + .map_err(storage_error)?; + conn.execute(schema::CREATE_EVOLUTION_RECORDS_TENANT_PARENT_INDEX, ()) + .await + .map_err(storage_error)?; conn.execute(schema::CREATE_EVOLUTION_RECORDS_TYPE_INDEX, ()) .await .map_err(storage_error)?; @@ -280,10 +296,6 @@ impl TursoEventStore { .await .map_err(storage_error)?; - conn.execute(schema::CREATE_TENANT_SECRETS_TABLE, ()) - .await - .map_err(storage_error)?; - // Specs table extensions — add content_hash column for verification caching. let _ = conn.execute(schema::ALTER_SPECS_ADD_CONTENT_HASH, ()).await; let _ = conn.execute(schema::ALTER_SPECS_ADD_COMMITTED, ()).await; @@ -301,12 +313,16 @@ impl TursoEventStore { schema::ALTER_TRAJECTORIES_ADD_REQUEST_BODY, schema::ALTER_TRAJECTORIES_ADD_INTENT, schema::ALTER_TRAJECTORIES_ADD_MATCHED_POLICY_IDS, + schema::ALTER_TRAJECTORIES_ADD_CAPTURE_SEQ, ] { let _ = conn.execute(stmt, ()).await; // ignore "duplicate column" errors } conn.execute(schema::CREATE_TRAJECTORIES_AGENT_INDEX, ()) .await .map_err(storage_error)?; + conn.execute(schema::CREATE_TRAJECTORIES_SESSION_INDEX, ()) + .await + .map_err(storage_error)?; // OTS trajectory storage — full agent execution traces for GEPA. conn.execute(schema::CREATE_OTS_TRAJECTORIES_TABLE, ()) @@ -320,6 +336,10 @@ impl TursoEventStore { ] { let _ = conn.execute(stmt, ()).await; } + // Runs after the column migrations, so the rebuild copies a table that + // already has every column, and before the indexes, which the rebuild + // drops along with the old table. + Self::rebuild_ots_trajectories_for_tenant_identity(&conn).await?; conn.execute(schema::CREATE_OTS_TRAJECTORIES_AGENT_INDEX, ()) .await .map_err(storage_error)?; @@ -409,6 +429,69 @@ impl TursoEventStore { Ok(()) } + /// Rekey an existing `ots_trajectories` table from `trajectory_id` alone to + /// `(tenant, trajectory_id)`. + /// + /// SQLite has no `ALTER TABLE ... PRIMARY KEY`, so the only way to change a + /// key is to rebuild the table. The rebuild runs once: it reads the stored + /// DDL first and returns immediately on a table that is already scoped, so + /// it costs one `sqlite_master` read per startup after that. + /// + /// All four steps run in one transaction. Run as separate statements, a + /// process that died between the `DROP` and the `RENAME` would leave no + /// `ots_trajectories` at all: the next boot's `CREATE TABLE IF NOT EXISTS` + /// would make an empty one, already carrying the tenant-scoped marker, so + /// this function would skip the rebuild and every stored trajectory would + /// stay stranded in `ots_trajectories_rebuild`. SQLite's DDL is + /// transactional, so the whole rekey either lands or does not. + async fn rebuild_ots_trajectories_for_tenant_identity( + conn: &InstrumentedConnection, + ) -> Result<(), PersistenceError> { + // Scoped so the cursor is closed before the rebuild runs: an open read + // on `sqlite_master` holds a lock that `DROP TABLE` cannot take. + let ddl = { + let mut rows = conn + .query(schema::SELECT_OTS_TRAJECTORIES_DDL, ()) + .await + .map_err(storage_error)?; + match rows.next().await.map_err(storage_error)? { + Some(row) => row.get::(0).unwrap_or_default(), + // No table at all: the CREATE above did not run, so there is + // nothing to rebuild and nothing to check. + None => return Ok(()), + } + }; + if ddl.contains(schema::OTS_TRAJECTORIES_TENANT_IDENTITY_MARKER) { + return Ok(()); + } + + tracing::info!("rekeying ots_trajectories by (tenant, trajectory_id)"); + conn.execute("BEGIN IMMEDIATE", ()) + .await + .map_err(storage_error)?; + for stmt in [ + schema::CREATE_OTS_TRAJECTORIES_REBUILD_TABLE, + schema::COPY_OTS_TRAJECTORIES_INTO_REBUILD, + schema::DROP_OTS_TRAJECTORIES_LEGACY_TABLE, + schema::RENAME_OTS_TRAJECTORIES_REBUILD, + ] { + if let Err(error) = conn.execute(stmt, ()).await { + // Leaving the transaction open would hold a write lock for the + // life of the connection, so the rollback runs before the + // error is returned and its own failure is reported alongside. + if let Err(rollback) = conn.execute("ROLLBACK", ()).await { + tracing::error!( + error = %rollback, + "failed to roll back the ots_trajectories rekey" + ); + } + return Err(storage_error(error)); + } + } + conn.execute("COMMIT", ()).await.map_err(storage_error)?; + Ok(()) + } + /// Obtain a connection handle to the underlying database. /// /// `Database::connect()` returns a lightweight handle, **not** a fresh TCP @@ -581,6 +664,11 @@ pub struct TursoTrajectoryRow { pub intent: Option, /// Cedar policy IDs that contributed to the authorization decision (JSON array). pub matched_policy_ids: Option>, + /// Monotonic capture order stamped by the process that recorded the row. + /// + /// Null on rows written before the column existed; see + /// [`crate::schema::ALTER_TRAJECTORIES_ADD_CAPTURE_SEQ`]. + pub capture_seq: Option, } /// Aggregated trajectory statistics. @@ -655,6 +743,8 @@ pub struct UnmetIntentAggRow { pub struct FeatureRequestRow { /// Feature request ID. pub id: String, + /// Tenant that owns the feature request. + pub tenant: String, /// Category label. pub category: String, /// Description of the feature request. @@ -678,6 +768,8 @@ pub struct FeatureRequestRow { pub struct EvolutionRecordRow { /// Record ID. pub id: String, + /// Tenant that owns the record chain. + pub tenant: String, /// Record type: Observation, Problem, Analysis, Decision, Insight. pub record_type: String, /// Status: Open, Resolved, Superseded, Rejected. diff --git a/crates/temper-store-turso/src/store/ots.rs b/crates/temper-store-turso/src/store/ots.rs index 866758c68..602f05f12 100644 --- a/crates/temper-store-turso/src/store/ots.rs +++ b/crates/temper-store-turso/src/store/ots.rs @@ -23,6 +23,22 @@ pub struct OtsTrajectoryRow { pub updated_at: String, } +/// A stored OTS trajectory document together with the run identity recorded +/// alongside it. +/// +/// The document itself carries no session or tenant — those live on the row — +/// so any consumer that needs the run identity would otherwise have to list +/// the table to find what it already asked for by id. +#[derive(Debug, Clone, serde::Serialize)] +pub struct OtsTrajectoryDocument { + pub trajectory_id: String, + pub tenant: String, + pub agent_id: String, + pub session_id: String, + pub outcome: String, + pub data: String, +} + /// Durable queued OTS trajectory row ready for outbox replay. #[derive(Debug, Clone, serde::Serialize)] pub struct OtsQueuedTrajectoryRow { @@ -49,6 +65,11 @@ pub struct OtsTrajectoryParams<'a> { impl TursoEventStore { /// Persist a full OTS trajectory JSON blob. + /// + /// Identity is `(tenant, trajectory_id)`: the id comes from the uploading + /// harness and one store holds every tenant's rows, so keying on the id + /// alone would let one tenant's upload replace another's row — tenant + /// column included. #[instrument(skip_all, fields( otel.name = "turso.persist_ots_trajectory", trajectory_id = %p.trajectory_id, @@ -61,9 +82,14 @@ impl TursoEventStore { let _timer = TursoQueryTimer::start("turso.persist_ots_trajectory"); let conn = self.connection()?; conn.execute( - "INSERT OR REPLACE INTO ots_trajectories \ + "INSERT INTO ots_trajectories \ (trajectory_id, tenant, agent_id, session_id, outcome, turn_count, data, persistence_status, persist_attempts, last_error, created_at, updated_at) \ - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, 'persisted', 0, NULL, datetime('now'), datetime('now'))", + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, 'persisted', 0, NULL, datetime('now'), datetime('now')) \ + ON CONFLICT(tenant, trajectory_id) DO UPDATE SET \ + agent_id = excluded.agent_id, session_id = excluded.session_id, \ + outcome = excluded.outcome, turn_count = excluded.turn_count, data = excluded.data, \ + persistence_status = 'persisted', persist_attempts = 0, last_error = NULL, \ + updated_at = datetime('now')", params![ p.trajectory_id.to_string(), p.tenant.to_string(), @@ -95,8 +121,8 @@ impl TursoEventStore { "INSERT INTO ots_trajectories \ (trajectory_id, tenant, agent_id, session_id, outcome, turn_count, data, persistence_status, persist_attempts, last_error, created_at, updated_at) \ VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, 'queued', 0, NULL, datetime('now'), datetime('now')) \ - ON CONFLICT(trajectory_id) DO UPDATE SET \ - tenant = excluded.tenant, agent_id = excluded.agent_id, session_id = excluded.session_id, \ + ON CONFLICT(tenant, trajectory_id) DO UPDATE SET \ + agent_id = excluded.agent_id, session_id = excluded.session_id, \ outcome = excluded.outcome, turn_count = excluded.turn_count, data = excluded.data, \ persistence_status = 'queued', last_error = NULL, updated_at = datetime('now')", params![ @@ -115,8 +141,13 @@ impl TursoEventStore { } /// Mark a queued OTS trajectory as persisted. + /// + /// Addressed by the same `(tenant, trajectory_id)` identity the row is + /// keyed by: two tenants may hold the same id, and an unscoped update would + /// declare both of them persisted. pub async fn mark_ots_trajectory_persisted( &self, + tenant: &str, trajectory_id: &str, ) -> Result<(), PersistenceError> { let _timer = TursoQueryTimer::start("turso.mark_ots_trajectory_persisted"); @@ -124,8 +155,8 @@ impl TursoEventStore { conn.execute( "UPDATE ots_trajectories \ SET persistence_status = 'persisted', last_error = NULL, updated_at = datetime('now') \ - WHERE trajectory_id = ?1", - params![trajectory_id.to_string()], + WHERE tenant = ?1 AND trajectory_id = ?2", + params![tenant.to_string(), trajectory_id.to_string()], ) .await .map_err(storage_error)?; @@ -135,6 +166,7 @@ impl TursoEventStore { /// Mark a queued OTS trajectory as failed after retries exhaust. pub async fn mark_ots_trajectory_failed( &self, + tenant: &str, trajectory_id: &str, error: &str, ) -> Result<(), PersistenceError> { @@ -142,9 +174,13 @@ impl TursoEventStore { let conn = self.connection()?; conn.execute( "UPDATE ots_trajectories \ - SET persistence_status = 'failed', persist_attempts = persist_attempts + 1, last_error = ?2, updated_at = datetime('now') \ - WHERE trajectory_id = ?1", - params![trajectory_id.to_string(), error.to_string()], + SET persistence_status = 'failed', persist_attempts = persist_attempts + 1, last_error = ?3, updated_at = datetime('now') \ + WHERE tenant = ?1 AND trajectory_id = ?2", + params![ + tenant.to_string(), + trajectory_id.to_string(), + error.to_string() + ], ) .await .map_err(storage_error)?; @@ -245,25 +281,37 @@ impl TursoEventStore { Ok(result) } - /// Load full OTS trajectory data by ID. + /// Load full OTS trajectory data by tenant and ID. + /// + /// The tenant is part of the lookup rather than a post-filter: one store + /// can hold every tenant's rows, so a caller that takes the trajectory id + /// from a request path would otherwise read across tenants. #[instrument(skip_all, fields(otel.name = "turso.get_ots_trajectory"))] pub async fn get_ots_trajectory( &self, + tenant: &str, trajectory_id: &str, - ) -> Result, PersistenceError> { + ) -> Result, PersistenceError> { let _timer = TursoQueryTimer::start("turso.get_ots_trajectory"); let conn = self.connection()?; let mut rows = conn .query( - "SELECT data FROM ots_trajectories WHERE trajectory_id = ?1", - params![trajectory_id.to_string()], + "SELECT agent_id, COALESCE(session_id, ''), outcome, data \ + FROM ots_trajectories WHERE tenant = ?1 AND trajectory_id = ?2", + params![tenant.to_string(), trajectory_id.to_string()], ) .await .map_err(storage_error)?; if let Some(row) = rows.next().await.map_err(storage_error)? { - let data: String = row.get(0).unwrap_or_default(); - Ok(Some(data)) + Ok(Some(OtsTrajectoryDocument { + trajectory_id: trajectory_id.to_string(), + tenant: tenant.to_string(), + agent_id: row.get(0).unwrap_or_default(), + session_id: row.get(1).unwrap_or_default(), + outcome: row.get(2).unwrap_or_default(), + data: row.get(3).unwrap_or_default(), + })) } else { Ok(None) } @@ -271,81 +319,5 @@ impl TursoEventStore { } #[cfg(test)] -mod tests { - use super::*; - - async fn test_store() -> (TursoEventStore, tempfile::TempDir) { - let dir = tempfile::tempdir().expect("create temp dir"); - let db_path = dir.path().join("ots-outbox.db"); - let db_url = format!("file:{}", db_path.display()); - let store = TursoEventStore::new(&db_url, None) - .await - .expect("create local turso store"); - (store, dir) - } - - fn params<'a>(trajectory_id: &'a str, data: &'a str) -> OtsTrajectoryParams<'a> { - OtsTrajectoryParams { - trajectory_id, - tenant: "tenant", - agent_id: "agent", - session_id: "session", - outcome: "success", - turn_count: 2, - data, - } - } - - #[tokio::test] - async fn ots_outbox_status_lifecycle_is_durable() { - let (store, _dir) = test_store().await; - let data = r#"{"trajectory_id":"traj-durable","turns":[]}"#; - - store - .enqueue_ots_trajectory(¶ms("traj-durable", data)) - .await - .expect("enqueue trajectory"); - - let rows = store - .list_ots_trajectories("tenant", None, None, 10) - .await - .expect("list trajectories"); - assert_eq!(rows.len(), 1); - assert_eq!(rows[0].persistence_status, "queued"); - - let queued = store - .list_queued_ots_trajectories(10) - .await - .expect("list queued trajectories"); - assert_eq!(queued.len(), 1); - assert_eq!(queued[0].trajectory_id, "traj-durable"); - assert_eq!(queued[0].data, data); - - store - .mark_ots_trajectory_persisted("traj-durable") - .await - .expect("mark persisted"); - let rows = store - .list_ots_trajectories("tenant", None, None, 10) - .await - .expect("list persisted trajectory"); - assert_eq!(rows[0].persistence_status, "persisted"); - assert!(rows[0].last_error.is_none()); - - store - .enqueue_ots_trajectory(¶ms("traj-durable", data)) - .await - .expect("requeue trajectory"); - store - .mark_ots_trajectory_failed("traj-durable", "transient") - .await - .expect("mark failed"); - let rows = store - .list_ots_trajectories("tenant", None, None, 10) - .await - .expect("list failed trajectory"); - assert_eq!(rows[0].persistence_status, "failed"); - assert_eq!(rows[0].persist_attempts, 1); - assert_eq!(rows[0].last_error.as_deref(), Some("transient")); - } -} +#[path = "ots_test.rs"] +mod tests; diff --git a/crates/temper-store-turso/src/store/ots_test.rs b/crates/temper-store-turso/src/store/ots_test.rs new file mode 100644 index 000000000..666a4a83a --- /dev/null +++ b/crates/temper-store-turso/src/store/ots_test.rs @@ -0,0 +1,252 @@ +use super::*; + +async fn test_store() -> (TursoEventStore, tempfile::TempDir) { + let dir = tempfile::tempdir().expect("create temp dir"); + let db_path = dir.path().join("ots-outbox.db"); + let db_url = format!("file:{}", db_path.display()); + let store = TursoEventStore::new(&db_url, None) + .await + .expect("create local turso store"); + (store, dir) +} + +fn params<'a>(trajectory_id: &'a str, data: &'a str) -> OtsTrajectoryParams<'a> { + tenant_params("tenant", trajectory_id, data) +} + +fn tenant_params<'a>( + tenant: &'a str, + trajectory_id: &'a str, + data: &'a str, +) -> OtsTrajectoryParams<'a> { + OtsTrajectoryParams { + trajectory_id, + tenant, + agent_id: "agent", + session_id: "session", + outcome: "success", + turn_count: 2, + data, + } +} + +#[tokio::test] +async fn ots_outbox_status_lifecycle_is_durable() { + let (store, _dir) = test_store().await; + let data = r#"{"trajectory_id":"traj-durable","turns":[]}"#; + + store + .enqueue_ots_trajectory(¶ms("traj-durable", data)) + .await + .expect("enqueue trajectory"); + + let rows = store + .list_ots_trajectories("tenant", None, None, 10) + .await + .expect("list trajectories"); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].persistence_status, "queued"); + + let queued = store + .list_queued_ots_trajectories(10) + .await + .expect("list queued trajectories"); + assert_eq!(queued.len(), 1); + assert_eq!(queued[0].trajectory_id, "traj-durable"); + assert_eq!(queued[0].data, data); + + store + .mark_ots_trajectory_persisted("tenant", "traj-durable") + .await + .expect("mark persisted"); + let rows = store + .list_ots_trajectories("tenant", None, None, 10) + .await + .expect("list persisted trajectory"); + assert_eq!(rows[0].persistence_status, "persisted"); + assert!(rows[0].last_error.is_none()); + + store + .enqueue_ots_trajectory(¶ms("traj-durable", data)) + .await + .expect("requeue trajectory"); + store + .mark_ots_trajectory_failed("tenant", "traj-durable", "transient") + .await + .expect("mark failed"); + let rows = store + .list_ots_trajectories("tenant", None, None, 10) + .await + .expect("list failed trajectory"); + assert_eq!(rows[0].persistence_status, "failed"); + assert_eq!(rows[0].persist_attempts, 1); + assert_eq!(rows[0].last_error.as_deref(), Some("transient")); +} + +#[tokio::test] +async fn get_ots_trajectory_is_scoped_to_its_tenant() { + let (store, _dir) = test_store().await; + let data = r#"{"trajectory_id":"traj-tenant-a","turns":[]}"#; + store + .persist_ots_trajectory(¶ms("traj-tenant-a", data)) + .await + .expect("persist trajectory"); + + let document = store + .get_ots_trajectory("tenant", "traj-tenant-a") + .await + .expect("read own tenant") + .expect("document present"); + assert_eq!(document.data, data); + assert_eq!(document.session_id, "session"); + assert_eq!(document.agent_id, "agent"); + assert!( + store + .get_ots_trajectory("other-tenant", "traj-tenant-a") + .await + .expect("read foreign tenant") + .is_none(), + "a foreign tenant must not read another tenant's trajectory by id" + ); +} + +#[tokio::test] +async fn one_tenant_s_upload_cannot_replace_another_s_row_with_the_same_id() { + // The trajectory id is chosen by the uploading harness, so two tenants + // colliding on one is an ordinary event, not an attack. + let (store, _dir) = test_store().await; + let alpha = r#"{"trajectory_id":"traj-shared","turns":[],"owner":"alpha"}"#; + let beta = r#"{"trajectory_id":"traj-shared","turns":[],"owner":"beta"}"#; + + store + .persist_ots_trajectory(&tenant_params("alpha", "traj-shared", alpha)) + .await + .expect("persist alpha"); + store + .persist_ots_trajectory(&tenant_params("beta", "traj-shared", beta)) + .await + .expect("persist beta"); + + let alpha_row = store + .get_ots_trajectory("alpha", "traj-shared") + .await + .expect("read alpha") + .expect("alpha still has its row"); + assert_eq!( + alpha_row.data, alpha, + "a second tenant's upload must not overwrite the first tenant's trajectory" + ); + let beta_row = store + .get_ots_trajectory("beta", "traj-shared") + .await + .expect("read beta") + .expect("beta has its own row"); + assert_eq!(beta_row.data, beta); +} + +#[tokio::test] +async fn marking_one_tenant_s_trajectory_leaves_another_s_alone() { + let (store, _dir) = test_store().await; + let data = r#"{"trajectory_id":"traj-shared","turns":[]}"#; + store + .enqueue_ots_trajectory(&tenant_params("alpha", "traj-shared", data)) + .await + .expect("enqueue alpha"); + store + .enqueue_ots_trajectory(&tenant_params("beta", "traj-shared", data)) + .await + .expect("enqueue beta"); + + store + .mark_ots_trajectory_failed("alpha", "traj-shared", "transient") + .await + .expect("mark alpha failed"); + + let beta = store + .list_ots_trajectories("beta", None, None, 10) + .await + .expect("list beta"); + assert_eq!(beta.len(), 1); + assert_eq!( + beta[0].persistence_status, "queued", + "a status update addressed at one tenant must not land on another's row" + ); + assert!(beta[0].last_error.is_none()); +} + +#[tokio::test] +async fn a_globally_keyed_table_is_rekeyed_by_tenant_on_open() { + // Databases created before the identity fix carry a table keyed on + // `trajectory_id` alone. Opening the store has to rekey it, keeping + // the rows it already holds. + let dir = tempfile::tempdir().expect("create temp dir"); + let db_path = dir.path().join("legacy-ots.db"); + let db_url = format!("file:{}", db_path.display()); + + { + let legacy = libsql::Builder::new_local(&db_path) + .build() + .await + .expect("open legacy db"); + let conn = legacy.connect().expect("connect legacy db"); + conn.execute( + "CREATE TABLE ots_trajectories ( + trajectory_id TEXT PRIMARY KEY, + tenant TEXT NOT NULL, + agent_id TEXT NOT NULL, + session_id TEXT, + outcome TEXT NOT NULL DEFAULT 'unknown', + entity_type TEXT, + turn_count INTEGER NOT NULL DEFAULT 0, + data TEXT NOT NULL, + persistence_status TEXT NOT NULL DEFAULT 'persisted', + persist_attempts INTEGER NOT NULL DEFAULT 0, + last_error TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) + );", + (), + ) + .await + .expect("create legacy table"); + conn.execute( + "INSERT INTO ots_trajectories \ + (trajectory_id, tenant, agent_id, session_id, outcome, turn_count, data) \ + VALUES ('traj-legacy', 'alpha', 'agent', 'session', 'success', 1, '{}')", + (), + ) + .await + .expect("seed legacy row"); + } + + let store = TursoEventStore::new(&db_url, None) + .await + .expect("open the store, running the rekey"); + + assert!( + store + .get_ots_trajectory("alpha", "traj-legacy") + .await + .expect("read the carried-over row") + .is_some(), + "the rekey must carry existing rows across" + ); + store + .persist_ots_trajectory(&tenant_params( + "beta", + "traj-legacy", + "{\"owner\":\"beta\"}", + )) + .await + .expect("a second tenant can now hold the same id"); + assert_eq!( + store + .get_ots_trajectory("alpha", "traj-legacy") + .await + .expect("read alpha") + .expect("alpha kept its row") + .data, + "{}", + "the rekeyed table must not let one tenant clobber another" + ); +} diff --git a/crates/temper-store-turso/src/store/policy.rs b/crates/temper-store-turso/src/store/policy.rs index 09dba4674..b89f221ba 100644 --- a/crates/temper-store-turso/src/store/policy.rs +++ b/crates/temper-store-turso/src/store/policy.rs @@ -392,7 +392,7 @@ impl TursoEventStore { /// /// Identical inputs always produce the same digest, enabling cheap change /// detection before issuing an expensive Turso write. -fn compute_policy_hash(cedar_text: &str) -> String { +pub(super) fn compute_policy_hash(cedar_text: &str) -> String { let mut hasher = Sha256::new(); hasher.update(cedar_text.as_bytes()); format!("{:x}", hasher.finalize()) diff --git a/crates/temper-store-turso/src/store/policy_approval.rs b/crates/temper-store-turso/src/store/policy_approval.rs new file mode 100644 index 000000000..3e6d11887 --- /dev/null +++ b/crates/temper-store-turso/src/store/policy_approval.rs @@ -0,0 +1,104 @@ +//! Transactional persistence for Cedar policy approvals. + +use libsql::{TransactionBehavior, params}; +use temper_runtime::persistence::{PersistenceError, storage_error}; + +use super::TursoEventStore; +use super::policy::compute_policy_hash; +use crate::TursoPolicyApprovalCommit; + +impl TursoEventStore { + /// Atomically insert an approved policy and transition its decision. + pub async fn commit_policy_approval( + &self, + commit: TursoPolicyApprovalCommit<'_>, + ) -> Result<(), PersistenceError> { + let TursoPolicyApprovalCommit { + tenant, + decision_id, + approved_decision_json, + policy_id, + cedar_text, + created_by, + } = commit; + let policy_hash = compute_policy_hash(cedar_text); + let connection = self.configured_connection().await?; + let transaction = connection + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .map_err(storage_error)?; + + let policy_rows = transaction + .execute( + "INSERT INTO policies \ + (tenant, policy_id, cedar_text, policy_hash, created_at, created_by, enabled) \ + VALUES (?1, ?2, ?3, ?4, datetime('now'), ?5, 1) \ + ON CONFLICT(tenant, policy_id) DO NOTHING", + params![tenant, policy_id, cedar_text, policy_hash, created_by], + ) + .await + .map_err(storage_error)?; + if policy_rows != 1 { + return Err(PersistenceError::Storage(format!( + "policy approval '{policy_id}' already exists" + ))); + } + + let decision_rows = transaction + .execute( + "UPDATE pending_decisions \ + SET status = 'approved', data = ?3, updated_at = datetime('now') \ + WHERE tenant = ?1 AND id = ?2 AND status = 'pending'", + params![tenant, decision_id, approved_decision_json], + ) + .await + .map_err(storage_error)?; + if decision_rows != 1 { + return Err(PersistenceError::Storage(format!( + "pending decision '{decision_id}' was not available for approval" + ))); + } + + transaction.commit().await.map_err(storage_error) + } + + /// Compensate a committed approval when runtime activation fails. + pub async fn rollback_policy_approval( + &self, + tenant: &str, + decision_id: &str, + pending_decision_json: &str, + policy_id: &str, + ) -> Result<(), PersistenceError> { + let connection = self.configured_connection().await?; + let transaction = connection + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .map_err(storage_error)?; + + transaction + .execute( + "DELETE FROM policies WHERE tenant = ?1 AND policy_id = ?2", + params![tenant, policy_id], + ) + .await + .map_err(storage_error)?; + + let decision_rows = transaction + .execute( + "UPDATE pending_decisions \ + SET status = 'pending', data = ?3, updated_at = datetime('now') \ + WHERE tenant = ?1 AND id = ?2 AND status = 'approved'", + params![tenant, decision_id, pending_decision_json], + ) + .await + .map_err(storage_error)?; + if decision_rows != 1 { + return Err(PersistenceError::Storage(format!( + "approved decision '{decision_id}' was not available for rollback" + ))); + } + + transaction.commit().await.map_err(storage_error) + } +} diff --git a/crates/temper-store-turso/src/store/tests/evolution_tenant.rs b/crates/temper-store-turso/src/store/tests/evolution_tenant.rs new file mode 100644 index 000000000..8990cd815 --- /dev/null +++ b/crates/temper-store-turso/src/store/tests/evolution_tenant.rs @@ -0,0 +1,170 @@ +use crate::TursoTrajectoryInsert; + +use super::{make_store, sqlite_test_url}; + +#[tokio::test] +async fn legacy_evolution_tables_gain_default_tenant_ownership() { + let url = sqlite_test_url("evolution-legacy-migration"); + let path = url.strip_prefix("file:").unwrap(); + let database = libsql::Builder::new_local(path).build().await.unwrap(); + let connection = database.connect().unwrap(); + connection + .execute_batch( + "CREATE TABLE feature_requests (\ + id TEXT PRIMARY KEY, category TEXT NOT NULL, description TEXT NOT NULL, \ + frequency INTEGER NOT NULL DEFAULT 0, trajectory_refs TEXT NOT NULL DEFAULT '[]', \ + disposition TEXT NOT NULL DEFAULT 'Open', developer_notes TEXT, \ + created_at TEXT NOT NULL DEFAULT (datetime('now')), \ + updated_at TEXT NOT NULL DEFAULT (datetime('now'))\ + ); \ + CREATE TABLE evolution_records (\ + id TEXT PRIMARY KEY, record_type TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'Open', \ + created_by TEXT NOT NULL, derived_from TEXT, data TEXT NOT NULL, \ + timestamp TEXT NOT NULL DEFAULT (datetime('now'))\ + ); \ + INSERT INTO feature_requests (id, category, description) \ + VALUES ('legacy-feature', 'Workflow', 'legacy'); \ + INSERT INTO evolution_records (id, record_type, created_by, data) \ + VALUES ('O-legacy', 'Observation', 'legacy', '{}');", + ) + .await + .unwrap(); + drop(connection); + drop(database); + + let store = crate::TursoEventStore::new(&url, None).await.unwrap(); + let features = store.list_feature_requests("default", None).await.unwrap(); + assert_eq!(features[0].id, "legacy-feature"); + assert_eq!(features[0].tenant, "default"); + let records = store + .list_evolution_records("default", None, None) + .await + .unwrap(); + assert_eq!(records[0].id, "O-legacy"); + assert_eq!(records[0].tenant, "default"); +} + +#[tokio::test] +async fn feature_requests_and_evolution_records_are_tenant_owned() { + let store = make_store("evolution-tenant-owned").await; + store + .upsert_feature_request( + "tenant-a", + "feature-a", + "Workflow", + "tenant A request", + 4, + "[]", + "Open", + None, + ) + .await + .unwrap(); + store + .upsert_feature_request( + "tenant-b", + "feature-b", + "Workflow", + "tenant B request", + 9, + "[]", + "Open", + None, + ) + .await + .unwrap(); + + let tenant_a_features = store.list_feature_requests("tenant-a", None).await.unwrap(); + assert_eq!(tenant_a_features.len(), 1); + assert_eq!(tenant_a_features[0].id, "feature-a"); + assert_eq!(tenant_a_features[0].tenant, "tenant-a"); + assert!( + !store + .update_feature_request("tenant-a", "feature-b", "Resolved", None) + .await + .unwrap() + ); + + store + .insert_evolution_record(crate::TursoEvolutionRecordInsert { + tenant: "tenant-a", + id: "O-a", + record_type: "Observation", + status: "Open", + created_by: "test", + derived_from: None, + data_json: "{}", + }) + .await + .unwrap(); + store + .insert_evolution_record(crate::TursoEvolutionRecordInsert { + tenant: "tenant-b", + id: "O-b", + record_type: "Observation", + status: "Open", + created_by: "test", + derived_from: None, + data_json: "{}", + }) + .await + .unwrap(); + + assert!( + store + .get_evolution_record("tenant-a", "O-b") + .await + .unwrap() + .is_none() + ); + let tenant_a_records = store + .list_evolution_records("tenant-a", None, None) + .await + .unwrap(); + assert_eq!(tenant_a_records.len(), 1); + assert_eq!(tenant_a_records[0].id, "O-a"); + assert_eq!(tenant_a_records[0].tenant, "tenant-a"); +} + +#[tokio::test] +async fn trajectory_tenant_predicate_is_applied_before_limit() { + let store = make_store("trajectory-tenant-limit").await; + for (tenant, entity_id, created_at) in [ + ("tenant-a", "a-old", "2026-01-01T00:00:00Z"), + ("tenant-b", "b-new", "2026-02-01T00:00:00Z"), + ] { + store + .persist_trajectory(TursoTrajectoryInsert { + tenant, + entity_type: "Order", + entity_id, + action: "Submit", + success: false, + from_status: None, + to_status: None, + error: Some("unmet"), + agent_id: None, + session_id: None, + authz_denied: Some(false), + denied_resource: None, + denied_module: None, + source: Some("Entity"), + spec_governed: Some(true), + created_at, + request_body: None, + intent: None, + matched_policy_ids: None, + capture_seq: None, + }) + .await + .unwrap(); + } + + let rows = store.load_recent_trajectories("tenant-a", 1).await.unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].entity_id, "a-old"); + assert_eq!(rows[0].tenant, "tenant-a"); + let unmet = store.load_unmet_intent_rows("tenant-a").await.unwrap(); + assert_eq!(unmet.len(), 1); + assert_eq!(unmet[0].count, 1); +} diff --git a/crates/temper-store-turso/src/store/tests/mod.rs b/crates/temper-store-turso/src/store/tests/mod.rs index f75aa4c01..530d17e86 100644 --- a/crates/temper-store-turso/src/store/tests/mod.rs +++ b/crates/temper-store-turso/src/store/tests/mod.rs @@ -9,6 +9,9 @@ use temper_runtime::persistence::{ use super::{PublishedArtifactUpsert, QueryProjectionUpsert, TursoEventStore}; use crate::TursoSpecVerificationUpdate; +mod evolution_tenant; +mod policy_approval; + fn test_envelope(event_type: &str, payload: serde_json::Value) -> PersistenceEnvelope { PersistenceEnvelope { sequence_nr: 0, @@ -39,6 +42,82 @@ async fn make_store(test_name: &str) -> TursoEventStore { .expect("create store") } +#[tokio::test] +async fn trajectory_stats_never_cross_tenants() { + // Each of the three statistics queries reads the shared `trajectories` + // table, and the failed-intent list returns whole rows — error strings and + // entity ids. One unfiltered query hands a tenant another tenant's + // operational detail, so all three are asserted here (ADR-0157). + let store = make_store("trajectory-stats-tenant").await; + + let row = + |tenant: &'static str, action: &'static str, success: bool, error: Option<&'static str>| { + crate::TursoTrajectoryInsert { + tenant, + entity_type: "Order", + entity_id: "order-1", + action, + success, + from_status: None, + to_status: None, + error, + agent_id: Some("agent-1"), + session_id: Some("session-1"), + authz_denied: None, + denied_resource: None, + denied_module: None, + source: Some("Entity"), + spec_governed: Some(true), + created_at: "2026-01-01T00:00:00Z", + request_body: None, + intent: None, + matched_policy_ids: None, + capture_seq: None, + } + }; + + store + .persist_trajectory(row("mine", "MyAction", true, None)) + .await + .expect("mine ok"); + store + .persist_trajectory(row( + "theirs", + "TheirAction", + false, + Some("their-secret-error"), + )) + .await + .expect("theirs err"); + + let stats = store + .query_trajectory_stats("mine", None, None, None, 10) + .await + .expect("stats"); + + assert_eq!(stats.total, 1, "totals must count only the caller's tenant"); + assert!( + stats.by_action.contains_key("MyAction"), + "the caller's own action is missing: {:?}", + stats.by_action.keys().collect::>() + ); + assert!( + !stats.by_action.contains_key("TheirAction"), + "another tenant's action names leaked into by_action" + ); + assert!( + stats.failed_intents.iter().all(|r| r.tenant == "mine"), + "another tenant's failed rows leaked into failed_intents" + ); + assert!( + !stats + .failed_intents + .iter() + .any(|r| r.error.as_deref() == Some("their-secret-error")), + "another tenant's error string leaked" + ); +} + #[tokio::test] async fn append_and_read_events_roundtrip() { let store = make_store("append-read").await; diff --git a/crates/temper-store-turso/src/store/tests/policy_approval.rs b/crates/temper-store-turso/src/store/tests/policy_approval.rs new file mode 100644 index 000000000..b68bef301 --- /dev/null +++ b/crates/temper-store-turso/src/store/tests/policy_approval.rs @@ -0,0 +1,137 @@ +use super::make_store; + +fn decision_json(status: &str) -> String { + serde_json::json!({ + "id": "decision-1", + "tenant": "tenant-a", + "status": status, + }) + .to_string() +} + +#[tokio::test] +async fn pending_decision_upsert_cannot_move_ownership() { + let store = make_store("pending-decision-owner").await; + let pending = decision_json("pending"); + store + .upsert_pending_decision("decision-1", "tenant-a", "pending", &pending) + .await + .unwrap(); + + let error = store + .upsert_pending_decision("decision-1", "tenant-b", "pending", &pending) + .await + .unwrap_err(); + assert!(error.to_string().contains("owned by another tenant")); + assert!( + store + .get_pending_decision("tenant-b", "decision-1") + .await + .unwrap() + .is_none() + ); + assert!( + store + .get_pending_decision("tenant-a", "decision-1") + .await + .unwrap() + .is_some() + ); +} + +#[tokio::test] +async fn policy_and_decision_commit_and_rollback_together() { + let store = make_store("policy-approval-transaction").await; + let pending = decision_json("pending"); + let approved = decision_json("approved"); + store + .upsert_pending_decision("decision-1", "tenant-a", "pending", &pending) + .await + .unwrap(); + + store + .commit_policy_approval(crate::TursoPolicyApprovalCommit { + tenant: "tenant-a", + decision_id: "decision-1", + approved_decision_json: &approved, + policy_id: "decision:decision-1", + cedar_text: "permit(principal, action, resource);", + created_by: "reviewer", + }) + .await + .unwrap(); + + let policies = store.load_policies_for_tenant("tenant-a").await.unwrap(); + assert_eq!(policies.len(), 1); + assert_eq!( + store + .get_pending_decision("tenant-a", "decision-1") + .await + .unwrap() + .unwrap(), + approved + ); + + store + .rollback_policy_approval("tenant-a", "decision-1", &pending, "decision:decision-1") + .await + .unwrap(); + assert!( + store + .load_policies_for_tenant("tenant-a") + .await + .unwrap() + .is_empty() + ); + assert_eq!( + store + .get_pending_decision("tenant-a", "decision-1") + .await + .unwrap() + .unwrap(), + pending + ); +} + +#[tokio::test] +async fn policy_conflict_leaves_decision_pending() { + let store = make_store("policy-approval-conflict").await; + let pending = decision_json("pending"); + store + .upsert_pending_decision("decision-1", "tenant-a", "pending", &pending) + .await + .unwrap(); + store + .save_policy( + "tenant-a", + "decision:decision-1", + "forbid(principal, action, resource);", + "existing", + ) + .await + .unwrap(); + + let error = store + .commit_policy_approval(crate::TursoPolicyApprovalCommit { + tenant: "tenant-a", + decision_id: "decision-1", + approved_decision_json: &decision_json("approved"), + policy_id: "decision:decision-1", + cedar_text: "permit(principal, action, resource);", + created_by: "reviewer", + }) + .await + .unwrap_err(); + assert!(error.to_string().contains("already exists")); + assert_eq!( + store + .get_pending_decision("tenant-a", "decision-1") + .await + .unwrap() + .unwrap(), + pending + ); + let policies = store.load_policies_for_tenant("tenant-a").await.unwrap(); + assert_eq!(policies.len(), 1); + assert!(policies[0].cedar_text.starts_with("forbid")); +} diff --git a/crates/temper-store-turso/src/store/trajectory.rs b/crates/temper-store-turso/src/store/trajectory.rs index c3ac92921..961a62040 100644 --- a/crates/temper-store-turso/src/store/trajectory.rs +++ b/crates/temper-store-turso/src/store/trajectory.rs @@ -1,15 +1,14 @@ -//! Trajectory persistence and query methods. +//! Trajectory persistence. +//! +//! The read path lives in the sibling `trajectory_queries` module. use libsql::params; use std::time::Duration; use temper_runtime::persistence::{PersistenceError, storage_error}; use tracing::instrument; +use super::TursoEventStore; use super::write_gate::WritePriority; -use super::{ - ActionStats, AgentSummary, TrajectoryStats, TursoEventStore, TursoTrajectoryRow, - UnmetIntentAggRow, -}; use crate::TursoTrajectoryInsert; use crate::metrics::TursoQueryTimer; use crate::retry::retry_persistence_with_max_attempts; @@ -49,8 +48,8 @@ impl TursoEventStore { .execute( "INSERT INTO trajectories \ (tenant, entity_type, entity_id, action, success, from_status, to_status, error, \ - agent_id, session_id, authz_denied, denied_resource, denied_module, source, spec_governed, created_at, request_body, intent, matched_policy_ids) \ - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18, ?19)", + agent_id, session_id, authz_denied, denied_resource, denied_module, source, spec_governed, created_at, request_body, intent, matched_policy_ids, capture_seq) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18, ?19, ?20)", params![ entry.tenant, entry.entity_type, @@ -70,7 +69,8 @@ impl TursoEventStore { entry.created_at, entry.request_body, entry.intent, - entry.matched_policy_ids + entry.matched_policy_ids, + entry.capture_seq ], ) .await @@ -125,393 +125,165 @@ impl TursoEventStore { ); Ok(()) } +} - /// Load recent trajectory entries (newest first, up to `limit`). - #[instrument(skip_all, fields(otel.name = "turso.load_recent_trajectories", row_count = tracing::field::Empty))] - pub async fn load_recent_trajectories( - &self, - limit: i64, - ) -> Result, PersistenceError> { - let _query_timer = TursoQueryTimer::start("turso.load_recent_trajectories"); - let conn = self.configured_connection().await?; - let mut rows = conn - .query( - "SELECT tenant, entity_type, entity_id, action, success, from_status, to_status, error, \ - agent_id, session_id, authz_denied, denied_resource, denied_module, source, spec_governed, created_at, request_body, intent, matched_policy_ids \ - FROM trajectories \ - ORDER BY created_at DESC \ - LIMIT ?1", - params![limit], - ) - .await - .map_err(|e| { - let error = storage_error(e); - tracing::warn!(limit, error = %error, "trajectory.store.read"); - error - })?; +fn trajectory_attempt_timeout() -> Duration { + const DEFAULT_TRAJECTORY_ATTEMPT_TIMEOUT_MS: u64 = 1_000; + const MIN_TRAJECTORY_ATTEMPT_TIMEOUT_MS: u64 = 100; - let mut out = Vec::new(); - while let Some(row) = rows.next().await.map_err(storage_error)? { - out.push(Self::row_to_trajectory(&row)?); - } - tracing::Span::current().record("row_count", out.len()); - tracing::debug!(limit, count = out.len(), "trajectory.store.read"); - Ok(out) - } + let configured = std::env::var("TEMPER_TURSO_TRAJECTORY_TIMEOUT_MS") + .ok() + .and_then(|value| value.parse::().ok()) + .unwrap_or(DEFAULT_TRAJECTORY_ATTEMPT_TIMEOUT_MS); - /// Load aggregated unmet-intent failure groups (SQL GROUP BY, ≤100 rows). - /// - /// Returns one row per (entity_type, error) group with counts and - /// timestamps. This replaces the previous pattern of loading up to 10,000 - /// raw trajectory rows and grouping them in Rust. - #[instrument(skip_all, fields(otel.name = "turso.load_unmet_intent_rows", row_count = tracing::field::Empty))] - pub async fn load_unmet_intent_rows(&self) -> Result, PersistenceError> { - let _query_timer = TursoQueryTimer::start("turso.load_unmet_intent_rows"); - let conn = self.configured_connection().await?; - let mut rows = conn - .query( - "SELECT entity_type, \ - MAX(action) AS action, \ - error, \ - COUNT(*) AS cnt, \ - MIN(created_at) AS first_seen, \ - MAX(created_at) AS last_seen \ - FROM trajectories \ - WHERE success = 0 \ - AND (authz_denied IS NULL OR authz_denied = 0) \ - GROUP BY entity_type, error \ - ORDER BY cnt DESC \ - LIMIT 100", - (), - ) - .await - .map_err(|e| { - let error = storage_error(e); - tracing::warn!(error = %error, "turso.load_unmet_intent_rows"); - error - })?; + Duration::from_millis(configured.max(MIN_TRAJECTORY_ATTEMPT_TIMEOUT_MS)) +} - let mut out = Vec::new(); - while let Some(row) = rows.next().await.map_err(storage_error)? { - out.push(UnmetIntentAggRow { - entity_type: row.get::(0).map_err(storage_error)?, - action: row.get::(1).map_err(storage_error)?, - error: row.get::>(2).map_err(storage_error)?, - count: row.get::(3).map_err(storage_error)? as u64, - first_seen: row.get::(4).map_err(storage_error)?, - last_seen: row.get::(5).map_err(storage_error)?, - }); - } - tracing::Span::current().record("row_count", out.len()); - tracing::debug!(count = out.len(), "turso.load_unmet_intent_rows"); - Ok(out) - } +fn trajectory_max_attempts() -> usize { + const DEFAULT_TRAJECTORY_MAX_ATTEMPTS: usize = 1; - /// Load the latest SubmitSpec success timestamp per entity_type. - /// - /// Used alongside [`load_unmet_intent_rows`] to determine which unmet-intent - /// groups have been resolved by a subsequent spec submission. - #[instrument(skip_all, fields(otel.name = "turso.load_submit_spec_timestamps"))] - pub async fn load_submit_spec_timestamps( - &self, - ) -> Result, PersistenceError> { - let _query_timer = TursoQueryTimer::start("turso.load_submit_spec_timestamps"); - let conn = self.configured_connection().await?; - let mut rows = conn - .query( - "SELECT entity_type, MAX(created_at) AS latest_at \ - FROM trajectories \ - WHERE success = 1 AND action = 'SubmitSpec' \ - GROUP BY entity_type", - (), - ) - .await - .map_err(storage_error)?; + std::env::var("TEMPER_TURSO_TRAJECTORY_MAX_ATTEMPTS") + .ok() + .and_then(|value| value.parse::().ok()) + .unwrap_or(DEFAULT_TRAJECTORY_MAX_ATTEMPTS) + .max(1) +} - let mut out = std::collections::BTreeMap::new(); - while let Some(row) = rows.next().await.map_err(storage_error)? { - let entity_type = row.get::(0).map_err(storage_error)?; - let latest_at = row.get::(1).map_err(storage_error)?; - out.insert(entity_type, latest_at); - } - Ok(out) - } +#[cfg(test)] +mod tests { + use super::*; + use crate::TursoTrajectoryInsert; - /// Count trajectory rows per tenant (single aggregate query). - /// - /// Replaces the previous pattern of loading 100,000 raw rows just to - /// produce per-tenant counts. - #[instrument(skip_all, fields(otel.name = "turso.count_trajectories_by_tenant"))] - pub async fn count_trajectories_by_tenant( - &self, - ) -> Result, PersistenceError> { - let _query_timer = TursoQueryTimer::start("turso.count_trajectories_by_tenant"); - let conn = self.configured_connection().await?; - let mut rows = conn - .query( - "SELECT tenant, COUNT(*) AS cnt FROM trajectories GROUP BY tenant", - (), - ) + async fn test_store() -> (TursoEventStore, tempfile::TempDir) { + let dir = tempfile::tempdir().expect("create temp dir"); + let db_path = dir.path().join("trajectory-session.db"); + let db_url = format!("file:{}", db_path.display()); + let store = TursoEventStore::new(&db_url, None) .await - .map_err(storage_error)?; - - let mut out = std::collections::BTreeMap::new(); - while let Some(row) = rows.next().await.map_err(storage_error)? { - let tenant = row.get::(0).map_err(storage_error)?; - let count = row.get::(1).map_err(storage_error)? as u64; - out.insert(tenant, count); - } - Ok(out) + .expect("create local turso store"); + (store, dir) } - /// Parse a trajectory row from a libsql Row (16 columns). - fn row_to_trajectory(row: &libsql::Row) -> Result { - Ok(TursoTrajectoryRow { - tenant: row.get::(0).map_err(storage_error)?, - entity_type: row.get::(1).map_err(storage_error)?, - entity_id: row.get::(2).map_err(storage_error)?, - action: row.get::(3).map_err(storage_error)?, - success: row.get::(4).map_err(storage_error)? != 0, - from_status: row.get::>(5).map_err(storage_error)?, - to_status: row.get::>(6).map_err(storage_error)?, - error: row.get::>(7).map_err(storage_error)?, - agent_id: row.get::>(8).map_err(storage_error)?, - session_id: row.get::>(9).map_err(storage_error)?, - authz_denied: row - .get::>(10) - .map_err(storage_error)? - .map(|v| v != 0), - denied_resource: row.get::>(11).map_err(storage_error)?, - denied_module: row.get::>(12).map_err(storage_error)?, - source: row.get::>(13).map_err(storage_error)?, - spec_governed: row - .get::>(14) - .map_err(storage_error)? - .map(|v| v != 0), - created_at: row.get::(15).map_err(storage_error)?, - request_body: row.get::>(16).map_err(storage_error)?, - intent: row.get::>(17).map_err(storage_error)?, - matched_policy_ids: row - .get::>(18) - .map_err(storage_error)? - .and_then(|s| serde_json::from_str(&s).ok()), - }) + fn insert<'a>( + action: &'a str, + session_id: &'a str, + created_at: &'a str, + ) -> TursoTrajectoryInsert<'a> { + TursoTrajectoryInsert { + tenant: "tenant", + entity_type: "Order", + entity_id: "order-1", + action, + success: true, + from_status: None, + to_status: None, + error: None, + agent_id: Some("agent-1"), + session_id: Some(session_id), + authz_denied: None, + denied_resource: None, + denied_module: None, + source: Some("Entity"), + spec_governed: Some(true), + created_at, + request_body: None, + intent: None, + matched_policy_ids: None, + capture_seq: None, + } } - /// Query trajectory statistics with optional filters. - #[instrument(skip_all, fields(otel.name = "turso.query_trajectory_stats"))] - pub async fn query_trajectory_stats( - &self, - entity_type: Option<&str>, - action: Option<&str>, - success_filter: Option, - failed_limit: i64, - ) -> Result { - let _query_timer = TursoQueryTimer::start("turso.query_trajectory_stats"); - let conn = self.configured_connection().await?; + #[tokio::test] + async fn session_query_returns_write_order_and_only_that_session() { + let (store, _dir) = test_store().await; + // Same `created_at` on the first two rows: the tiebreaker, not the + // timestamp, has to keep them in write order. + for entry in [ + insert("AddItem", "session-a", "2026-01-01T00:00:00Z"), + insert("SubmitOrder", "session-a", "2026-01-01T00:00:00Z"), + insert("ConfirmOrder", "session-a", "2026-01-01T00:00:01Z"), + insert("CancelOrder", "session-b", "2026-01-01T00:00:02Z"), + ] { + store + .persist_trajectory(entry) + .await + .expect("persist trajectory"); + } - // Total + success count. - let mut rows = conn - .query( - "SELECT COUNT(*) AS total, \ - COALESCE(SUM(CASE WHEN success = 1 THEN 1 ELSE 0 END), 0) AS success_count \ - FROM trajectories \ - WHERE (?1 IS NULL OR entity_type = ?1) \ - AND (?2 IS NULL OR action = ?2) \ - AND (?3 IS NULL OR success = ?3)", - params![entity_type, action, success_filter.map(|b| b as i64)], - ) + let rows = store + .query_trajectories_by_session("session-a", Some("tenant"), None, 100) .await - .map_err(storage_error)?; - - let (total, success_count) = match rows.next().await.map_err(storage_error)? { - Some(row) => ( - row.get::(0).map_err(storage_error)? as u64, - row.get::(1).map_err(storage_error)? as u64, - ), - None => (0, 0), - }; - drop(rows); + .expect("query session"); + let actions: Vec<&str> = rows.iter().map(|r| r.action.as_str()).collect(); + assert_eq!(actions, vec!["AddItem", "SubmitOrder", "ConfirmOrder"]); - // Per-action breakdown. - let mut rows = conn - .query( - "SELECT action, COUNT(*) AS total, \ - COALESCE(SUM(CASE WHEN success = 1 THEN 1 ELSE 0 END), 0) AS success, \ - COALESCE(SUM(CASE WHEN success = 0 THEN 1 ELSE 0 END), 0) AS error \ - FROM trajectories \ - GROUP BY action", - (), - ) + let other_tenant = store + .query_trajectories_by_session("session-a", Some("elsewhere"), None, 100) .await - .map_err(storage_error)?; - - let mut by_action = std::collections::BTreeMap::new(); - while let Some(row) = rows.next().await.map_err(storage_error)? { - let name = row.get::(0).map_err(storage_error)?; - by_action.insert( - name, - ActionStats { - total: row.get::(1).map_err(storage_error)? as u64, - success: row.get::(2).map_err(storage_error)? as u64, - error: row.get::(3).map_err(storage_error)? as u64, - }, - ); - } - drop(rows); + .expect("query session for a foreign tenant"); + assert!(other_tenant.is_empty()); - // Failed intents (newest first). - let mut rows = conn - .query( - "SELECT tenant, entity_type, entity_id, action, success, from_status, to_status, error, \ - agent_id, session_id, authz_denied, denied_resource, denied_module, source, spec_governed, created_at, request_body, intent, matched_policy_ids \ - FROM trajectories \ - WHERE success = 0 \ - ORDER BY created_at DESC \ - LIMIT ?1", - params![failed_limit], - ) + let filtered = store + .query_trajectories_by_session("session-a", Some("tenant"), Some("Invoice"), 100) .await - .map_err(storage_error)?; + .expect("query session with entity filter"); + assert!(filtered.is_empty()); + } - let mut failed_intents = Vec::new(); - while let Some(row) = rows.next().await.map_err(storage_error)? { - failed_intents.push(Self::row_to_trajectory(&row)?); + #[tokio::test] + async fn capture_order_outranks_insert_order_inside_one_timestamp() { + // Rows are written by independently spawned tasks, so the row that was + // captured first can be inserted second. Inside one `created_at` tick + // the autoincrement id would then replay the run backwards; the + // capture sequence is what puts it right. + let (store, _dir) = test_store().await; + for (action, capture_seq) in [("SubmitOrder", 2i64), ("AddItem", 1i64)] { + store + .persist_trajectory(TursoTrajectoryInsert { + capture_seq: Some(capture_seq), + ..insert(action, "session-race", "2026-01-01T00:00:00Z") + }) + .await + .expect("persist trajectory"); } - let error_count = total.saturating_sub(success_count); - tracing::info!( - entity_type, - action, - success_filter, - total, - success_count, - error_count, - failed_limit, - "trajectory.store.read" + let rows = store + .query_trajectories_by_session("session-race", Some("tenant"), None, 100) + .await + .expect("query session"); + let actions: Vec<&str> = rows.iter().map(|r| r.action.as_str()).collect(); + assert_eq!( + actions, + vec!["AddItem", "SubmitOrder"], + "the read must follow capture order, not the order the writes landed" + ); + assert_eq!( + rows.iter() + .filter_map(|r| r.capture_seq) + .collect::>(), + vec![1, 2] ); - Ok(TrajectoryStats { - total, - success_count, - error_count, - success_rate: if total > 0 { - success_count as f64 / total as f64 - } else { - 0.0 - }, - by_action, - failed_intents, - }) } - /// Query trajectories for a specific agent. - #[instrument(skip_all, fields(agent_id, otel.name = "turso.query_trajectories_by_agent"))] - pub async fn query_trajectories_by_agent( - &self, - agent_id: &str, - tenant: Option<&str>, - entity_type: Option<&str>, - limit: i64, - ) -> Result, PersistenceError> { - let _query_timer = TursoQueryTimer::start("turso.query_trajectories_by_agent"); - let conn = self.configured_connection().await?; - let mut rows = conn - .query( - "SELECT tenant, entity_type, entity_id, action, success, from_status, to_status, error, \ - agent_id, session_id, authz_denied, denied_resource, denied_module, source, spec_governed, created_at, request_body, intent, matched_policy_ids \ - FROM trajectories \ - WHERE agent_id = ?1 \ - AND (?2 IS NULL OR tenant = ?2) \ - AND (?3 IS NULL OR entity_type = ?3) \ - ORDER BY created_at DESC \ - LIMIT ?4", - params![agent_id, tenant, entity_type, limit], - ) - .await - .map_err(storage_error)?; - - let mut out = Vec::new(); - while let Some(row) = rows.next().await.map_err(storage_error)? { - out.push(Self::row_to_trajectory(&row)?); + #[tokio::test] + async fn rows_without_a_capture_sequence_still_order_by_write_order() { + // Rows written before the column existed carry no sequence; they must + // still read back deterministically rather than in engine-defined + // NULL order. + let (store, _dir) = test_store().await; + for action in ["AddItem", "SubmitOrder", "ConfirmOrder"] { + store + .persist_trajectory(insert(action, "session-legacy", "2026-01-01T00:00:00Z")) + .await + .expect("persist trajectory"); } - tracing::info!( - agent_id, - tenant, - entity_type, - limit, - count = out.len(), - "trajectory.store.read" - ); - Ok(out) - } - /// Query agent summaries (grouped by agent_id). - #[instrument(skip_all, fields(otel.name = "turso.query_agent_summaries"))] - pub async fn query_agent_summaries( - &self, - tenant: Option<&str>, - ) -> Result, PersistenceError> { - let _query_timer = TursoQueryTimer::start("turso.query_agent_summaries"); - let conn = self.configured_connection().await?; - let mut rows = conn - .query( - "SELECT agent_id, \ - COUNT(*) AS total_actions, \ - COALESCE(SUM(CASE WHEN success = 1 THEN 1 ELSE 0 END), 0) AS success_count, \ - COALESCE(SUM(CASE WHEN success = 0 THEN 1 ELSE 0 END), 0) AS error_count, \ - COALESCE(SUM(CASE WHEN authz_denied = 1 THEN 1 ELSE 0 END), 0) AS denial_count, \ - MAX(created_at) AS last_active_at \ - FROM trajectories \ - WHERE agent_id IS NOT NULL \ - AND (?1 IS NULL OR tenant = ?1) \ - GROUP BY agent_id \ - ORDER BY last_active_at DESC", - params![tenant], - ) + let rows = store + .query_trajectories_by_session("session-legacy", Some("tenant"), None, 100) .await - .map_err(storage_error)?; - - let mut out = Vec::new(); - while let Some(row) = rows.next().await.map_err(storage_error)? { - let total = row.get::(1).map_err(storage_error)? as u64; - let success = row.get::(2).map_err(storage_error)? as u64; - out.push(AgentSummary { - agent_id: row.get::(0).map_err(storage_error)?, - total_actions: total, - success_count: success, - error_count: row.get::(3).map_err(storage_error)? as u64, - denial_count: row.get::(4).map_err(storage_error)? as u64, - success_rate: if total > 0 { - success as f64 / total as f64 - } else { - 0.0 - }, - last_active_at: row.get::(5).map_err(storage_error)?, - }); - } - tracing::info!(tenant, count = out.len(), "trajectory.store.read"); - Ok(out) + .expect("query session"); + let actions: Vec<&str> = rows.iter().map(|r| r.action.as_str()).collect(); + assert_eq!(actions, vec!["AddItem", "SubmitOrder", "ConfirmOrder"]); + assert!(rows.iter().all(|r| r.capture_seq.is_none())); } } - -fn trajectory_attempt_timeout() -> Duration { - const DEFAULT_TRAJECTORY_ATTEMPT_TIMEOUT_MS: u64 = 1_000; - const MIN_TRAJECTORY_ATTEMPT_TIMEOUT_MS: u64 = 100; - - let configured = std::env::var("TEMPER_TURSO_TRAJECTORY_TIMEOUT_MS") - .ok() - .and_then(|value| value.parse::().ok()) - .unwrap_or(DEFAULT_TRAJECTORY_ATTEMPT_TIMEOUT_MS); - - Duration::from_millis(configured.max(MIN_TRAJECTORY_ATTEMPT_TIMEOUT_MS)) -} - -fn trajectory_max_attempts() -> usize { - const DEFAULT_TRAJECTORY_MAX_ATTEMPTS: usize = 1; - - std::env::var("TEMPER_TURSO_TRAJECTORY_MAX_ATTEMPTS") - .ok() - .and_then(|value| value.parse::().ok()) - .unwrap_or(DEFAULT_TRAJECTORY_MAX_ATTEMPTS) - .max(1) -} diff --git a/crates/temper-store-turso/src/store/trajectory_queries.rs b/crates/temper-store-turso/src/store/trajectory_queries.rs new file mode 100644 index 000000000..35ecf6193 --- /dev/null +++ b/crates/temper-store-turso/src/store/trajectory_queries.rs @@ -0,0 +1,465 @@ +//! Trajectory read queries. +//! +//! The write path (`persist_trajectory`) lives in the sibling `trajectory` +//! module; everything that reads trajectories back — recent rows, unmet +//! intents, aggregate stats, and the agent- and session-scoped replays — +//! lives here. + +use libsql::params; +use temper_runtime::persistence::{PersistenceError, storage_error}; +use tracing::instrument; + +use super::{ + ActionStats, AgentSummary, TrajectoryStats, TursoEventStore, TursoTrajectoryRow, + UnmetIntentAggRow, +}; +use crate::metrics::TursoQueryTimer; + +impl TursoEventStore { + /// Load recent trajectory entries for one tenant (newest first, up to `limit`). + /// + /// Scoped to `tenant` in SQL: the caller resolves the tenant from the + /// authenticated credential, so an observe read can only return rows that + /// credential owns (ADR-0157). + #[instrument(skip_all, fields(otel.name = "turso.load_recent_trajectories", row_count = tracing::field::Empty))] + pub async fn load_recent_trajectories( + &self, + tenant: &str, + limit: i64, + ) -> Result, PersistenceError> { + let _query_timer = TursoQueryTimer::start("turso.load_recent_trajectories"); + let conn = self.configured_connection().await?; + let mut rows = conn + .query( + "SELECT tenant, entity_type, entity_id, action, success, from_status, to_status, error, \ + agent_id, session_id, authz_denied, denied_resource, denied_module, source, spec_governed, created_at, request_body, intent, matched_policy_ids, capture_seq \ + FROM trajectories \ + WHERE tenant = ?1 \ + ORDER BY created_at DESC \ + LIMIT ?2", + params![tenant, limit], + ) + .await + .map_err(|e| { + let error = storage_error(e); + tracing::warn!(limit, error = %error, "trajectory.store.read"); + error + })?; + + let mut out = Vec::new(); + while let Some(row) = rows.next().await.map_err(storage_error)? { + out.push(Self::row_to_trajectory(&row)?); + } + tracing::Span::current().record("row_count", out.len()); + tracing::debug!(limit, count = out.len(), "trajectory.store.read"); + Ok(out) + } + + /// Load aggregated unmet-intent failure groups (SQL GROUP BY, ≤100 rows). + /// + /// Returns one row per (entity_type, error) group with counts and + /// timestamps. This replaces the previous pattern of loading up to 10,000 + /// raw trajectory rows and grouping them in Rust. + #[instrument(skip_all, fields(otel.name = "turso.load_unmet_intent_rows", row_count = tracing::field::Empty))] + pub async fn load_unmet_intent_rows( + &self, + tenant: &str, + ) -> Result, PersistenceError> { + let _query_timer = TursoQueryTimer::start("turso.load_unmet_intent_rows"); + let conn = self.configured_connection().await?; + let mut rows = conn + .query( + "SELECT entity_type, \ + MAX(action) AS action, \ + error, \ + COUNT(*) AS cnt, \ + MIN(created_at) AS first_seen, \ + MAX(created_at) AS last_seen \ + FROM trajectories \ + WHERE tenant = ?1 \ + AND success = 0 \ + AND (authz_denied IS NULL OR authz_denied = 0) \ + GROUP BY entity_type, error \ + ORDER BY cnt DESC \ + LIMIT 100", + params![tenant], + ) + .await + .map_err(|e| { + let error = storage_error(e); + tracing::warn!(error = %error, "turso.load_unmet_intent_rows"); + error + })?; + + let mut out = Vec::new(); + while let Some(row) = rows.next().await.map_err(storage_error)? { + out.push(UnmetIntentAggRow { + entity_type: row.get::(0).map_err(storage_error)?, + action: row.get::(1).map_err(storage_error)?, + error: row.get::>(2).map_err(storage_error)?, + count: row.get::(3).map_err(storage_error)? as u64, + first_seen: row.get::(4).map_err(storage_error)?, + last_seen: row.get::(5).map_err(storage_error)?, + }); + } + tracing::Span::current().record("row_count", out.len()); + tracing::debug!(count = out.len(), "turso.load_unmet_intent_rows"); + Ok(out) + } + + /// Load the latest SubmitSpec success timestamp per entity_type. + /// + /// Used alongside [`load_unmet_intent_rows`] to determine which unmet-intent + /// groups have been resolved by a subsequent spec submission. + #[instrument(skip_all, fields(otel.name = "turso.load_submit_spec_timestamps"))] + pub async fn load_submit_spec_timestamps( + &self, + tenant: &str, + ) -> Result, PersistenceError> { + let _query_timer = TursoQueryTimer::start("turso.load_submit_spec_timestamps"); + let conn = self.configured_connection().await?; + let mut rows = conn + .query( + "SELECT entity_type, MAX(created_at) AS latest_at \ + FROM trajectories \ + WHERE tenant = ?1 AND success = 1 AND action = 'SubmitSpec' \ + GROUP BY entity_type", + params![tenant], + ) + .await + .map_err(storage_error)?; + + let mut out = std::collections::BTreeMap::new(); + while let Some(row) = rows.next().await.map_err(storage_error)? { + let entity_type = row.get::(0).map_err(storage_error)?; + let latest_at = row.get::(1).map_err(storage_error)?; + out.insert(entity_type, latest_at); + } + Ok(out) + } + + /// Count trajectory rows per tenant (single aggregate query). + /// + /// Replaces the previous pattern of loading 100,000 raw rows just to + /// produce per-tenant counts. + #[instrument(skip_all, fields(otel.name = "turso.count_trajectories_by_tenant"))] + pub async fn count_trajectories_by_tenant( + &self, + ) -> Result, PersistenceError> { + let _query_timer = TursoQueryTimer::start("turso.count_trajectories_by_tenant"); + let conn = self.configured_connection().await?; + let mut rows = conn + .query( + "SELECT tenant, COUNT(*) AS cnt FROM trajectories GROUP BY tenant", + (), + ) + .await + .map_err(storage_error)?; + + let mut out = std::collections::BTreeMap::new(); + while let Some(row) = rows.next().await.map_err(storage_error)? { + let tenant = row.get::(0).map_err(storage_error)?; + let count = row.get::(1).map_err(storage_error)? as u64; + out.insert(tenant, count); + } + Ok(out) + } + + /// Parse a trajectory row from a libsql Row (16 columns). + fn row_to_trajectory(row: &libsql::Row) -> Result { + Ok(TursoTrajectoryRow { + tenant: row.get::(0).map_err(storage_error)?, + entity_type: row.get::(1).map_err(storage_error)?, + entity_id: row.get::(2).map_err(storage_error)?, + action: row.get::(3).map_err(storage_error)?, + success: row.get::(4).map_err(storage_error)? != 0, + from_status: row.get::>(5).map_err(storage_error)?, + to_status: row.get::>(6).map_err(storage_error)?, + error: row.get::>(7).map_err(storage_error)?, + agent_id: row.get::>(8).map_err(storage_error)?, + session_id: row.get::>(9).map_err(storage_error)?, + authz_denied: row + .get::>(10) + .map_err(storage_error)? + .map(|v| v != 0), + denied_resource: row.get::>(11).map_err(storage_error)?, + denied_module: row.get::>(12).map_err(storage_error)?, + source: row.get::>(13).map_err(storage_error)?, + spec_governed: row + .get::>(14) + .map_err(storage_error)? + .map(|v| v != 0), + created_at: row.get::(15).map_err(storage_error)?, + request_body: row.get::>(16).map_err(storage_error)?, + intent: row.get::>(17).map_err(storage_error)?, + matched_policy_ids: row + .get::>(18) + .map_err(storage_error)? + .and_then(|s| serde_json::from_str(&s).ok()), + capture_seq: row.get::>(19).map_err(storage_error)?, + }) + } + + /// Query trajectory statistics with optional filters. + #[instrument(skip_all, fields(otel.name = "turso.query_trajectory_stats"))] + /// Trajectory statistics for one tenant. + /// + /// Scoped to `tenant` in SQL: the failed-intent list returns whole rows — + /// error strings, entity ids — so an unscoped read would hand one tenant + /// another's operational detail (ADR-0157). + pub async fn query_trajectory_stats( + &self, + tenant: &str, + entity_type: Option<&str>, + action: Option<&str>, + success_filter: Option, + failed_limit: i64, + ) -> Result { + let _query_timer = TursoQueryTimer::start("turso.query_trajectory_stats"); + let conn = self.configured_connection().await?; + + // Total + success count. + let mut rows = conn + .query( + "SELECT COUNT(*) AS total, \ + COALESCE(SUM(CASE WHEN success = 1 THEN 1 ELSE 0 END), 0) AS success_count \ + FROM trajectories \ + WHERE tenant = ?1 \ + AND (?2 IS NULL OR entity_type = ?2) \ + AND (?3 IS NULL OR action = ?3) \ + AND (?4 IS NULL OR success = ?4)", + params![ + tenant, + entity_type, + action, + success_filter.map(|b| b as i64) + ], + ) + .await + .map_err(storage_error)?; + + let (total, success_count) = match rows.next().await.map_err(storage_error)? { + Some(row) => ( + row.get::(0).map_err(storage_error)? as u64, + row.get::(1).map_err(storage_error)? as u64, + ), + None => (0, 0), + }; + drop(rows); + + // Per-action breakdown. + let mut rows = conn + .query( + "SELECT action, COUNT(*) AS total, \ + COALESCE(SUM(CASE WHEN success = 1 THEN 1 ELSE 0 END), 0) AS success, \ + COALESCE(SUM(CASE WHEN success = 0 THEN 1 ELSE 0 END), 0) AS error \ + FROM trajectories \ + WHERE tenant = ?1 \ + GROUP BY action", + params![tenant], + ) + .await + .map_err(storage_error)?; + + let mut by_action = std::collections::BTreeMap::new(); + while let Some(row) = rows.next().await.map_err(storage_error)? { + let name = row.get::(0).map_err(storage_error)?; + by_action.insert( + name, + ActionStats { + total: row.get::(1).map_err(storage_error)? as u64, + success: row.get::(2).map_err(storage_error)? as u64, + error: row.get::(3).map_err(storage_error)? as u64, + }, + ); + } + drop(rows); + + // Failed intents (newest first). + let mut rows = conn + .query( + "SELECT tenant, entity_type, entity_id, action, success, from_status, to_status, error, \ + agent_id, session_id, authz_denied, denied_resource, denied_module, source, spec_governed, created_at, request_body, intent, matched_policy_ids, capture_seq \ + FROM trajectories \ + WHERE tenant = ?1 \ + AND success = 0 \ + ORDER BY created_at DESC \ + LIMIT ?2", + params![tenant, failed_limit], + ) + .await + .map_err(storage_error)?; + + let mut failed_intents = Vec::new(); + while let Some(row) = rows.next().await.map_err(storage_error)? { + failed_intents.push(Self::row_to_trajectory(&row)?); + } + + let error_count = total.saturating_sub(success_count); + tracing::info!( + entity_type, + action, + success_filter, + total, + success_count, + error_count, + failed_limit, + "trajectory.store.read" + ); + Ok(TrajectoryStats { + total, + success_count, + error_count, + success_rate: if total > 0 { + success_count as f64 / total as f64 + } else { + 0.0 + }, + by_action, + failed_intents, + }) + } + + /// Query trajectories for a specific agent. + #[instrument(skip_all, fields(agent_id, otel.name = "turso.query_trajectories_by_agent"))] + pub async fn query_trajectories_by_agent( + &self, + agent_id: &str, + tenant: Option<&str>, + entity_type: Option<&str>, + limit: i64, + ) -> Result, PersistenceError> { + let _query_timer = TursoQueryTimer::start("turso.query_trajectories_by_agent"); + let conn = self.configured_connection().await?; + let mut rows = conn + .query( + "SELECT tenant, entity_type, entity_id, action, success, from_status, to_status, error, \ + agent_id, session_id, authz_denied, denied_resource, denied_module, source, spec_governed, created_at, request_body, intent, matched_policy_ids, capture_seq \ + FROM trajectories \ + WHERE agent_id = ?1 \ + AND (?2 IS NULL OR tenant = ?2) \ + AND (?3 IS NULL OR entity_type = ?3) \ + ORDER BY created_at DESC \ + LIMIT ?4", + params![agent_id, tenant, entity_type, limit], + ) + .await + .map_err(storage_error)?; + + let mut out = Vec::new(); + while let Some(row) = rows.next().await.map_err(storage_error)? { + out.push(Self::row_to_trajectory(&row)?); + } + tracing::info!( + agent_id, + tenant, + entity_type, + limit, + count = out.len(), + "trajectory.store.read" + ); + Ok(out) + } + + /// Query one session's trajectory rows in the order the kernel wrote them. + /// + /// Ordered ascending — oldest first — because the conformance checker + /// replays a session as a state-machine run and a newest-first read would + /// hand it the run backwards. + /// + /// Ties inside one `created_at` tick are broken by `capture_seq`, the + /// order the capturing process stamped on the entry, and only then by + /// `id`. `id` alone is the order the writes landed: rows are persisted by + /// independently spawned tasks, so two entries captured in one logical + /// tick can be inserted in either order and a denial/retry pair would be + /// replayed backwards. `COALESCE` sorts rows written before the column + /// existed first and identically on both backends, rather than leaving it + /// to each engine's NULL ordering. + #[instrument(skip_all, fields(session_id, otel.name = "turso.query_trajectories_by_session"))] + pub async fn query_trajectories_by_session( + &self, + session_id: &str, + tenant: Option<&str>, + entity_type: Option<&str>, + limit: i64, + ) -> Result, PersistenceError> { + let _query_timer = TursoQueryTimer::start("turso.query_trajectories_by_session"); + let conn = self.configured_connection().await?; + let mut rows = conn + .query( + "SELECT tenant, entity_type, entity_id, action, success, from_status, to_status, error, \ + agent_id, session_id, authz_denied, denied_resource, denied_module, source, spec_governed, created_at, request_body, intent, matched_policy_ids, capture_seq \ + FROM trajectories \ + WHERE session_id = ?1 \ + AND (?2 IS NULL OR tenant = ?2) \ + AND (?3 IS NULL OR entity_type = ?3) \ + ORDER BY created_at ASC, COALESCE(capture_seq, 0) ASC, id ASC \ + LIMIT ?4", + params![session_id, tenant, entity_type, limit], + ) + .await + .map_err(storage_error)?; + + let mut out = Vec::new(); + while let Some(row) = rows.next().await.map_err(storage_error)? { + out.push(Self::row_to_trajectory(&row)?); + } + tracing::info!( + session_id, + tenant, + entity_type, + limit, + count = out.len(), + "trajectory.store.read" + ); + Ok(out) + } + + /// Query agent summaries (grouped by agent_id). + #[instrument(skip_all, fields(otel.name = "turso.query_agent_summaries"))] + pub async fn query_agent_summaries( + &self, + tenant: Option<&str>, + ) -> Result, PersistenceError> { + let _query_timer = TursoQueryTimer::start("turso.query_agent_summaries"); + let conn = self.configured_connection().await?; + let mut rows = conn + .query( + "SELECT agent_id, \ + COUNT(*) AS total_actions, \ + COALESCE(SUM(CASE WHEN success = 1 THEN 1 ELSE 0 END), 0) AS success_count, \ + COALESCE(SUM(CASE WHEN success = 0 THEN 1 ELSE 0 END), 0) AS error_count, \ + COALESCE(SUM(CASE WHEN authz_denied = 1 THEN 1 ELSE 0 END), 0) AS denial_count, \ + MAX(created_at) AS last_active_at \ + FROM trajectories \ + WHERE agent_id IS NOT NULL \ + AND (?1 IS NULL OR tenant = ?1) \ + GROUP BY agent_id \ + ORDER BY last_active_at DESC", + params![tenant], + ) + .await + .map_err(storage_error)?; + + let mut out = Vec::new(); + while let Some(row) = rows.next().await.map_err(storage_error)? { + let total = row.get::(1).map_err(storage_error)? as u64; + let success = row.get::(2).map_err(storage_error)? as u64; + out.push(AgentSummary { + agent_id: row.get::(0).map_err(storage_error)?, + total_actions: total, + success_count: success, + error_count: row.get::(3).map_err(storage_error)? as u64, + denial_count: row.get::(4).map_err(storage_error)? as u64, + success_rate: if total > 0 { + success as f64 / total as f64 + } else { + 0.0 + }, + last_active_at: row.get::(5).map_err(storage_error)?, + }); + } + tracing::info!(tenant, count = out.len(), "trajectory.store.read"); + Ok(out) + } +} diff --git a/crates/temper-wasm/Cargo.toml b/crates/temper-wasm/Cargo.toml index 641abf910..2c56d660a 100644 --- a/crates/temper-wasm/Cargo.toml +++ b/crates/temper-wasm/Cargo.toml @@ -33,6 +33,8 @@ test-helpers = [] [dev-dependencies] base64 = "0.22" tokio-test = { workspace = true } -opentelemetry_sdk = { workspace = true } +# `testing` gives the in-memory span exporter used by the ARN-243 behavioral +# redaction test. Dev-only: not built into any production artifact. +opentelemetry_sdk = { workspace = true, features = ["testing"] } tracing-subscriber = { workspace = true } axum = { workspace = true } diff --git a/crates/temper-wasm/src/authorized_host.rs b/crates/temper-wasm/src/authorized_host.rs index c6d8310c5..38c682be0 100644 --- a/crates/temper-wasm/src/authorized_host.rs +++ b/crates/temper-wasm/src/authorized_host.rs @@ -97,6 +97,14 @@ impl AuthorizedWasmHost { #[async_trait] impl WasmHost for AuthorizedWasmHost { + /// Forward the wrapped host's per-tenant content decision (ADR-0166). This + /// wrapper is what dispatch hands to the engine, so without forwarding the + /// engine would read the trait default and redact even for a tenant that + /// opted in — safe, but silently useless. + fn exports_llm_content(&self) -> bool { + self.inner.exports_llm_content() + } + async fn http_call( &self, method: &str, @@ -311,177 +319,5 @@ impl WasmHost for AuthorizedWasmHost { } #[cfg(test)] -mod tests { - use super::*; - use crate::host_trait::SimWasmHost; - - /// A gate that denies everything. - struct DenyAllGate; - impl WasmAuthzGate for DenyAllGate { - fn authorize_http_call( - &self, - _domain: &str, - _method: &str, - _url: &str, - _ctx: &WasmAuthzContext, - ) -> WasmAuthzDecision { - WasmAuthzDecision::Deny("denied by policy".into()) - } - fn authorize_secret_access( - &self, - _key: &str, - _ctx: &WasmAuthzContext, - ) -> WasmAuthzDecision { - WasmAuthzDecision::Deny("denied by policy".into()) - } - } - - /// A gate that allows everything. - struct AllowAllGate; - impl WasmAuthzGate for AllowAllGate { - fn authorize_http_call( - &self, - _domain: &str, - _method: &str, - _url: &str, - _ctx: &WasmAuthzContext, - ) -> WasmAuthzDecision { - WasmAuthzDecision::Allow - } - fn authorize_secret_access( - &self, - _key: &str, - _ctx: &WasmAuthzContext, - ) -> WasmAuthzDecision { - WasmAuthzDecision::Allow - } - } - - fn test_ctx() -> WasmAuthzContext { - WasmAuthzContext::test_fixture() - } - - #[tokio::test] - async fn deny_gate_blocks_http_call() { - let inner = Arc::new(SimWasmHost::new()); - let gate = Arc::new(DenyAllGate); - let host = AuthorizedWasmHost::new(inner, gate, test_ctx()); - - let result = host - .http_call("POST", "https://api.stripe.com/v1/charges", &[], "") - .await; - assert!(result.is_err()); - assert!(result.unwrap_err().contains("authorization denied")); - } - - #[tokio::test] - async fn deny_gate_blocks_secret_access() { - let inner = Arc::new(SimWasmHost::new().with_secret("STRIPE_API_KEY", "sk-test")); - let gate = Arc::new(DenyAllGate); - let host = AuthorizedWasmHost::new(inner, gate, test_ctx()); - - let result = host.get_secret("STRIPE_API_KEY"); - assert!(result.is_err()); - assert!(result.unwrap_err().contains("authorization denied")); - } - - #[tokio::test] - async fn allow_gate_delegates_http_call() { - let inner = Arc::new(SimWasmHost::new()); - let gate = Arc::new(AllowAllGate); - let host = AuthorizedWasmHost::new(inner, gate, test_ctx()); - - let result = host - .http_call("GET", "https://api.stripe.com/v1/charges", &[], "") - .await; - assert!(result.is_ok()); - let (status, _body) = result.unwrap(); - assert_eq!(status, 200); - } - - #[tokio::test] - async fn allow_gate_delegates_secret_access() { - let inner = Arc::new(SimWasmHost::new().with_secret("KEY", "val")); - let gate = Arc::new(AllowAllGate); - let host = AuthorizedWasmHost::new(inner, gate, test_ctx()); - - let result = host.get_secret("KEY"); - assert_eq!(result, Ok("val".into())); - } - - #[test] - fn allow_gate_delegates_evaluate_spec() { - let ioa_source = "[automaton]\nname = \"Issue\""; - let ioa_hash = format!("{:x}", ioa_source.len()); - let inner = Arc::new(SimWasmHost::new().with_spec_eval_response( - &ioa_hash, - "Reassign", - r#"{"success":true,"new_state":"InProgress"}"#, - )); - let gate = Arc::new(AllowAllGate); - let host = AuthorizedWasmHost::new(inner, gate, test_ctx()); - - let result = host.evaluate_spec(ioa_source, "Backlog", "Reassign", "{}"); - assert!( - result.is_ok(), - "evaluate_spec should delegate to inner host" - ); - assert!( - result.unwrap_or_default().contains(r#""success":true"#), - "expected canned evaluate_spec response from inner host" - ); - } - - #[test] - fn logging_always_allowed() { - let inner = Arc::new(SimWasmHost::new()); - let gate = Arc::new(DenyAllGate); - let host = AuthorizedWasmHost::new(inner, gate, test_ctx()); - // Should not panic - host.log("info", "test message"); - } - - #[test] - fn extract_domain_https() { - assert_eq!( - extract_domain("https://api.stripe.com/v1/charges"), - "api.stripe.com" - ); - } - - #[test] - fn extract_domain_http() { - assert_eq!(extract_domain("http://localhost:8080/api"), "localhost"); - } - - #[test] - fn extract_domain_with_port() { - assert_eq!( - extract_domain("https://example.com:443/path"), - "example.com" - ); - } - - #[test] - fn extract_domain_no_scheme() { - assert_eq!(extract_domain("api.stripe.com/path"), "api.stripe.com"); - } - - #[test] - fn extract_domain_bare() { - assert_eq!(extract_domain("https://example.com"), "example.com"); - } - - #[test] - fn extract_domain_ip() { - assert_eq!(extract_domain("http://127.0.0.1:3000/api"), "127.0.0.1"); - } - - #[test] - fn extract_domain_strips_userinfo() { - assert_eq!( - extract_domain("https://attacker:pass@localhost/exploit"), - "localhost" - ); - } -} +#[path = "authorized_host_test.rs"] +mod tests; diff --git a/crates/temper-wasm/src/authorized_host_test.rs b/crates/temper-wasm/src/authorized_host_test.rs new file mode 100644 index 000000000..3ca566edd --- /dev/null +++ b/crates/temper-wasm/src/authorized_host_test.rs @@ -0,0 +1,226 @@ +//! Authorization-wrapper tests, including ADR-0166 policy forwarding. +use super::*; +use crate::host_trait::SimWasmHost; + +/// A gate that denies everything. +struct DenyAllGate; +impl WasmAuthzGate for DenyAllGate { + fn authorize_http_call( + &self, + _domain: &str, + _method: &str, + _url: &str, + _ctx: &WasmAuthzContext, + ) -> WasmAuthzDecision { + WasmAuthzDecision::Deny("denied by policy".into()) + } + fn authorize_secret_access(&self, _key: &str, _ctx: &WasmAuthzContext) -> WasmAuthzDecision { + WasmAuthzDecision::Deny("denied by policy".into()) + } +} + +/// A gate that allows everything. +struct AllowAllGate; +impl WasmAuthzGate for AllowAllGate { + fn authorize_http_call( + &self, + _domain: &str, + _method: &str, + _url: &str, + _ctx: &WasmAuthzContext, + ) -> WasmAuthzDecision { + WasmAuthzDecision::Allow + } + fn authorize_secret_access(&self, _key: &str, _ctx: &WasmAuthzContext) -> WasmAuthzDecision { + WasmAuthzDecision::Allow + } +} + +fn test_ctx() -> WasmAuthzContext { + WasmAuthzContext::test_fixture() +} + +/// ADR-0166. This wrapper is what dispatch hands to the engine, so the +/// engine's view of the tenant's content decision is whatever this forwards. +/// Both directions matter: not forwarding `true` silently disables the opt-in +/// for every tenant, and not forwarding `false` would leak. +#[test] +fn authorized_host_forwards_the_llm_content_export_decision() { + use crate::host_trait::ProductionWasmHost; + use std::collections::BTreeMap; + + let opted_in: Arc = + Arc::new(ProductionWasmHost::new(BTreeMap::new()).with_llm_content_export(true)); + let wrapped = AuthorizedWasmHost::new(opted_in, Arc::new(AllowAllGate), test_ctx()); + assert!( + wrapped.exports_llm_content(), + "an opted-in tenant's decision must survive the authz wrapper" + ); + + let redacted: Arc = + Arc::new(ProductionWasmHost::new(BTreeMap::new()).with_llm_content_export(false)); + let wrapped = AuthorizedWasmHost::new(redacted, Arc::new(AllowAllGate), test_ctx()); + assert!(!wrapped.exports_llm_content()); +} + +/// A host that does not answer must redact. Any future `WasmHost` that forgets +/// to implement the method inherits this, so the default is the whole +/// protection for that host. +#[test] +fn wasm_host_defaults_to_redacting_llm_content() { + struct MinimalHost; + #[async_trait] + impl WasmHost for MinimalHost { + async fn http_call( + &self, + _method: &str, + _url: &str, + _headers: &[(String, String)], + _body: &str, + ) -> Result<(u16, String), String> { + Ok((200, String::new())) + } + async fn http_call_binary( + &self, + _method: &str, + _url: &str, + _headers: &[(String, String)], + _body: &[u8], + ) -> Result<(u16, Vec), String> { + Ok((200, Vec::new())) + } + fn get_secret(&self, _key: &str) -> Result { + Err("no secrets".to_string()) + } + fn log(&self, _level: &str, _message: &str) {} + } + assert!( + !MinimalHost.exports_llm_content(), + "the WasmHost default must be redact; a host that does not opt in must \ + never export a tenant's LLM content" + ); +} + +#[tokio::test] +async fn deny_gate_blocks_http_call() { + let inner = Arc::new(SimWasmHost::new()); + let gate = Arc::new(DenyAllGate); + let host = AuthorizedWasmHost::new(inner, gate, test_ctx()); + + let result = host + .http_call("POST", "https://api.stripe.com/v1/charges", &[], "") + .await; + assert!(result.is_err()); + assert!(result.unwrap_err().contains("authorization denied")); +} + +#[tokio::test] +async fn deny_gate_blocks_secret_access() { + let inner = Arc::new(SimWasmHost::new().with_secret("STRIPE_API_KEY", "sk-test")); + let gate = Arc::new(DenyAllGate); + let host = AuthorizedWasmHost::new(inner, gate, test_ctx()); + + let result = host.get_secret("STRIPE_API_KEY"); + assert!(result.is_err()); + assert!(result.unwrap_err().contains("authorization denied")); +} + +#[tokio::test] +async fn allow_gate_delegates_http_call() { + let inner = Arc::new(SimWasmHost::new()); + let gate = Arc::new(AllowAllGate); + let host = AuthorizedWasmHost::new(inner, gate, test_ctx()); + + let result = host + .http_call("GET", "https://api.stripe.com/v1/charges", &[], "") + .await; + assert!(result.is_ok()); + let (status, _body) = result.unwrap(); + assert_eq!(status, 200); +} + +#[tokio::test] +async fn allow_gate_delegates_secret_access() { + let inner = Arc::new(SimWasmHost::new().with_secret("KEY", "val")); + let gate = Arc::new(AllowAllGate); + let host = AuthorizedWasmHost::new(inner, gate, test_ctx()); + + let result = host.get_secret("KEY"); + assert_eq!(result, Ok("val".into())); +} + +#[test] +fn allow_gate_delegates_evaluate_spec() { + let ioa_source = "[automaton]\nname = \"Issue\""; + let ioa_hash = format!("{:x}", ioa_source.len()); + let inner = Arc::new(SimWasmHost::new().with_spec_eval_response( + &ioa_hash, + "Reassign", + r#"{"success":true,"new_state":"InProgress"}"#, + )); + let gate = Arc::new(AllowAllGate); + let host = AuthorizedWasmHost::new(inner, gate, test_ctx()); + + let result = host.evaluate_spec(ioa_source, "Backlog", "Reassign", "{}"); + assert!( + result.is_ok(), + "evaluate_spec should delegate to inner host" + ); + assert!( + result.unwrap_or_default().contains(r#""success":true"#), + "expected canned evaluate_spec response from inner host" + ); +} + +#[test] +fn logging_always_allowed() { + let inner = Arc::new(SimWasmHost::new()); + let gate = Arc::new(DenyAllGate); + let host = AuthorizedWasmHost::new(inner, gate, test_ctx()); + // Should not panic + host.log("info", "test message"); +} + +#[test] +fn extract_domain_https() { + assert_eq!( + extract_domain("https://api.stripe.com/v1/charges"), + "api.stripe.com" + ); +} + +#[test] +fn extract_domain_http() { + assert_eq!(extract_domain("http://localhost:8080/api"), "localhost"); +} + +#[test] +fn extract_domain_with_port() { + assert_eq!( + extract_domain("https://example.com:443/path"), + "example.com" + ); +} + +#[test] +fn extract_domain_no_scheme() { + assert_eq!(extract_domain("api.stripe.com/path"), "api.stripe.com"); +} + +#[test] +fn extract_domain_bare() { + assert_eq!(extract_domain("https://example.com"), "example.com"); +} + +#[test] +fn extract_domain_ip() { + assert_eq!(extract_domain("http://127.0.0.1:3000/api"), "127.0.0.1"); +} + +#[test] +fn extract_domain_strips_userinfo() { + assert_eq!( + extract_domain("https://attacker:pass@localhost/exploit"), + "localhost" + ); +} diff --git a/crates/temper-wasm/src/engine/guest_read_bounds_test.rs b/crates/temper-wasm/src/engine/guest_read_bounds_test.rs new file mode 100644 index 000000000..43a809154 --- /dev/null +++ b/crates/temper-wasm/src/engine/guest_read_bounds_test.rs @@ -0,0 +1,308 @@ +//! ARN-226: guest reads are bounds-checked before the host allocates. +//! +//! These live in their own file (rather than `engine/tests.rs`) so the readability +//! ratchet counts them as tests: its production-file exclusion matches +//! `*_test.rs`, which `tests.rs` does not. + +use super::tests::{make_context, make_host, make_streams}; +use super::*; + +// ARN-226 (wiring): a guest returns a result pointer whose 4-byte length prefix is +// forged far larger than its linear memory. The host must reject it on the bounds +// check BEFORE allocating a buffer of that size. Memory here is 1 page (64 KiB) and +// the forged length is 64 MiB — above LARGE_ALLOC_THRESHOLD, so the allocation +// counter observes the *ordering*, not just the rejection. +const WAT_FORGED_RESULT_LENGTH: &str = r#" + (module + (memory (export "memory") 1) + (func (export "run") (param i32 i32) (result i32) + ;; length prefix at address 100 = 64 MiB, far beyond the 64 KiB memory + i32.const 100 + i32.const 67108864 + i32.store + ;; return 104 so the host reads the prefix at 104-4 = 100 + i32.const 104 + ) + ) +"#; + +// A result whose length is individually *smaller* than the guest's memory but whose +// `ptr + len` runs past the end. This pins the predicate to the full range: a weaker +// check that only compared `len` against the memory size would wrongly accept it. +const WAT_RESULT_LENGTH_OVERRUNS_END: &str = r#" + (module + (memory (export "memory") 1) + (func (export "run") (param i32 i32) (result i32) + ;; length 10000 stored at 59996, so the host reads it for ptr = 60000. + ;; 10000 < 65536, but 60000 + 10000 = 70000 > 65536. + i32.const 59996 + i32.const 10000 + i32.store + i32.const 60000 + ) + ) +"#; + +#[tokio::test] +async fn forged_result_length_is_rejected_before_allocating() { + // This test locks the WIRING, not just the predicate: it asserts the specific + // error the bounds check produces. Deleting the `guest_read_bounds_ok` call on + // the result-read path makes the host allocate the forged size and then fail + // with "failed to read result" instead, which fails this assertion. + use std::sync::atomic::Ordering; + let _serialize = ALLOC_OBSERVER_LOCK.lock().await; + + let engine = WasmEngine::new().unwrap(); + let hash = engine + .compile_and_cache(WAT_FORGED_RESULT_LENGTH.as_bytes()) + .unwrap(); + + let before = LARGE_ALLOCS.load(Ordering::SeqCst); + let err = engine + .invoke( + &hash, + &make_context(), + make_host(), + &WasmResourceLimits::default(), + make_streams(), + ) + .await + .expect_err("a forged result length must be rejected"); + let large_allocs = LARGE_ALLOCS.load(Ordering::SeqCst) - before; + + let message = format!("{err:?}"); + assert!( + message.contains("result length exceeds guest linear memory"), + "expected the bounds check to reject before allocating, got: {message}" + ); + // Ordering, not just rejection: moving the allocation above the guard would + // still produce the error above, but would trip the counter. + assert_eq!( + large_allocs, 0, + "the forged length must be rejected BEFORE allocating; {large_allocs} \ + allocation(s) >= {LARGE_ALLOC_THRESHOLD} bytes were made" + ); +} + +#[tokio::test] +async fn result_length_overrunning_memory_end_is_rejected() { + // Pins `ptr + len`, not just `len`: the length here is well under the guest's + // memory size, so only a check on the whole range rejects it. + let engine = WasmEngine::new().unwrap(); + let hash = engine + .compile_and_cache(WAT_RESULT_LENGTH_OVERRUNS_END.as_bytes()) + .unwrap(); + + let err = engine + .invoke( + &hash, + &make_context(), + make_host(), + &WasmResourceLimits::default(), + make_streams(), + ) + .await + .expect_err("a result range running past the end of memory must be rejected"); + + let message = format!("{err:?}"); + assert!( + message.contains("result length exceeds guest linear memory"), + "expected the range check to reject ptr+len past the end, got: {message}" + ); +} + +// --------------------------------------------------------------------------- +// ARN-226 (wiring, helper path): an allocation-observing guard. +// +// A guest calling a helper-backed host function with a huge `len` gets the same +// `-1` back whether the bounds check runs before the allocation or after it, so +// the return value cannot distinguish the two. This counting allocator makes the +// ordering observable: it records allocations at or above a threshold no +// legitimate path in this test should reach. With the guard in place the count +// stays zero; delete the guard in `read_guest_string` / `read_guest_bytes` and +// the host allocates the guest-chosen size first, which the assertion catches. +// --------------------------------------------------------------------------- + +/// Allocation size at or above which we consider a host allocation "large". +/// The guest memory in these tests is one 64 KiB page, so nothing legitimate in +/// the guest-read path approaches this. It must stay **below** +/// `WasmResourceLimits::default().max_memory` (64 MiB) so an unguarded read of a +/// guest-chosen length is always counted — and above any legitimate allocation in +/// this test binary (the largest is `MAX_MODULE_SIZE + 1`, 10 MiB). A future test +/// that grows guest memory past this and reads it in-bounds would need a rethink. +const LARGE_ALLOC_THRESHOLD: usize = 32 * 1024 * 1024; + +static LARGE_ALLOCS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + +/// The counter is process-wide, so allocation-observing tests take this lock to +/// avoid attributing each other's allocations. Nothing else in this crate's suite +/// allocates near the threshold — the largest is `MAX_MODULE_SIZE + 1` (10 MiB) in +/// `module_too_large_rejected` — so serializing these is sufficient. +static ALLOC_OBSERVER_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + +struct LargeAllocCounter; + +unsafe impl std::alloc::GlobalAlloc for LargeAllocCounter { + unsafe fn alloc(&self, layout: std::alloc::Layout) -> *mut u8 { + if layout.size() >= LARGE_ALLOC_THRESHOLD { + LARGE_ALLOCS.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + } + unsafe { std::alloc::System.alloc(layout) } + } + unsafe fn alloc_zeroed(&self, layout: std::alloc::Layout) -> *mut u8 { + if layout.size() >= LARGE_ALLOC_THRESHOLD { + LARGE_ALLOCS.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + } + unsafe { std::alloc::System.alloc_zeroed(layout) } + } + unsafe fn dealloc(&self, ptr: *mut u8, layout: std::alloc::Layout) { + unsafe { std::alloc::System.dealloc(ptr, layout) } + } + unsafe fn realloc(&self, ptr: *mut u8, layout: std::alloc::Layout, new_size: usize) -> *mut u8 { + if new_size >= LARGE_ALLOC_THRESHOLD { + LARGE_ALLOCS.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + } + unsafe { std::alloc::System.realloc(ptr, layout, new_size) } + } +} + +#[global_allocator] +static ALLOC_COUNTER: LargeAllocCounter = LargeAllocCounter; + +// Calls host_emit_progress(0, 64 MiB) against a single 64 KiB page — an +// out-of-bounds read whose length is far beyond the guest's own memory. Traps if +// the host does NOT return the -1 error sentinel. +const WAT_OVERSIZED_HOST_READ: &str = r#" + (module + (import "env" "host_emit_progress" (func $emit (param i32 i32) (result i32))) + (memory (export "memory") 1) + (func (export "run") (param i32 i32) (result i32) + i32.const 0 + i32.const 67108864 + call $emit + i32.const -1 + i32.ne + if + unreachable + end + i32.const 0 + ) + ) +"#; + +// Same shape as WAT_OVERSIZED_HOST_READ but through `host_cache_contains`, which +// reads via `read_guest_lossy` -> `read_guest_bytes` — the other helper. Its ABI is +// boolean with no error sentinel (0 = not cached), so the guest cannot observe the +// refusal; only the allocation counter can. +const WAT_OVERSIZED_HOST_BYTES_READ: &str = r#" + (module + (import "env" "host_cache_contains" (func $contains (param i32 i32) (result i32))) + (memory (export "memory") 1) + (func (export "run") (param i32 i32) (result i32) + i32.const 0 + i32.const 67108864 + call $contains + drop + i32.const 0 + ) + ) +"#; + +#[tokio::test] +async fn oversized_host_bytes_read_is_rejected_before_allocating() { + use std::sync::atomic::Ordering; + let _serialize = ALLOC_OBSERVER_LOCK.lock().await; + + let engine = WasmEngine::new().unwrap(); + let hash = engine + .compile_and_cache(WAT_OVERSIZED_HOST_BYTES_READ.as_bytes()) + .unwrap(); + + let before = LARGE_ALLOCS.load(Ordering::SeqCst); + let result = engine + .invoke( + &hash, + &make_context(), + make_host(), + &WasmResourceLimits::default(), + make_streams(), + ) + .await; + let large_allocs = LARGE_ALLOCS.load(Ordering::SeqCst) - before; + + assert!( + result.is_ok(), + "the guest itself must run cleanly: {result:?}" + ); + assert_eq!( + large_allocs, 0, + "read_guest_bytes must refuse the 64 MiB length before allocating; \ + {large_allocs} allocation(s) >= {LARGE_ALLOC_THRESHOLD} bytes were made" + ); +} + +#[tokio::test] +async fn oversized_host_read_is_rejected_before_allocating() { + use std::sync::atomic::Ordering; + let _serialize = ALLOC_OBSERVER_LOCK.lock().await; + + let engine = WasmEngine::new().unwrap(); + let hash = engine + .compile_and_cache(WAT_OVERSIZED_HOST_READ.as_bytes()) + .unwrap(); + + let before = LARGE_ALLOCS.load(Ordering::SeqCst); + let result = engine + .invoke( + &hash, + &make_context(), + make_host(), + &WasmResourceLimits::default(), + make_streams(), + ) + .await; + let large_allocs = LARGE_ALLOCS.load(Ordering::SeqCst) - before; + + // The guest asserts it got -1 back, so a successful invocation proves the + // read was refused rather than served. + assert!( + result.is_ok(), + "host must return the error sentinel for an out-of-bounds read: {result:?}" + ); + // And it was refused *before* allocating the guest-chosen 64 MiB. + assert_eq!( + large_allocs, 0, + "a guest-supplied length must not drive a large host allocation; \ + {large_allocs} allocation(s) >= {LARGE_ALLOC_THRESHOLD} bytes were made" + ); +} + +#[test] +fn guest_memory_reads_stay_inside_the_bounds_checked_helpers() { + // ARN-226 (class guard). The individual guards are locked by the mutation + // tests above, but nothing stopped a future edit from reintroducing a raw + // `memory.read` + `vec![0u8; len]` at a new call site — which is exactly how + // the original vulnerability was spread across ten places. This asserts the + // structural invariant instead: in this file, guest memory is only ever read + // inside `read_guest_string` / `read_guest_bytes`, which bounds-check first. + // + // If you are adding a host function, call one of those helpers rather than + // relaxing this test. + const ALLOWED_READ_SITES: usize = 2; + let source = include_str!("host_functions.rs"); + let reads = source + .lines() + .filter(|line| { + let trimmed = line.trim_start(); + // A guest-memory read passes arguments (store/caller, offset, buffer). + // `RwLock::read()` takes none, so the empty-paren form is excluded. + !trimmed.starts_with("//") && trimmed.contains(".read(") && !trimmed.contains(".read()") + }) + .count(); + assert_eq!( + reads, ALLOWED_READ_SITES, + "guest memory must only be read inside the bounds-checked helpers; \ + found {reads} `.read(` call sites in host_functions.rs (expected \ + {ALLOWED_READ_SITES}: one in read_guest_string, one in read_guest_bytes)" + ); +} diff --git a/crates/temper-wasm/src/engine/guest_spans.rs b/crates/temper-wasm/src/engine/guest_spans.rs index 1fcc5b678..207475043 100644 --- a/crates/temper-wasm/src/engine/guest_spans.rs +++ b/crates/temper-wasm/src/engine/guest_spans.rs @@ -8,14 +8,18 @@ use serde::Deserialize; use serde_json::Value; use tracing_opentelemetry::OpenTelemetrySpanExt as _; +use crate::host_trait::span_hints::clamp_redacted_metadata_value; use crate::host_trait::{datadog_visible_span_hint_field, truncate_for_span_attr}; use crate::types::WasmInvocationContext; mod export; +mod redaction; #[cfg(test)] #[path = "guest_spans_test.rs"] mod tests; +use redaction::{allowed_attributes, guest_span_attribute_allowed}; + use export::{ export_manual_span, manual_parent_context, manual_span_attributes, merge_end_status_attributes, status_from_end_payload, tracing_span_ids, @@ -86,6 +90,10 @@ struct GuestSpanManualEvent { pub(crate) struct GuestSpanRegistry { context: WasmInvocationContext, manual_export: bool, + /// Whether this tenant opted into exporting raw LLM content (ADR-0166). + /// `false` means guest-supplied `gen_ai.*` attributes are restricted to + /// recognised, length-bounded metadata before they reach any span. + export_llm_content: bool, next_id: i64, total_started: usize, max_spans: usize, @@ -96,18 +104,34 @@ pub(crate) struct GuestSpanRegistry { impl GuestSpanRegistry { #[cfg(test)] pub(crate) fn new(context: WasmInvocationContext) -> Self { - Self::with_manual_export(context, false) + Self::with_manual_export(context, false, false) + } + + /// Test-only constructor for an opted-in tenant, so the redaction tests can + /// assert both directions rather than only the safe one. + #[cfg(test)] + pub(crate) fn new_exporting_content(context: WasmInvocationContext) -> Self { + Self::with_manual_export(context, false, true) } - pub(crate) fn for_invocation(context: WasmInvocationContext, needs_wasi: bool) -> Self { - Self::with_manual_export(context, needs_wasi) + pub(crate) fn for_invocation( + context: WasmInvocationContext, + needs_wasi: bool, + export_llm_content: bool, + ) -> Self { + Self::with_manual_export(context, needs_wasi, export_llm_content) } - fn with_manual_export(context: WasmInvocationContext, manual_export: bool) -> Self { + fn with_manual_export( + context: WasmInvocationContext, + manual_export: bool, + export_llm_content: bool, + ) -> Self { clear_thread_local_guest_spans(); Self { context, manual_export, + export_llm_content, next_id: 1, total_started: 0, max_spans: DEFAULT_MAX_GUEST_SPANS, @@ -191,11 +215,20 @@ impl GuestSpanRegistry { error.message = tracing::field::Empty, exception.message = tracing::field::Empty, ); - update_span_name(&span, name); - apply_attributes(&span, &payload.attributes); + // ADR-0166: the span name is guest-supplied free text on the same channel + // as the attributes, so it takes the same bound. The hint path clamps + // `X-Temper-Span-Name` for the same reason. + let name = if self.export_llm_content { + name.to_string() + } else { + clamp_redacted_metadata_value(name).unwrap_or_else(|| name.to_string()) + }; + update_span_name(&span, &name); + let attributes = allowed_attributes(&payload.attributes, self.export_llm_content); + apply_attributes(&span, &attributes); enter_thread_local_guest_span(&span); let (trace_id, span_id) = tracing_span_ids(&span); - let attributes = manual_span_attributes(&self.context, id, &payload.attributes); + let attributes = manual_span_attributes(&self.context, id, &attributes); self.spans.insert( id, @@ -226,9 +259,12 @@ impl GuestSpanRegistry { if name.is_empty() { return Err("guest span event name must not be empty".to_string()); } + let export_llm_content = self.export_llm_content; let entry = self.span_mut(span_id)?; - let attrs = payload - .attributes + // Span events are a third way for a guest to attach attributes to a span, + // so they pass the same filter as `start_span` / `set_span_attributes`. + let event_attributes = allowed_attributes(&payload.attributes, export_llm_content); + let attrs = event_attributes .iter() .filter_map(|(key, value)| key_value_from_json(key, value)) .collect::>(); @@ -240,7 +276,7 @@ impl GuestSpanRegistry { entry.events.push(GuestSpanManualEvent { name: name.to_string(), timestamp: SystemTime::now(), - attributes: allowed_attributes(&payload.attributes), + attributes: event_attributes, }); Ok(()) } @@ -252,9 +288,11 @@ impl GuestSpanRegistry { ) -> Result<(), String> { let payload: GuestSpanAttributesPayload = serde_json::from_str(payload_json) .map_err(|e| format!("invalid guest span attributes payload: {e}"))?; + let export_llm_content = self.export_llm_content; let entry = self.span_mut(span_id)?; - apply_attributes(&entry.span, &payload.attributes); - merge_allowed_attributes(&mut entry.attributes, &payload.attributes); + let attributes = allowed_attributes(&payload.attributes, export_llm_content); + apply_attributes(&entry.span, &attributes); + entry.attributes.extend(attributes); Ok(()) } @@ -286,9 +324,11 @@ impl GuestSpanRegistry { .ok_or_else(|| format!("unknown guest span id: {span_id}"))?; self.stack.pop(); let mut entry = entry; - merge_allowed_attributes(&mut entry.attributes, &payload.attributes); + let export_llm_content = self.export_llm_content; + let attributes = allowed_attributes(&payload.attributes, export_llm_content); + entry.attributes.extend(attributes.clone()); merge_end_status_attributes(&mut entry.attributes, &payload); - apply_attributes(&entry.span, &payload.attributes); + apply_attributes(&entry.span, &attributes); apply_end_status(&entry.span, &payload); exit_thread_local_guest_span(); end_otel_span(&entry.span); @@ -342,27 +382,6 @@ impl GuestSpanRegistry { } } -pub(crate) fn guest_span_attribute_allowed(key: &str) -> bool { - let key = key.trim(); - if key.is_empty() { - return false; - } - !matches!( - key, - "otel.name" - | "otel.kind" - | "trace_id" - | "span_id" - | "parent_span_id" - | "dd.trace_id" - | "dd.span_id" - | "otel.trace_id" - | "otel.span_id" - | "_otel.parent_trace_id" - | "_otel.parent_span_id" - ) && !key.starts_with("_otel.") -} - fn update_span_name(span: &tracing::Span, name: &str) { span.record("otel.name", name); span.context().span().update_name(name.to_string()); @@ -384,25 +403,13 @@ fn clear_thread_local_guest_spans() { ENTERED_GUEST_SPANS.with(|guards| guards.borrow_mut().clear()); } -fn allowed_attributes(attributes: &BTreeMap) -> BTreeMap { - attributes - .iter() - .filter(|(key, _)| guest_span_attribute_allowed(key)) - .map(|(key, value)| (key.clone(), value.clone())) - .collect() -} - -fn merge_allowed_attributes( - target: &mut BTreeMap, - attributes: &BTreeMap, -) { - for (key, value) in attributes { - if guest_span_attribute_allowed(key) { - target.insert(key.clone(), value.clone()); - } - } -} - +/// Record already-filtered attributes onto a span. +/// +/// This takes the output of [`allowed_attributes`] rather than raw guest input, +/// deliberately: filtering in two places meant a mutation to either could be +/// masked by the other, and adversarial review showed that a filter dropped here +/// left every test green. There is now one place where guest attributes are +/// judged, and every path reaches the span through it. fn apply_attributes(span: &tracing::Span, attributes: &BTreeMap) { for (key, value) in attributes { let Some(kv) = key_value_from_json(key, value) else { diff --git a/crates/temper-wasm/src/engine/guest_spans/export.rs b/crates/temper-wasm/src/engine/guest_spans/export.rs index d39f3fe91..ca1953538 100644 --- a/crates/temper-wasm/src/engine/guest_spans/export.rs +++ b/crates/temper-wasm/src/engine/guest_spans/export.rs @@ -9,7 +9,7 @@ use tracing_opentelemetry::OpenTelemetrySpanExt as _; use crate::types::WasmInvocationContext; -use super::{GuestSpanEndPayload, GuestSpanEntry, allowed_attributes, key_value_from_json}; +use super::{GuestSpanEndPayload, GuestSpanEntry, key_value_from_json}; pub(super) fn manual_parent_context() -> Option { let context = tracing::Span::current().context(); @@ -34,7 +34,7 @@ pub(super) fn manual_span_attributes( span_id: i64, attributes: &BTreeMap, ) -> BTreeMap { - let mut manual = allowed_attributes(attributes); + let mut manual = attributes.clone(); manual.insert("tenant".to_string(), Value::String(context.tenant.clone())); manual.insert( "entity_type".to_string(), diff --git a/crates/temper-wasm/src/engine/guest_spans/redaction.rs b/crates/temper-wasm/src/engine/guest_spans/redaction.rs new file mode 100644 index 000000000..3b8317509 --- /dev/null +++ b/crates/temper-wasm/src/engine/guest_spans/redaction.rs @@ -0,0 +1,81 @@ +//! ADR-0166: what a guest may name in its own span attributes. +//! +//! A guest module chooses these keys, so the `gen_ai.*` namespace — the one LLM +//! Observability and the GenAI dashboards read — is attacker-chosen text. For a +//! tenant that has not opted into content export, only recognised metadata keys +//! survive inside that namespace, with bounded values. + +use std::collections::BTreeMap; + +use serde_json::Value; + +use crate::host_trait::span_hints::{ + clamp_llm_metadata_json, is_llm_namespace_key, llm_namespace_attr_allowed, +}; + +/// The single filter every guest-supplied span attribute passes through, on both +/// the OTel path and the manual-export path. +pub(super) fn allowed_attributes( + attributes: &BTreeMap, + export_llm_content: bool, +) -> BTreeMap { + attributes + .iter() + .filter(|(key, _)| guest_span_attribute_allowed(key)) + .filter_map(|(key, value)| { + redacted_guest_attribute_value(key, value, export_llm_content) + .map(|value| (key.clone(), value)) + }) + .collect() +} + +/// Apply ADR-0166 to one guest-supplied span attribute. Returns `None` when the +/// attribute must not be recorded at all. +/// +/// A guest module names its own span attributes, so the `gen_ai.*` namespace — +/// the one LLM Observability and the GenAI dashboards read — is attacker-chosen +/// text. For a tenant that has not opted into content export, only recognised +/// metadata keys survive inside that namespace, and their values are bounded so +/// a prompt cannot ride inside `gen_ai.request.model`. Attributes outside the +/// namespace are the module's own application telemetry and pass through: they +/// carry no agreed meaning to redact against, and dropping them would remove +/// working guest observability. That boundary is stated in ADR-0166. +fn redacted_guest_attribute_value( + key: &str, + value: &Value, + export_llm_content: bool, +) -> Option { + if export_llm_content { + return Some(value.clone()); + } + if !llm_namespace_attr_allowed(key) { + return None; + } + if !is_llm_namespace_key(key) { + return Some(value.clone()); + } + clamp_llm_metadata_json(value) +} + +/// Keys a guest may not set, because the host owns them: OTel span identity and +/// the internal `_otel.*` correlation fields. +pub(super) fn guest_span_attribute_allowed(key: &str) -> bool { + let key = key.trim(); + if key.is_empty() { + return false; + } + !matches!( + key, + "otel.name" + | "otel.kind" + | "trace_id" + | "span_id" + | "parent_span_id" + | "dd.trace_id" + | "dd.span_id" + | "otel.trace_id" + | "otel.span_id" + | "_otel.parent_trace_id" + | "_otel.parent_span_id" + ) && !key.starts_with("_otel.") +} diff --git a/crates/temper-wasm/src/engine/guest_spans_test.rs b/crates/temper-wasm/src/engine/guest_spans_test.rs index 5c0bfef39..1977d125a 100644 --- a/crates/temper-wasm/src/engine/guest_spans_test.rs +++ b/crates/temper-wasm/src/engine/guest_spans_test.rs @@ -126,3 +126,178 @@ fn cleanup_closes_unended_spans() { registry.cleanup_unclosed(); assert!(registry.enter_active().is_none()); } + +/// ARN-243 / ADR-0166. The guest manual-span API lets an untrusted module name +/// its own span attributes, including the canonical `gen_ai.*` keys that LLM +/// Observability reads. For a tenant that has not opted into content export, a +/// module holding a prompt and a completion must not be able to publish them +/// simply by calling `host_start_span` with those names. +#[test] +fn guest_span_attributes_drop_llm_content_for_non_opted_in_tenant() { + let mut attributes = BTreeMap::new(); + attributes.insert( + "gen_ai.input.messages".to_string(), + Value::String("SECRET PROMPT".to_string()), + ); + attributes.insert( + "gen_ai.completion".to_string(), + Value::String("SECRET COMPLETION".to_string()), + ); + attributes.insert( + "gen_ai.request.model".to_string(), + Value::String("claude-opus-4-8".to_string()), + ); + // A name inside the namespace that no denylist enumerates. + attributes.insert( + "gen_ai.response.text".to_string(), + Value::String("SECRET COMPLETION".to_string()), + ); + // The module's own application telemetry, which must keep working. + attributes.insert("order.id".to_string(), Value::String("A-17".to_string())); + + let redacted = manual_span_attributes(&context(), 1, &allowed_attributes(&attributes, false)); + + assert_eq!( + redacted.get("gen_ai.input.messages"), + None, + "prompt must not export" + ); + assert_eq!( + redacted.get("gen_ai.completion"), + None, + "completion must not export" + ); + assert_eq!( + redacted.get("gen_ai.response.text"), + None, + "an unrecognized gen_ai.* key must not export just because it is unlisted" + ); + assert_eq!( + redacted.get("gen_ai.request.model"), + Some(&Value::String("claude-opus-4-8".to_string())), + "recognized metadata must survive" + ); + assert_eq!( + redacted.get("order.id"), + Some(&Value::String("A-17".to_string())), + "non-LLM guest telemetry must keep working" + ); + + // The opted-in tenant is unaffected. + let exported = manual_span_attributes(&context(), 1, &allowed_attributes(&attributes, true)); + assert_eq!( + exported.get("gen_ai.completion"), + Some(&Value::String("SECRET COMPLETION".to_string())), + "an opted-in tenant still exports content" + ); +} + +/// A key name cannot make an untrusted value into metadata: without a bound, a +/// module hides the whole prompt inside `gen_ai.request.model` and the allowlist +/// waves it through. +#[test] +fn guest_span_metadata_values_are_bounded_for_non_opted_in_tenant() { + let prompt = "P".repeat(4096); + let mut attributes = BTreeMap::new(); + attributes.insert( + "gen_ai.request.model".to_string(), + Value::String(prompt.clone()), + ); + + let redacted = manual_span_attributes(&context(), 1, &allowed_attributes(&attributes, false)); + let Some(Value::String(value)) = redacted.get("gen_ai.request.model") else { + panic!("metadata key should survive, bounded"); + }; + assert!( + value.len() <= 256, + "metadata value must be clamped, got {} bytes", + value.len() + ); + + let exported = manual_span_attributes(&context(), 1, &allowed_attributes(&attributes, true)); + assert_eq!( + exported.get("gen_ai.request.model"), + Some(&Value::String(prompt)), + "an opted-in tenant is not clamped" + ); +} + +/// A clamp that only matches `Value::String` is not a clamp. `gen_ai.request.model +/// = [""]` is a recognised metadata key holding unbounded +/// content, and its serialized form reaches the backend all the same. No key in +/// the metadata allowlist is structured, so structured values are dropped. +#[test] +fn structured_llm_metadata_values_do_not_escape_the_clamp() { + let prompt = "P".repeat(4096); + let mut attributes = BTreeMap::new(); + attributes.insert( + "gen_ai.request.model".to_string(), + Value::Array(vec![Value::String(prompt.clone())]), + ); + attributes.insert( + "gen_ai.response.model".to_string(), + serde_json::json!({ "smuggled": prompt }), + ); + // Scalars that are legitimately structured-adjacent still work. + attributes.insert( + "gen_ai.usage.input_tokens".to_string(), + Value::Number(42.into()), + ); + attributes.insert( + "gen_ai.request.temperature".to_string(), + serde_json::json!(0.7), + ); + + let redacted = allowed_attributes(&attributes, false); + + assert_eq!( + redacted.get("gen_ai.request.model"), + None, + "an array under a metadata key must be dropped, not passed through" + ); + assert_eq!( + redacted.get("gen_ai.response.model"), + None, + "an object under a metadata key must be dropped" + ); + assert_eq!( + redacted.get("gen_ai.usage.input_tokens"), + Some(&Value::Number(42.into())), + "numeric metadata must keep working" + ); + assert!(redacted.contains_key("gen_ai.request.temperature")); + + // The opted-in tenant keeps everything. + let exported = allowed_attributes(&attributes, true); + assert!(exported.contains_key("gen_ai.request.model")); +} + +/// The guest chooses its span *name* too, and it is unbounded free text on the +/// same channel as the attributes. +#[test] +fn guest_span_name_is_bounded_for_non_opted_in_tenant() { + let registry_name = |export: bool| { + let mut registry = if export { + GuestSpanRegistry::new_exporting_content(context()) + } else { + GuestSpanRegistry::new(context()) + }; + let payload = serde_json::json!({ "name": "N".repeat(4096) }).to_string(); + registry.start_span(&payload).expect("span starts"); + registry + }; + + // Non-opted-in: the recorded name is clamped. + let registry = registry_name(false); + let entry = registry.spans.values().next().expect("one span"); + assert!( + entry.name.len() <= 256, + "guest span name must be clamped, got {} bytes", + entry.name.len() + ); + + // Opted-in: untouched. + let registry = registry_name(true); + let entry = registry.spans.values().next().expect("one span"); + assert_eq!(entry.name.len(), 4096, "an opted-in tenant is not clamped"); +} diff --git a/crates/temper-wasm/src/engine/host_functions.rs b/crates/temper-wasm/src/engine/host_functions.rs index abe0d711b..66c22e4c0 100644 --- a/crates/temper-wasm/src/engine/host_functions.rs +++ b/crates/temper-wasm/src/engine/host_functions.rs @@ -97,6 +97,14 @@ pub(crate) enum FieldResolution { HostError, } +/// True if a `[ptr, ptr + len)` read lies fully within a guest linear memory of +/// `mem_size` bytes. Checked BEFORE allocating a read buffer so a guest-supplied +/// `len` can't force a large host allocation ahead of wasmtime's own bounds check +/// (ARN-226). `checked_add` also rejects a `ptr + len` that overflows `usize`. +pub(super) fn guest_read_bounds_ok(mem_size: usize, ptr: usize, len: usize) -> bool { + ptr.checked_add(len).is_some_and(|end| end <= mem_size) +} + fn read_guest_string(caller: &mut Caller<'_, HostState>, ptr: i32, len: i32) -> Result { if ptr < 0 || len < 0 { return Err(()); @@ -105,9 +113,18 @@ fn read_guest_string(caller: &mut Caller<'_, HostState>, ptr: i32, len: i32) -> let Some(memory) = memory else { return Err(()); }; + if !guest_read_bounds_ok(memory.data_size(&mut *caller), ptr as usize, len as usize) { + tracing::warn!( + host_fn = "read_guest_string", + ptr, + len, + "guest string read range exceeds linear memory; returning error before allocating" + ); + return Err(()); + } let mut buf = vec![0u8; len as usize]; memory - .read(caller, ptr as usize, &mut buf) + .read(&mut *caller, ptr as usize, &mut buf) .map_err(|_| ())?; String::from_utf8(buf).map_err(|_| ()) } @@ -136,6 +153,16 @@ fn read_guest_bytes( ); return Err(()); } + if !guest_read_bounds_ok(memory.data_size(caller), ptr as usize, len as usize) { + tracing::warn!( + host_fn, + operand = what, + ptr, + len, + "guest read range exceeds linear memory; returning error before allocating" + ); + return Err(()); + } let mut buf = vec![0u8; len as usize]; if let Err(error) = memory.read(caller, ptr as usize, &mut buf) { tracing::warn!( @@ -160,7 +187,12 @@ fn read_guest_lossy( what: &'static str, ) -> Result { let buf = read_guest_bytes(caller, memory, ptr, len, host_fn, what)?; - Ok(String::from_utf8_lossy(&buf).to_string()) + // Take the buffer by value when it is already valid UTF-8 (the common case); + // only the invalid path pays for a second, lossy copy. + Ok(match String::from_utf8(buf) { + Ok(text) => text, + Err(error) => String::from_utf8_lossy(error.as_bytes()).into_owned(), + }) } /// Write `bytes` into guest memory at `ptr`. @@ -424,15 +456,9 @@ pub(super) fn link_host_functions(linker: &mut Linker) -> Result<(), "env", "host_emit_progress", |mut caller: Caller<'_, HostState>, ptr: i32, len: i32| -> i32 { - let memory = caller.get_export("memory").and_then(|e| e.into_memory()); - let Some(memory) = memory else { - return -1; - }; - let mut buf = vec![0u8; len as usize]; - if memory.read(&caller, ptr as usize, &mut buf).is_err() { - return -1; - } - let Ok(payload) = String::from_utf8(buf) else { + // ARN-226: read via the bounds-checked helper so a guest-supplied + // `len` can't drive a host allocation ahead of the bounds check. + let Ok(payload) = read_guest_string(&mut caller, ptr, len) else { return -1; }; let _guest_span = caller.data().guest_spans.enter_active(); @@ -450,15 +476,9 @@ pub(super) fn link_host_functions(linker: &mut Linker) -> Result<(), "env", "host_emit_wide_event", |mut caller: Caller<'_, HostState>, ptr: i32, len: i32| -> i32 { - let memory = caller.get_export("memory").and_then(|e| e.into_memory()); - let Some(memory) = memory else { - return -1; - }; - let mut buf = vec![0u8; len as usize]; - if memory.read(&caller, ptr as usize, &mut buf).is_err() { - return -1; - } - let Ok(payload) = String::from_utf8(buf) else { + // ARN-226: read via the bounds-checked helper so a guest-supplied + // `len` can't drive a host allocation ahead of the bounds check. + let Ok(payload) = read_guest_string(&mut caller, ptr, len) else { return -1; }; let _guest_span = caller.data().guest_spans.enter_active(); @@ -476,15 +496,9 @@ pub(super) fn link_host_functions(linker: &mut Linker) -> Result<(), "env", "host_log_structured", |mut caller: Caller<'_, HostState>, ptr: i32, len: i32| -> i32 { - let memory = caller.get_export("memory").and_then(|e| e.into_memory()); - let Some(memory) = memory else { - return -1; - }; - let mut buf = vec![0u8; len as usize]; - if memory.read(&caller, ptr as usize, &mut buf).is_err() { - return -1; - } - let Ok(payload) = String::from_utf8(buf) else { + // ARN-226: read via the bounds-checked helper so a guest-supplied + // `len` can't drive a host allocation ahead of the bounds check. + let Ok(payload) = read_guest_string(&mut caller, ptr, len) else { return -1; }; let _guest_span = caller.data().guest_spans.enter_active(); @@ -502,15 +516,9 @@ pub(super) fn link_host_functions(linker: &mut Linker) -> Result<(), "env", "host_emit_metric", |mut caller: Caller<'_, HostState>, ptr: i32, len: i32| -> i32 { - let memory = caller.get_export("memory").and_then(|e| e.into_memory()); - let Some(memory) = memory else { - return -1; - }; - let mut buf = vec![0u8; len as usize]; - if memory.read(&caller, ptr as usize, &mut buf).is_err() { - return -1; - } - let Ok(payload) = String::from_utf8(buf) else { + // ARN-226: read via the bounds-checked helper so a guest-supplied + // `len` can't drive a host allocation ahead of the bounds check. + let Ok(payload) = read_guest_string(&mut caller, ptr, len) else { return -1; }; let _guest_span = caller.data().guest_spans.enter_active(); @@ -1323,14 +1331,17 @@ pub(super) fn link_host_functions(linker: &mut Linker) -> Result<(), return -3; }; - let mut name_buf = vec![0u8; field_name_len as usize]; - if memory - .read(&caller, field_name_ptr as usize, &mut name_buf) - .is_err() - { + // ARN-226: bounds-checked read before allocating. + let Ok(field_name) = read_guest_lossy( + &caller, + &memory, + field_name_ptr, + field_name_len, + "host_read_field", + "field_name", + ) else { return -3; - } - let field_name = String::from_utf8_lossy(&name_buf).to_string(); + }; let started = Instant::now(); let _guest_span = caller.data().guest_spans.enter_active(); let span = tracing::info_span!( @@ -1708,46 +1719,51 @@ pub(super) fn link_host_functions(linker: &mut Linker) -> Result<(), return -1; }; - // Read IOA source - let mut ioa_buf = vec![0u8; ioa_len as usize]; - if memory - .read(&caller, ioa_ptr as usize, &mut ioa_buf) - .is_err() - { + // ARN-226: bounds-checked reads before allocating. + let Ok(ioa_source) = read_guest_lossy( + &caller, + &memory, + ioa_ptr, + ioa_len, + "host_evaluate_spec", + "ioa_source", + ) else { return -1; - } - let ioa_source = String::from_utf8_lossy(&ioa_buf).to_string(); - - // Read current state - let mut state_buf = vec![0u8; state_len as usize]; - if memory - .read(&caller, state_ptr as usize, &mut state_buf) - .is_err() - { + }; + let Ok(current_state) = read_guest_lossy( + &caller, + &memory, + state_ptr, + state_len, + "host_evaluate_spec", + "current_state", + ) else { return -1; - } - let current_state = String::from_utf8_lossy(&state_buf).to_string(); - - // Read action - let mut action_buf = vec![0u8; action_len as usize]; - if memory - .read(&caller, action_ptr as usize, &mut action_buf) - .is_err() - { + }; + let Ok(action) = read_guest_lossy( + &caller, + &memory, + action_ptr, + action_len, + "host_evaluate_spec", + "action", + ) else { return -1; - } - let action = String::from_utf8_lossy(&action_buf).to_string(); + }; - // Read params JSON + // Read params JSON (ARN-226: bounds-checked read before allocating). let params_json = if params_len > 0 { - let mut params_buf = vec![0u8; params_len as usize]; - if memory - .read(&caller, params_ptr as usize, &mut params_buf) - .is_err() - { - return -1; + match read_guest_lossy( + &caller, + &memory, + params_ptr, + params_len, + "host_evaluate_spec", + "params_json", + ) { + Ok(s) => s, + Err(()) => return -1, } - String::from_utf8_lossy(¶ms_buf).to_string() } else { "{}".to_string() }; @@ -2148,10 +2164,17 @@ pub(super) fn link_host_functions(linker: &mut Linker) -> Result<(), let memory = caller.get_export("memory").and_then(|e| e.into_memory()); let Some(memory) = memory else { return -4 }; - let mut buf = vec![0u8; head_len as usize]; - if memory.read(&caller, head_ptr as usize, &mut buf).is_err() { + // ARN-226: bounds-checked read before allocating. + let Ok(buf) = read_guest_bytes( + &caller, + &memory, + head_ptr, + head_len, + "host_http_stream_send_response_head", + "head", + ) else { return -4; - } + }; #[derive(serde::Deserialize)] struct RawHead { status: u16, @@ -2218,6 +2241,41 @@ pub(super) fn link_host_functions(linker: &mut Linker) -> Result<(), mod tests { use super::*; + #[test] + fn guest_read_bounds_ok_rejects_out_of_bounds_and_overflow() { + // ARN-226: a guest-supplied `len` must be validated against the guest memory + // size BEFORE a read buffer is allocated. Out-of-bounds or overflowing + // `(ptr, len)` ranges must be rejected so a huge `len` can't drive a large + // host allocation. + let mem = 64 * 1024; // 64 KiB guest linear memory + assert!( + !guest_read_bounds_ok(mem, 0, i32::MAX as usize), + "an i32::MAX len must be rejected before allocating ~2 GiB" + ); + assert!( + !guest_read_bounds_ok(mem, 0, mem + 1), + "a len past the end of memory must be rejected" + ); + assert!( + !guest_read_bounds_ok(mem, mem, 1), + "a read starting at the end of memory must be rejected" + ); + assert!( + !guest_read_bounds_ok(mem, usize::MAX, 1), + "a ptr + len that overflows usize must be rejected" + ); + // Legitimate in-bounds reads are still allowed. + assert!( + guest_read_bounds_ok(mem, 0, mem), + "a full in-bounds read is allowed" + ); + assert!(guest_read_bounds_ok(mem, 0, 0), "an empty read is allowed"); + assert!( + guest_read_bounds_ok(mem, 100, 200), + "an interior read is allowed" + ); + } + fn ctx_json_with_fields(fields: serde_json::Value) -> String { serde_json::json!({ "entity_state": { "fields": fields } diff --git a/crates/temper-wasm/src/engine/mod.rs b/crates/temper-wasm/src/engine/mod.rs index 63a5a359b..3c9d35eea 100644 --- a/crates/temper-wasm/src/engine/mod.rs +++ b/crates/temper-wasm/src/engine/mod.rs @@ -3,6 +3,9 @@ //! Modules are compiled once and cached by SHA-256 hash. Each invocation //! gets a fresh `Store` with fuel + memory limits (TigerStyle budgets). +#[cfg(test)] +#[path = "guest_read_bounds_test.rs"] +mod guest_read_bounds_test; mod guest_spans; mod host_functions; mod telemetry; @@ -96,13 +99,48 @@ impl ResourceLimiter for MemoryLimiter { fn table_growing( &mut self, _current: usize, - _desired: usize, + desired: usize, _maximum: Option, ) -> Result { - Ok(true) + // Bound table growth too: table elements are host allocations (a funcref/ + // externref slot each) that the memory limiter does not cover, so an + // unbounded `table.grow` is a host-memory exhaustion vector (ARN-169). + // Deny past the cap (grow returns -1) rather than trapping. + Ok(desired <= MAX_TABLE_ELEMENTS) + } + + // The per-table/per-memory element caps above only bound host allocation if + // the *number* of tables and memories is also bounded — wasmtime's default + // limiter allows 10,000 of each per store, so a guest could otherwise declare + // thousands of tables each at the element cap and multiply the budget. Cap the + // counts to what a single-module guest actually needs (ARN-169). + fn memories(&self) -> usize { + MAX_MEMORIES + } + + fn tables(&self) -> usize { + MAX_TABLES } } +/// Maximum number of elements any single guest table may hold. Table slots are +/// host allocations outside the linear-memory budget; this caps the host memory a +/// guest can force through `table.grow` (ARN-169). +const MAX_TABLE_ELEMENTS: usize = 1_000_000; + +/// Maximum number of linear memories a guest store may create. With +/// `wasm_multi_memory(false)` a module already declares at most one; this makes +/// the store-level bound explicit so the per-memory `max_memory` budget cannot be +/// multiplied (ARN-169). +const MAX_MEMORIES: usize = 1; + +/// Maximum number of tables a guest store may create. Together with +/// `MAX_TABLE_ELEMENTS` this bounds total table host memory to +/// `MAX_TABLES * MAX_TABLE_ELEMENTS` slots. Generous enough for ordinary +/// reference-types modules (typically one funcref table) while preventing the +/// default 10,000-table multiplier (ARN-169). +const MAX_TABLES: usize = 8; + /// Compiled module cache entry. struct CachedModule { /// SHA-256 hash that keys this compiled module in the cache. @@ -229,6 +267,18 @@ impl WasmEngine { config.consume_fuel(true); config.epoch_interruption(true); config.wasm_component_model(true); + // Pin the WASM feature surface across the wasmtime 29 -> 36 bump (ARN-169). + // wasmtime 30+ turns several proposals on by default; leaving them at the + // 36 defaults would silently widen the guest attack surface relative to 29: + // - memory64: rejected at compile on 29, and a prerequisite for the very + // advisory this bump fixes (RUSTSEC-2026-0096) — keep it off; + // - threads/shared memory and multiple memories: each lets a guest hold a + // linear memory the per-memory `max_memory` limiter does not sum, so a + // guest could exceed the intended per-invocation memory budget. + // Temper's guests are single-memory wasm32 modules and use none of these. + config.wasm_memory64(false); + config.wasm_threads(false); + config.wasm_multi_memory(false); if let Some(strategy) = configured_profiling_strategy() { config.profiler(strategy); } @@ -484,7 +534,7 @@ impl WasmEngine { ); let _entered = phase.enter(); let wasi_stderr_pipe = if needs_wasi { - Some(wasmtime_wasi::pipe::MemoryOutputPipe::new(64 * 1024)) + Some(wasmtime_wasi::p2::pipe::MemoryOutputPipe::new(64 * 1024)) } else { None }; @@ -498,6 +548,10 @@ impl WasmEngine { None }; + // ADR-0166: the guest-facing span API is fed by an untrusted module, so + // it needs the same per-tenant content decision the host HTTP path uses. + // Read it from the host itself, whose default is redact. + let export_llm_content = host.exports_llm_content(); let host_state = HostState { context_json: context_json.clone(), result_json: None, @@ -509,7 +563,11 @@ impl WasmEngine { streams, wasi_ctx, blob_cache, - guest_spans: GuestSpanRegistry::for_invocation(context.clone(), needs_wasi), + guest_spans: GuestSpanRegistry::for_invocation( + context.clone(), + needs_wasi, + export_llm_content, + ), }; (wasi_stderr_pipe, host_state) }; @@ -663,6 +721,19 @@ impl WasmEngine { let result_len = u32::from_le_bytes(len_bytes) as usize; phase.record("result_bytes", result_len as u64); + // ARN-226: bound the result allocation by the guest's memory size + // before allocating, so a forged length prefix can't drive a large + // host allocation ahead of the bounds check. + if !host_functions::guest_read_bounds_ok( + memory.data_size(&store), + result_ptr as usize, + result_len, + ) { + store.data_mut().guest_spans.cleanup_unclosed(); + return Err(WasmError::Invocation( + "result length exceeds guest linear memory".to_string(), + )); + } let mut result_bytes = vec![0u8; result_len]; if let Err(e) = memory.read(&store, result_ptr as usize, &mut result_bytes) { store.data_mut().guest_spans.cleanup_unclosed(); diff --git a/crates/temper-wasm/src/engine/tests.rs b/crates/temper-wasm/src/engine/tests.rs index 8cf060639..9b8d46c9c 100644 --- a/crates/temper-wasm/src/engine/tests.rs +++ b/crates/temper-wasm/src/engine/tests.rs @@ -35,13 +35,20 @@ const WAT_INFINITE_LOOP: &str = r#" ) "#; -// Tries to grow memory by 1000 pages (64 MB) — exceeds 16 MB default. +// Tries to grow memory by 1000 pages (64 MB) — exceeds the budget. Traps +// (`unreachable`) if the grow unexpectedly *succeeds* (result != -1), so a test +// asserting the invocation is Ok proves the limiter actually denied the growth — +// it is not vacuous. const WAT_MEMORY_GROW: &str = r#" (module (memory (export "memory") 1) (func (export "run") (param i32 i32) (result i32) (memory.grow (i32.const 1000)) - drop + i32.const -1 + i32.ne + if + unreachable + end i32.const 0 ) ) @@ -176,7 +183,7 @@ impl WasmHost for SlowHttpHost { fn log(&self, _level: &str, _message: &str) {} } -fn make_context() -> WasmInvocationContext { +pub(super) fn make_context() -> WasmInvocationContext { WasmInvocationContext { tenant: "test".into(), entity_type: "Order".into(), @@ -196,11 +203,11 @@ fn make_context() -> WasmInvocationContext { } } -fn make_host() -> Arc { +pub(super) fn make_host() -> Arc { Arc::new(SimWasmHost::new()) } -fn make_streams() -> Arc> { +pub(super) fn make_streams() -> Arc> { Arc::new(RwLock::new(StreamRegistry::default())) } @@ -455,20 +462,14 @@ async fn memory_growth_denied_by_limiter() { .invoke(&hash, &make_context(), make_host(), &limits, make_streams()) .await; - // Module returns normally (memory.grow returned -1 per spec — not a trap). - // The invocation itself succeeds (no crash), but the result is empty - // because the module didn't call host_set_result. - assert!( - result.is_ok() || matches!(result, Err(WasmError::Invocation(_))), - "memory denial should not cause fuel/timeout error, got: {result:?}" - ); - // Critically: no panic, no FuelExhausted, no Timeout. + // The module traps (`unreachable`) if `memory.grow` returned anything other + // than -1, so a successful invocation proves the limiter actually denied the + // 1000-page growth (grow returned -1 per spec, not a crash). If the limiter + // had NOT fired, the grow would succeed and the module would trap — surfacing + // as an error, failing this assertion. Not vacuous. assert!( - !matches!( - result, - Err(WasmError::FuelExhausted) | Err(WasmError::Timeout(_)) - ), - "memory denial should not be misclassified" + result.is_ok(), + "memory.grow must return -1 (limiter denied growth), got: {result:?}" ); } @@ -495,3 +496,229 @@ async fn noop_module_completes() { "noop should not hit resource limits, got: {result:?}" ); } + +// ARN-169: WASI compatibility + sandbox-limit regressions across the +// wasmtime 29 -> 36 bump. A module declaring more initial memory than the +// budget must be rejected at instantiation, and a WASIp1 module must still +// invoke end to end after the `wasmtime_wasi::p2` module reshuffle. + +const WAT_INITIAL_MEMORY_TWO_PAGES: &str = r#" + (module + (memory (export "memory") 2) + (func (export "run") (param i32 i32) (result i32) + i32.const 0 + ) + ) +"#; + +const WAT_WASI_STDERR: &str = r#" + (module + (import "wasi_snapshot_preview1" "fd_write" + (func $fd_write (param i32 i32 i32 i32) (result i32))) + (memory (export "memory") 1) + (data (i32.const 32) "wasi stderr\n") + (func (export "run") (param i32 i32) (result i32) + i32.const 0 + i32.const 32 + i32.store + i32.const 4 + i32.const 12 + i32.store + i32.const 2 + i32.const 0 + i32.const 1 + i32.const 16 + call $fd_write + if + unreachable + end + i32.const 0 + ) + ) +"#; + +#[tokio::test] +async fn initial_memory_over_budget_is_rejected() { + let engine = WasmEngine::new().unwrap(); + let hash = engine + .compile_and_cache(WAT_INITIAL_MEMORY_TWO_PAGES.as_bytes()) + .unwrap(); + let limits = WasmResourceLimits { + max_memory: 64 * 1024, // 1 WASM page; the module declares 2 + ..WasmResourceLimits::default() + }; + + let result = engine + .invoke(&hash, &make_context(), make_host(), &limits, make_streams()) + .await; + + assert!( + matches!(result, Err(WasmError::Instantiation(_))), + "oversized initial memory must fail at instantiation: {result:?}" + ); +} + +#[tokio::test] +async fn wasi_preview1_module_invokes_end_to_end() { + let engine = WasmEngine::new().unwrap(); + let hash = engine + .compile_and_cache(WAT_WASI_STDERR.as_bytes()) + .unwrap(); + + let result = engine + .invoke( + &hash, + &make_context(), + make_host(), + &WasmResourceLimits::default(), + make_streams(), + ) + .await; + + assert!(result.is_ok(), "WASIp1 invocation failed: {result:?}"); +} + +// ARN-169: the wasmtime 36 feature surface is pinned to match the 29 sandbox — +// memory64 and multiple memories are rejected at compile, and table growth is +// bounded like linear memory. + +const WAT_MEMORY64: &str = r#" + (module + (memory i64 1) + (func (export "run") (param i32 i32) (result i32) + i32.const 0 + ) + ) +"#; + +const WAT_MULTI_MEMORY: &str = r#" + (module + (memory 1) + (memory 1) + (func (export "run") (param i32 i32) (result i32) + i32.const 0 + ) + ) +"#; + +const WAT_TABLE_GROW: &str = r#" + (module + (memory (export "memory") 1) + (table 1 funcref) + (func (export "run") (param i32 i32) (result i32) + (table.grow (ref.null func) (i32.const 2000000)) + i32.const -1 + i32.ne + if + unreachable + end + i32.const 0 + ) + ) +"#; + +#[test] +fn memory64_module_is_rejected() { + let engine = WasmEngine::new().unwrap(); + assert!( + engine.compile_and_cache(WAT_MEMORY64.as_bytes()).is_err(), + "memory64 must be rejected at compile — it widens the surface and was a \ + RUSTSEC-2026-0096 prerequisite" + ); +} + +#[test] +fn multi_memory_module_is_rejected() { + let engine = WasmEngine::new().unwrap(); + assert!( + engine + .compile_and_cache(WAT_MULTI_MEMORY.as_bytes()) + .is_err(), + "multiple memories must be rejected: the per-memory limiter does not sum them" + ); +} + +#[tokio::test] +async fn table_growth_denied_past_cap() { + let engine = WasmEngine::new().unwrap(); + let hash = engine.compile_and_cache(WAT_TABLE_GROW.as_bytes()).unwrap(); + + let result = engine + .invoke( + &hash, + &make_context(), + make_host(), + &WasmResourceLimits::default(), + make_streams(), + ) + .await; + + // The module traps if table.grow returned anything but -1, so a successful + // invocation proves the limiter denied the 2,000,000-element growth. + assert!( + result.is_ok(), + "table.grow past the cap must return -1 (limiter denied), got: {result:?}" + ); +} + +// A shared memory (threads proposal) must be rejected — the pin is load-bearing +// because wasmtime does not invoke the memory limiter for shared memories. +const WAT_SHARED_MEMORY: &str = r#" + (module + (memory 1 1 shared) + (func (export "run") (param i32 i32) (result i32) + i32.const 0 + ) + ) +"#; + +const WAT_MANY_TABLES: &str = r#" + (module + (memory (export "memory") 1) + (table $0 1 funcref) + (table $1 1 funcref) + (table $2 1 funcref) + (table $3 1 funcref) + (table $4 1 funcref) + (table $5 1 funcref) + (table $6 1 funcref) + (table $7 1 funcref) + (table $8 1 funcref) + (func (export "run") (param i32 i32) (result i32) + i32.const 0 + ) + ) +"#; +#[test] +fn shared_memory_module_is_rejected() { + let engine = WasmEngine::new().unwrap(); + assert!( + engine + .compile_and_cache(WAT_SHARED_MEMORY.as_bytes()) + .is_err(), + "shared memory must be rejected: the limiter is not consulted for it" + ); +} + +#[tokio::test] +async fn too_many_tables_is_rejected() { + // MAX_TABLES tables is allowed; more must be denied at instantiation so the + // per-table element cap is a real store-wide budget, not per-table only. + let engine = WasmEngine::new().unwrap(); + let hash = engine + .compile_and_cache(WAT_MANY_TABLES.as_bytes()) + .unwrap(); + let result = engine + .invoke( + &hash, + &make_context(), + make_host(), + &WasmResourceLimits::default(), + make_streams(), + ) + .await; + assert!( + matches!(result, Err(WasmError::Instantiation(_))), + "a module declaring more than MAX_TABLES tables must fail instantiation: {result:?}" + ); +} diff --git a/crates/temper-wasm/src/host_trait.rs b/crates/temper-wasm/src/host_trait.rs index 9851b398b..960421be1 100644 --- a/crates/temper-wasm/src/host_trait.rs +++ b/crates/temper-wasm/src/host_trait.rs @@ -21,13 +21,21 @@ use crate::workflow_headers::add_workflow_observability_headers; use temper_observe::wide_event::{self, EventKind, WideEvent}; mod guest_progress; -mod span_hints; +mod internal_http; +pub(crate) mod span_hints; +pub use internal_http::{InternalHttpCapability, InternalHttpCapabilityIssuerFn}; + +/// Re-exported so every channel that records guest-supplied `gen_ai.*` metadata +/// bounds it the same way, including the dispatch callback path in +/// `temper-server`. A second copy of the bound would drift. See ADR-0166. +pub use span_hints::{MAX_REDACTED_LLM_METADATA_VALUE_BYTES, clamp_redacted_metadata_value}; #[cfg(test)] pub(crate) use span_hints::{MAX_RESPONSE_CAPTURE_BYTES, SpanHints}; pub(crate) use span_hints::{ - apply_response_captures, apply_span_hints, datadog_visible_span_hint_field, - span_hint_otel_name, split_span_hint_headers, truncate_for_span_attr, + apply_response_captures, apply_span_hints, clamp_llm_metadata_json, + datadog_visible_span_hint_field, is_llm_namespace_key, llm_namespace_attr_allowed, + redact_llm_content_hints, span_hint_otel_name, split_span_hint_headers, truncate_for_span_attr, }; /// Host capabilities provided to WASM modules. @@ -50,6 +58,14 @@ pub struct HttpBatchResponse { #[async_trait] pub trait WasmHost: Send + Sync { + /// Whether this invocation's tenant opted into exporting raw LLM content to + /// telemetry (ADR-0166). Defaults to `false` so a host that does not answer + /// redacts: the guest-facing telemetry APIs read this, and the only safe + /// default for a channel fed by an untrusted module is to withhold content. + fn exports_llm_content(&self) -> bool { + false + } + /// Make an HTTP request. Returns (status_code, response_body). async fn http_call( &self, @@ -331,14 +347,16 @@ type TextHttpInterceptorFuture = pub struct ProductionWasmHost { /// HTTP client for making real requests. client: reqwest::Client, + /// Separate client whose redirect policy is disabled for capabilities. + internal_client: reqwest::Client, /// Secrets from env vars or a secret store. secrets: BTreeMap, /// Optional lazy secret resolver authoritative for guest secret lookups. secret_resolver: Option, /// Canonical base URL for the local Temper API when known directly by the server. internal_api_base_url: Option, - /// Ambient platform bearer token for internal loopback calls. - internal_api_key: Option, + /// Trusted callback that issues one request-bound internal capability. + internal_capability_issuer: Option, /// Optional spec evaluator (provided by temper-server at construction). spec_evaluator: Option, /// Optional progress emitter (provided by temper-server at construction). @@ -351,6 +369,11 @@ pub struct ProductionWasmHost { text_http_interceptor: Option, /// Invocation context for auto-enriching guest telemetry. invocation_context: Option, + /// Whether this invocation's tenant may export raw LLM content (prompts, + /// completions, system instructions, tool arguments/results) captured from + /// guest HTTP calls to the telemetry backend. Defaults to `false` (redact); + /// the server sets it per tenant. See ADR-0166. + export_llm_content: bool, /// Registry of active streaming HTTP exchanges (ADR-0057). /// One per host instance; handle IDs are unique within the host. http_streams: Arc, @@ -444,6 +467,39 @@ fn blob_transport_semaphore() -> &'static Semaphore { SEMAPHORE.get_or_init(|| Semaphore::new(blob_transport_max_concurrency())) } +fn build_production_http_client( + secrets: &BTreeMap, + timeout: std::time::Duration, + disable_redirects: bool, +) -> reqwest::Client { + let mut builder = reqwest::Client::builder() + .connect_timeout(std::time::Duration::from_secs(10)) + .timeout(timeout); + if disable_redirects { + builder = builder + .redirect(reqwest::redirect::Policy::none()) + .no_proxy(); + } + + for (key, pem) in secrets { + if !key.starts_with("ca_cert:") { + continue; + } + match reqwest::Certificate::from_pem(pem.as_bytes()) { + Ok(cert) => { + builder = builder.add_root_certificate(cert); + } + Err(error) => { + tracing::warn!(key, error = %error, "failed to parse CA certificate from secret"); + } + } + } + + builder + .build() + .expect("production HTTP client configuration must be valid") +} + fn remote_blob_backend<'a>(secrets: &'a BTreeMap, url: &str) -> Option<&'a str> { let endpoint = secrets.get("blob_endpoint")?.trim_end_matches('/'); if endpoint.is_empty() || !url.starts_with(endpoint) { @@ -464,6 +520,53 @@ fn remote_blob_backend<'a>(secrets: &'a BTreeMap, url: &str) -> Some(backend) } +/// Apply ADR-0166 to a map of guest-supplied string tags. Shared by guest wide +/// events and guest metrics: both let an untrusted module choose tag names *and* +/// string values, so both are the same channel as the span APIs. +fn redact_guest_string_tags(tags: &mut BTreeMap, export_llm_content: bool) { + if export_llm_content { + return; + } + tags.retain(|key, _| llm_namespace_attr_allowed(key)); + for (key, value) in tags.iter_mut() { + if is_llm_namespace_key(key) + && let Some(clamped) = clamp_redacted_metadata_value(value) + { + *value = clamped; + } + } +} + +/// Apply ADR-0166 to the guest-supplied half of a wide event. Inside the +/// `gen_ai.*` namespace a non-opted-in tenant keeps only recognised, bounded +/// metadata; keys outside it are the module's own application telemetry and are +/// left alone, matching the guest-span boundary. +fn redact_guest_wide_event_fields( + tags: &mut BTreeMap, + attributes: &mut BTreeMap, + export_llm_content: bool, +) { + if export_llm_content { + return; + } + redact_guest_string_tags(tags, export_llm_content); + attributes.retain(|key, value| { + if !llm_namespace_attr_allowed(key) { + return false; + } + // A structured value under a recognised metadata key is dropped, not + // clamped: see `clamp_llm_metadata_json`. + !is_llm_namespace_key(key) || clamp_llm_metadata_json(value).is_some() + }); + for (key, value) in attributes.iter_mut() { + if is_llm_namespace_key(key) + && let Some(clamped) = clamp_llm_metadata_json(value) + { + *value = clamped; + } + } +} + impl ProductionWasmHost { /// Create with pre-loaded secrets and default HTTP timeout. /// @@ -499,37 +602,23 @@ impl ProductionWasmHost { /// lets operators provision private CA trust via the same secret store /// that WASM modules already use, with no filesystem or env var coupling. pub fn with_timeout(secrets: BTreeMap, timeout: std::time::Duration) -> Self { - let mut builder = reqwest::Client::builder() - .connect_timeout(std::time::Duration::from_secs(10)) - .timeout(timeout); - - for (key, pem) in &secrets { - if !key.starts_with("ca_cert:") { - continue; - } - match reqwest::Certificate::from_pem(pem.as_bytes()) { - Ok(cert) => { - tracing::info!(key, "loaded CA certificate from secret store"); - builder = builder.add_root_certificate(cert); - } - Err(e) => { - tracing::warn!(key, error = %e, "failed to parse CA certificate from secret"); - } - } - } + let client = build_production_http_client(&secrets, timeout, false); + let internal_client = build_production_http_client(&secrets, timeout, true); Self { - client: builder.build().unwrap_or_default(), + client, + internal_client, secrets, secret_resolver: None, internal_api_base_url: None, - internal_api_key: None, + internal_capability_issuer: None, spec_evaluator: None, progress_emitter: None, trace_id: None, binary_http_interceptor: None, text_http_interceptor: None, invocation_context: None, + export_llm_content: false, http_streams: Arc::new(crate::http_stream::HttpStreamRegistry::new()), } } @@ -564,6 +653,13 @@ impl ProductionWasmHost { self } + /// Set whether this invocation's tenant may export raw LLM content captured + /// from guest HTTP calls. Defaults to `false` (redact). See ADR-0166. + pub fn with_llm_content_export(mut self, export_content: bool) -> Self { + self.export_llm_content = export_content; + self + } + /// Attach the canonical local Temper API base URL for internal call detection. pub fn with_internal_api_base_url(mut self, api_url: Option) -> Self { self.internal_api_base_url = api_url @@ -572,9 +668,12 @@ impl ProductionWasmHost { self } - /// Attach the platform API token used for internal loopback calls. - pub fn with_internal_api_key(mut self, api_key: Option) -> Self { - self.internal_api_key = api_key.filter(|s| !s.is_empty()); + /// Attach the trusted issuer for request-bound internal capabilities. + pub fn with_internal_capability_issuer( + mut self, + issuer: InternalHttpCapabilityIssuerFn, + ) -> Self { + self.internal_capability_issuer = Some(issuer); self } @@ -591,74 +690,54 @@ impl ProductionWasmHost { } fn is_internal_temper_url(&self, url: &str) -> bool { - self.internal_api_base_url - .as_ref() - .is_some_and(|api_url| url.starts_with(api_url.trim_end_matches('/'))) - || self - .secrets - .get("temper_api_url") - .is_some_and(|api_url| url.starts_with(api_url.trim_end_matches('/'))) + // Only the server-owned configuration can classify a target as + // internal. A tenant secret or guest integration value must never + // redirect a request-bound capability to an attacker-controlled host. + let configured = self.internal_api_base_url.iter().map(String::as_str); + internal_http::is_internal_url(url, configured) } fn add_internal_temper_headers( &self, mut builder: reqwest::RequestBuilder, + method: &str, url: &str, headers: &[(String, String)], - ) -> reqwest::RequestBuilder { + ) -> Result { if !self.is_internal_temper_url(url) { - return builder; + return Ok(builder); } - let Some(ref inv_ctx) = self.invocation_context else { - return builder; - }; - - let has_tenant = headers - .iter() - .any(|(k, _)| k.eq_ignore_ascii_case("x-tenant-id")); - let has_principal = headers - .iter() - .any(|(k, _)| k.eq_ignore_ascii_case("x-temper-principal-kind")); - let has_authorization = headers - .iter() - .any(|(k, _)| k.eq_ignore_ascii_case("authorization")); - - if !has_tenant { - builder = builder.header("x-tenant-id", inv_ctx.tenant.as_str()); - } - - if !has_principal { - let agent_type = if inv_ctx.entity_type.eq_ignore_ascii_case("Session") { - "agent" - } else { - "system" - }; - let principal_id = inv_ctx - .agent_id - .as_deref() - .filter(|s| !s.is_empty()) - .unwrap_or(inv_ctx.entity_id.as_str()); - builder = builder - .header("x-temper-principal-kind", "agent") - .header("x-temper-principal-id", principal_id) - .header("x-temper-agent-type", agent_type); - if let Some(ref sid) = inv_ctx.session_id { - builder = builder.header("x-temper-ctx-sessionid", sid.as_str()); - } - } + let issuer = self.internal_capability_issuer.as_ref().ok_or_else(|| { + "internal Temper HTTP call has no authenticated capability issuer".to_string() + })?; + let canonical_method = method.to_ascii_uppercase(); + let capability = issuer(&canonical_method, url)?; + builder = builder + .header( + "authorization", + format!("Bearer {}", capability.bearer_token()), + ) + .header("x-tenant-id", capability.tenant()); - if !has_authorization - && let Some(key) = self - .internal_api_key - .as_ref() - .or_else(|| self.secrets.get("temper_api_key")) - .filter(|k| !k.is_empty()) - { - builder = builder.header("authorization", format!("Bearer {key}")); + if let Some(invocation_context) = &self.invocation_context { + builder = add_workflow_observability_headers(builder, headers, invocation_context); } + Ok(builder) + } - add_workflow_observability_headers(builder, headers, inv_ctx) + fn outbound_headers( + &self, + url: &str, + headers: &[(String, String)], + ) -> (bool, Vec<(String, String)>) { + let is_internal = self.is_internal_temper_url(url); + let headers = if is_internal { + internal_http::sanitize_internal_headers(headers) + } else { + headers.to_vec() + }; + (is_internal, headers) } fn build_guest_wide_event(&self, event_json: &str) -> Result { @@ -669,6 +748,13 @@ impl ProductionWasmHost { let mut attributes = payload.attributes; let measurements = payload.measurements; + // ADR-0166: a wide event is a guest-authored telemetry record with + // guest-chosen tag and attribute names, so it is the same untrusted + // channel as the span APIs and takes the same per-tenant decision. Applied + // before any host-derived field is merged in below, so only guest input is + // judged. + redact_guest_wide_event_fields(&mut tags, &mut attributes, self.export_llm_content); + let entity_type = self .invocation_context .as_ref() @@ -761,6 +847,10 @@ impl ProductionWasmHost { #[async_trait] impl WasmHost for ProductionWasmHost { + fn exports_llm_content(&self) -> bool { + self.export_llm_content + } + async fn http_call( &self, method: &str, @@ -769,31 +859,34 @@ impl WasmHost for ProductionWasmHost { body: &str, ) -> Result<(u16, String), String> { let started = Instant::now(); + // Strip Temper span hint headers (X-Temper-Span-*) before the request + // is built, and capture them for the local tracing span. See + // ADR-0037: WASM guests annotate outgoing calls with + // `X-Temper-Span-Name` / `X-Temper-Span-Attr-*` so the resulting + // span has a semantically meaningful name (e.g., `tool.llm_call`) + // and attributes (e.g., `gen_ai.request.model`). + let (filtered_headers, mut span_hints) = split_span_hint_headers(headers); + // ARN-243: drop LLM content from span hints unless this tenant opted in. + redact_llm_content_hints(&mut span_hints, self.export_llm_content); + let (is_internal, outbound_headers) = self.outbound_headers(url, &filtered_headers); if let Some(interceptor) = &self.text_http_interceptor && let Some(result) = interceptor( method.to_string(), url.to_string(), - headers.to_vec(), + outbound_headers.clone(), body.to_string(), ) .await { return result; } - // Strip Temper span hint headers (X-Temper-Span-*) before the request - // is built, and capture them for the local tracing span. See - // ADR-0037: WASM guests annotate outgoing calls with - // `X-Temper-Span-Name` / `X-Temper-Span-Attr-*` so the resulting - // span has a semantically meaningful name (e.g., `tool.llm_call`) - // and attributes (e.g., `gen_ai.request.model`). - let (filtered_headers, span_hints) = split_span_hint_headers(headers); let span = tracing::info_span!( "wasm.host.http_call", otel.name = %span_hint_otel_name(&span_hints, "wasm.host.http_call"), http.method = %method, http.url = %telemetry_url(url), request_bytes = body.len() as u64, - header_count = filtered_headers.len() as u64, + header_count = outbound_headers.len() as u64, status_code = tracing::field::Empty, response_bytes = tracing::field::Empty, duration_ms = tracing::field::Empty, @@ -841,26 +934,30 @@ impl WasmHost for ProductionWasmHost { apply_span_hints(&span, &span_hints); let _guard = span.enter(); + let client = if is_internal { + &self.internal_client + } else { + &self.client + }; let mut builder = match method.to_uppercase().as_str() { - "GET" => self.client.get(url), - "POST" => self.client.post(url), - "PUT" => self.client.put(url), - "DELETE" => self.client.delete(url), - "PATCH" => self.client.patch(url), + "GET" => client.get(url), + "POST" => client.post(url), + "PUT" => client.put(url), + "DELETE" => client.delete(url), + "PATCH" => client.patch(url), other => return Err(format!("unsupported HTTP method: {other}")), }; - for (k, v) in &filtered_headers { + for (k, v) in &outbound_headers { builder = builder.header(k.as_str(), v.as_str()); } - let is_internal = self.is_internal_temper_url(url); - builder = self.add_internal_temper_headers(builder, url, &filtered_headers); + builder = self.add_internal_temper_headers(builder, method, url, &outbound_headers)?; // determinism-ok: is_internal check uses non-deterministic URL comparison, // but this runs in WasmHost (not simulation), so wall-clock/network access is fine. // Auto-inject traceparent for cross-request trace correlation. - if !filtered_headers + if !outbound_headers .iter() .any(|(k, _)| k.eq_ignore_ascii_case("traceparent")) && let Some(traceparent) = @@ -1014,14 +1111,17 @@ impl WasmHost for ProductionWasmHost { ) -> Result<(u16, Vec), String> { let started = Instant::now(); // See http_call for the span-hint-header rationale (ADR-0037). - let (filtered_headers, span_hints) = split_span_hint_headers(headers); + let (filtered_headers, mut span_hints) = split_span_hint_headers(headers); + // ARN-243: drop LLM content from span hints unless this tenant opted in. + redact_llm_content_hints(&mut span_hints, self.export_llm_content); + let (is_internal, outbound_headers) = self.outbound_headers(url, &filtered_headers); let span = tracing::info_span!( "wasm.host.http_call_binary", otel.name = %span_hint_otel_name(&span_hints, "wasm.host.http_call_binary"), http.method = %method, http.url = %telemetry_url(url), request_bytes = body.len() as u64, - header_count = filtered_headers.len() as u64, + header_count = outbound_headers.len() as u64, status_code = tracing::field::Empty, response_bytes = tracing::field::Empty, duration_ms = tracing::field::Empty, @@ -1077,7 +1177,7 @@ impl WasmHost for ProductionWasmHost { && let Some(result) = interceptor( method.to_string(), url.to_string(), - filtered_headers.clone(), + outbound_headers.clone(), body.to_vec(), ) .await @@ -1138,25 +1238,27 @@ impl WasmHost for ProductionWasmHost { None }; + let client = if is_internal { + &self.internal_client + } else { + &self.client + }; let mut builder = match method.to_uppercase().as_str() { - "GET" => self.client.get(url), - "POST" => self.client.post(url), - "PUT" => self.client.put(url), - "DELETE" => self.client.delete(url), - "PATCH" => self.client.patch(url), + "GET" => client.get(url), + "POST" => client.post(url), + "PUT" => client.put(url), + "DELETE" => client.delete(url), + "PATCH" => client.patch(url), other => return Err(format!("unsupported HTTP method: {other}")), }; - for (k, v) in &filtered_headers { + for (k, v) in &outbound_headers { builder = builder.header(k.as_str(), v.as_str()); } - let is_internal = self.is_internal_temper_url(url); - if is_internal && let Some(ref inv_ctx) = self.invocation_context { - builder = add_workflow_observability_headers(builder, &filtered_headers, inv_ctx); - } + builder = self.add_internal_temper_headers(builder, method, url, &outbound_headers)?; - if !filtered_headers + if !outbound_headers .iter() .any(|(k, _)| k.eq_ignore_ascii_case("traceparent")) && let Some(traceparent) = @@ -1210,14 +1312,17 @@ impl WasmHost for ProductionWasmHost { body: &str, ) -> Result, String> { let started = Instant::now(); - let (filtered_headers, span_hints) = split_span_hint_headers(headers); + let (filtered_headers, mut span_hints) = split_span_hint_headers(headers); + // ARN-243: drop LLM content from span hints unless this tenant opted in. + redact_llm_content_hints(&mut span_hints, self.export_llm_content); + let (is_internal, outbound_headers) = self.outbound_headers(url, &filtered_headers); let span = tracing::info_span!( "wasm.host.connect_call", otel.name = %span_hint_otel_name(&span_hints, "wasm.host.connect_call"), http.method = "POST", http.url = %telemetry_url(url), request_bytes = body.len() as u64, - header_count = filtered_headers.len() as u64, + header_count = outbound_headers.len() as u64, status_code = tracing::field::Empty, response_frames = tracing::field::Empty, response_bytes = tracing::field::Empty, @@ -1240,7 +1345,12 @@ impl WasmHost for ProductionWasmHost { apply_span_hints(&span, &span_hints); let _guard = span.enter(); - let mut builder = self.client.post(url); + let client = if is_internal { + &self.internal_client + } else { + &self.client + }; + let mut builder = client.post(url); // Set Connect protocol headers. // Use application/connect+json for envd-compatible services (E2B, etc.) @@ -1248,10 +1358,11 @@ impl WasmHost for ProductionWasmHost { .header("content-type", "application/connect+json") .header("connect-protocol-version", "1"); - for (k, v) in &filtered_headers { + for (k, v) in &outbound_headers { builder = builder.header(k.as_str(), v.as_str()); } - if !filtered_headers + builder = self.add_internal_temper_headers(builder, "POST", url, &outbound_headers)?; + if !outbound_headers .iter() .any(|(k, _)| k.eq_ignore_ascii_case("traceparent")) && let Some(traceparent) = @@ -1324,16 +1435,19 @@ impl WasmHost for ProductionWasmHost { ); record_invocation_context_on_span(&span, self.invocation_context.as_ref(), "secret_lookup"); let _guard = span.enter(); - let result = self - .secret_resolver - .as_ref() - .map(|resolver| resolver(key)) - .unwrap_or_else(|| { - self.secrets - .get(key) - .cloned() - .ok_or_else(|| format!("secret not found: {key}")) - }); + let result = if key.eq_ignore_ascii_case("temper_api_key") { + Err("secret 'temper_api_key' is reserved and unavailable to WASM guests".to_string()) + } else { + self.secret_resolver + .as_ref() + .map(|resolver| resolver(key)) + .unwrap_or_else(|| { + self.secrets + .get(key) + .cloned() + .ok_or_else(|| format!("secret not found: {key}")) + }) + }; tracing::Span::current().record("success", result.is_ok()); tracing::Span::current().record("duration_ms", started.elapsed().as_millis() as u64); if let Err(error) = &result { @@ -1530,8 +1644,12 @@ impl WasmHost for ProductionWasmHost { } fn emit_metric(&self, metric_json: &str) -> Result<(), String> { - let payload: GuestMetricInput = serde_json::from_str(metric_json) + let mut payload: GuestMetricInput = serde_json::from_str(metric_json) .map_err(|e| format!("invalid guest metric payload: {e}"))?; + // ADR-0166: guest metric tags are guest-named *and* guest-valued strings, + // and they reach two sinks below — the span event and the OTel meter. + // Redact before either reads them, so neither can observe the raw tags. + redact_guest_string_tags(&mut payload.tags, self.export_llm_content); record_guest_metric_span_event(&payload, self.invocation_context.as_ref()); let meter = opentelemetry::global::meter("temper"); let mut attrs: Vec = payload @@ -1599,14 +1717,22 @@ impl WasmHost for ProductionWasmHost { let bridge_resp = exchange.bridge_response_body; let head_tx = exchange.bridge_head_sender; let streams = self.http_streams.clone(); - let client = self.client.clone(); - let (filtered_headers, span_hints) = split_span_hint_headers(&request.headers); + let (filtered_headers, mut span_hints) = split_span_hint_headers(&request.headers); + // ARN-243: drop LLM content from span hints unless this tenant opted in. + redact_llm_content_hints(&mut span_hints, self.export_llm_content); + let (is_internal, outbound_headers) = + self.outbound_headers(&request.url, &filtered_headers); + let client = if is_internal { + self.internal_client.clone() + } else { + self.client.clone() + }; let span = tracing::info_span!( "wasm.host.http_stream", otel.name = %span_hint_otel_name(&span_hints, "wasm.host.http_stream"), http.method = %request.method, http.url = %telemetry_url(&request.url), - header_count = filtered_headers.len() as u64, + header_count = outbound_headers.len() as u64, status_code = tracing::field::Empty, response_bytes = tracing::field::Empty, duration_ms = tracing::field::Empty, @@ -1664,11 +1790,16 @@ impl WasmHost for ProductionWasmHost { other => return Err(format!("unsupported HTTP method: {other}")), }; let mut builder = builder; - for (k, v) in &filtered_headers { + for (k, v) in &outbound_headers { builder = builder.header(k.as_str(), v.as_str()); } - builder = self.add_internal_temper_headers(builder, &request.url, &filtered_headers); - if !filtered_headers + builder = self.add_internal_temper_headers( + builder, + &request.method, + &request.url, + &outbound_headers, + )?; + if !outbound_headers .iter() .any(|(k, _)| k.eq_ignore_ascii_case("traceparent")) && let Some(traceparent) = current_traceparent_header(&span, self.trace_id.as_deref()) diff --git a/crates/temper-wasm/src/host_trait/host_trait_test.rs b/crates/temper-wasm/src/host_trait/host_trait_test.rs index 7219f010f..f50fc5912 100644 --- a/crates/temper-wasm/src/host_trait/host_trait_test.rs +++ b/crates/temper-wasm/src/host_trait/host_trait_test.rs @@ -7,6 +7,31 @@ use std::sync::{Arc, Mutex}; use tracing_opentelemetry::OpenTelemetrySpanExt; use tracing_subscriber::prelude::*; +#[test] +fn llm_content_export_defaults_to_redact_and_is_opt_in() { + use std::collections::BTreeMap; + // Fail-safe: a host built without an explicit opt-in must redact LLM + // content, so any construction site that forgets `.with_llm_content_export` + // still defaults to safe. See ADR-0166 (ARN-243). + let default_host = ProductionWasmHost::new(BTreeMap::new()); + assert!( + !default_host.export_llm_content, + "host must default to redacting LLM content" + ); + + let opted_in = ProductionWasmHost::new(BTreeMap::new()).with_llm_content_export(true); + assert!( + opted_in.export_llm_content, + "with_llm_content_export(true) must opt in" + ); + + let opted_out = ProductionWasmHost::new(BTreeMap::new()).with_llm_content_export(false); + assert!( + !opted_out.export_llm_content, + "with_llm_content_export(false) must redact" + ); +} + #[test] fn guest_metric_count_kind_is_counter() { assert!(guest_metric_is_counter_kind(Some("count"))); @@ -190,7 +215,7 @@ fn current_traceparent_header_prefers_active_span_context() { } #[test] -fn internal_http_call_injects_bearer_even_with_explicit_principal_headers() { +fn internal_http_call_replaces_guest_authority_with_fresh_capability() { let listener = TcpListener::bind("127.0.0.1:0").expect("bind test listener"); let addr = listener.local_addr().expect("listener addr"); let captured = Arc::new(Mutex::new(String::new())); @@ -213,32 +238,38 @@ fn internal_http_call_injects_bearer_even_with_explicit_principal_headers() { .expect("write response"); }); - let mut secrets = BTreeMap::new(); - secrets.insert("temper_api_url".to_string(), format!("http://{addr}")); - secrets.insert("temper_api_key".to_string(), "secret123".to_string()); - - let host = ProductionWasmHost::new(secrets).with_invocation_context(WasmInvocationContext { - tenant: "default".to_string(), - entity_type: "Workspace".to_string(), - entity_id: "ws-1".to_string(), - trigger_action: "CreateFile".to_string(), - wasm_module: Some("workspace_fs".to_string()), - trigger_params: Value::Null, - entity_state: Value::Null, - agent_id: Some("operator".to_string()), - session_id: None, - integration_config: BTreeMap::new(), - trace_id: String::new(), - workflow_root_entity_type: Some("CurationQuery".to_string()), - workflow_root_entity_id: Some("cq-1".to_string()), - workflow_run_id: Some("CurationQuery:cq-1".to_string()), - http_request: None, - }); + let host = ProductionWasmHost::new(BTreeMap::new()) + .with_internal_api_base_url(Some(format!("http://{addr}"))) + .with_internal_capability_issuer(Arc::new(|method, url| { + assert_eq!(method, "GET"); + assert!(url.ends_with("/tdata/Directories")); + InternalHttpCapability::new("request-capability".to_string(), "tenant-a".to_string()) + })) + .with_invocation_context(WasmInvocationContext { + tenant: "tenant-a".to_string(), + entity_type: "Workspace".to_string(), + entity_id: "ws-1".to_string(), + trigger_action: "CreateFile".to_string(), + wasm_module: Some("workspace_fs".to_string()), + trigger_params: Value::Null, + entity_state: Value::Null, + agent_id: Some("operator".to_string()), + session_id: None, + integration_config: BTreeMap::new(), + trace_id: String::new(), + workflow_root_entity_type: Some("CurationQuery".to_string()), + workflow_root_entity_id: Some("cq-1".to_string()), + workflow_run_id: Some("CurationQuery:cq-1".to_string()), + http_request: None, + }); let headers = vec![ - ("X-Tenant-Id".to_string(), "default".to_string()), - ("x-temper-principal-kind".to_string(), "agent".to_string()), - ("x-temper-principal-id".to_string(), "system".to_string()), + ("Authorization".to_string(), "Bearer guest-root".to_string()), + ("X-Tenant-Id".to_string(), "victim".to_string()), + ("x-temper-principal-kind".to_string(), "admin".to_string()), + ("x-temper-principal-id".to_string(), "attacker".to_string()), + ("x-temper-attr-limit".to_string(), "999".to_string()), + ("x-regular".to_string(), "preserved".to_string()), ]; let (status, _) = tokio_test::block_on(host.http_call( @@ -254,13 +285,18 @@ fn internal_http_call_injects_bearer_even_with_explicit_principal_headers() { let request = captured.lock().expect("capture lock").to_lowercase(); assert!( - request.contains("authorization: bearer secret123"), + request.contains("authorization: bearer request-capability"), "expected bearer token in request, got: {request}" ); assert!( - request.contains("x-temper-principal-kind: agent"), - "expected explicit principal header to be preserved, got: {request}" + request.contains("x-tenant-id: tenant-a"), + "expected capability tenant in request, got: {request}" ); + assert!(!request.contains("guest-root"), "{request}"); + assert!(!request.contains("victim"), "{request}"); + assert!(!request.contains("x-temper-principal"), "{request}"); + assert!(!request.contains("x-temper-attr"), "{request}"); + assert!(request.contains("x-regular: preserved"), "{request}"); assert!( request.contains("x-temper-workflow-root-entity-type: curationquery"), "expected workflow root type header, got: {request}" @@ -276,133 +312,136 @@ fn internal_http_call_injects_bearer_even_with_explicit_principal_headers() { } #[test] -fn internal_http_call_injects_bearer_from_internal_api_key_context() { - let listener = TcpListener::bind("127.0.0.1:0").expect("bind test listener"); - let addr = listener.local_addr().expect("listener addr"); - let captured = Arc::new(Mutex::new(String::new())); - let captured_clone = Arc::clone(&captured); - - let server = std::thread::spawn(move || { - let (mut stream, _) = listener.accept().expect("accept request"); - let mut buf = [0u8; 8192]; - let len = stream.read(&mut buf).expect("read request"); - *captured_clone.lock().expect("capture lock") = - String::from_utf8_lossy(&buf[..len]).into_owned(); - - let body = "{}"; - write!( - stream, - "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", - body.len(), - body - ) - .expect("write response"); - }); - +fn internal_http_call_without_issuer_fails_before_network() { let mut secrets = BTreeMap::new(); - secrets.insert("temper_api_url".to_string(), format!("http://{addr}")); - + secrets.insert( + "temper_api_url".to_string(), + "http://127.0.0.1:9".to_string(), + ); + secrets.insert("temper_api_key".to_string(), "ambient-root".to_string()); let host = ProductionWasmHost::new(secrets) - .with_internal_api_key(Some("ambient-secret".to_string())) - .with_invocation_context(WasmInvocationContext { - tenant: "default".to_string(), - entity_type: "Workspace".to_string(), - entity_id: "ws-1".to_string(), - trigger_action: "CreateFile".to_string(), - wasm_module: Some("workspace_fs".to_string()), - trigger_params: Value::Null, - entity_state: Value::Null, - agent_id: Some("operator".to_string()), - session_id: None, - integration_config: BTreeMap::new(), - trace_id: String::new(), - workflow_root_entity_type: None, - workflow_root_entity_id: None, - workflow_run_id: None, - http_request: None, - }); - - let headers = vec![("X-Tenant-Id".to_string(), "default".to_string())]; + .with_internal_api_base_url(Some("http://127.0.0.1:9".to_string())); - let (status, _) = tokio_test::block_on(host.http_call( + let error = tokio_test::block_on(host.http_call( "GET", - &format!("http://{addr}/tdata/Directories"), - &headers, + "http://127.0.0.1:9/tdata/Directories", + &[], "", )) - .expect("internal call should succeed"); - - assert_eq!(status, 200); - server.join().expect("server thread"); - - let request = captured.lock().expect("capture lock").to_lowercase(); + .expect_err("internal calls without an issuer must fail closed"); assert!( - request.contains("authorization: bearer ambient-secret"), - "expected ambient bearer token in request, got: {request}" + error.contains("no authenticated capability issuer"), + "{error}" + ); +} + +#[test] +fn tenant_secret_cannot_reclassify_an_external_origin_as_internal() { + let mut secrets = BTreeMap::new(); + secrets.insert( + "temper_api_url".to_string(), + "http://attacker.example".to_string(), ); + let issuer_calls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let calls_for_issuer = Arc::clone(&issuer_calls); + let host = + ProductionWasmHost::new(secrets).with_internal_capability_issuer(Arc::new(move |_, _| { + calls_for_issuer.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + InternalHttpCapability::new("must-not-issue".to_string(), "tenant-a".to_string()) + })); + + assert!(!host.is_internal_temper_url("http://attacker.example/tdata/Orders")); + assert_eq!(issuer_calls.load(std::sync::atomic::Ordering::SeqCst), 0); } #[test] -fn internal_http_call_uses_configured_internal_api_base_url_without_secret() { +fn internal_binary_http_call_sanitizes_and_injects_capability() { let listener = TcpListener::bind("127.0.0.1:0").expect("bind test listener"); let addr = listener.local_addr().expect("listener addr"); let captured = Arc::new(Mutex::new(String::new())); - let captured_clone = Arc::clone(&captured); - + let captured_for_server = Arc::clone(&captured); let server = std::thread::spawn(move || { let (mut stream, _) = listener.accept().expect("accept request"); - let mut buf = [0u8; 8192]; - let len = stream.read(&mut buf).expect("read request"); - *captured_clone.lock().expect("capture lock") = - String::from_utf8_lossy(&buf[..len]).into_owned(); - - let body = "{}"; - write!( - stream, - "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", - body.len(), - body - ) - .expect("write response"); + let mut request = Vec::new(); + let mut buffer = [0_u8; 8_192]; + let read = stream.read(&mut buffer).expect("read request"); + request.extend_from_slice(&buffer[..read]); + *captured_for_server.lock().expect("capture lock") = + String::from_utf8_lossy(&request).into_owned(); + stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok") + .expect("write response"); }); let host = ProductionWasmHost::new(BTreeMap::new()) .with_internal_api_base_url(Some(format!("http://{addr}"))) - .with_internal_api_key(Some("ambient-secret".to_string())) - .with_invocation_context(WasmInvocationContext { - tenant: "default".to_string(), - entity_type: "Session".to_string(), - entity_id: "ss-1".to_string(), - trigger_action: "WorkspaceReady".to_string(), - wasm_module: Some("monty_repl".to_string()), - trigger_params: Value::Null, - entity_state: Value::Null, - agent_id: Some("ss-1".to_string()), - session_id: Some("ss-1".to_string()), - integration_config: BTreeMap::new(), - trace_id: String::new(), - workflow_root_entity_type: None, - workflow_root_entity_id: None, - workflow_run_id: None, - http_request: None, - }); - - let (status, _) = tokio_test::block_on(host.http_call( - "GET", - &format!("http://{addr}/tdata/Files('file-1')/$value"), - &[("X-Tenant-Id".to_string(), "default".to_string())], - "", + .with_internal_capability_issuer(Arc::new(|method, _| { + assert_eq!(method, "PUT"); + InternalHttpCapability::new("binary-capability".to_string(), "tenant-b".to_string()) + })); + let (status, body) = tokio_test::block_on(host.http_call_binary( + "PUT", + &format!("http://{addr}/api/blob?part=1"), + &[ + ("authorization".to_string(), "Bearer guest".to_string()), + ("x-tenant-id".to_string(), "victim".to_string()), + ("x-temper-principal-kind".to_string(), "admin".to_string()), + ], + b"bytes", )) - .expect("internal call should succeed"); - + .expect("binary internal request should succeed"); assert_eq!(status, 200); - server.join().expect("server thread"); + assert_eq!(body, b"ok"); + server.join().expect("server should finish"); let request = captured.lock().expect("capture lock").to_lowercase(); assert!( - request.contains("authorization: bearer ambient-secret"), - "expected ambient bearer token in request, got: {request}" + request.contains("authorization: bearer binary-capability"), + "{request}" ); + assert!(request.contains("x-tenant-id: tenant-b"), "{request}"); + assert!(!request.contains("bearer guest"), "{request}"); + assert!(!request.contains("victim"), "{request}"); + assert!(!request.contains("x-temper-principal"), "{request}"); +} + +#[test] +fn internal_capability_requests_do_not_follow_redirects() { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind test listener"); + let addr = listener.local_addr().expect("listener addr"); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().expect("accept request"); + let mut buffer = [0_u8; 8_192]; + let _ = stream.read(&mut buffer).expect("read request"); + stream + .write_all( + b"HTTP/1.1 302 Found\r\nLocation: http://127.0.0.1:9/leak\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + ) + .expect("write redirect"); + }); + + let host = ProductionWasmHost::new(BTreeMap::new()) + .with_internal_api_base_url(Some(format!("http://{addr}"))) + .with_internal_capability_issuer(Arc::new(|_, _| { + InternalHttpCapability::new("redirect-capability".to_string(), "tenant-a".to_string()) + })); + let (status, _) = + tokio_test::block_on(host.http_call("GET", &format!("http://{addr}/redirect"), &[], "")) + .expect("redirect must be returned without following it"); + assert_eq!(status, 302); + server.join().expect("server should finish"); +} + +#[test] +fn production_host_never_exposes_ambient_root_secret() { + let mut secrets = BTreeMap::new(); + secrets.insert("temper_api_key".to_string(), "ambient-root".to_string()); + let host = ProductionWasmHost::new(secrets); + + let error = host + .get_secret("temper_api_key") + .expect_err("reserved root secret must not be available"); + assert!(error.contains("reserved"), "{error}"); } #[test] @@ -512,3 +551,137 @@ mod host_boundary_observability; mod log_correlation; #[path = "tests/span_hint_tests.rs"] mod span_hint_tests; + +/// ARN-243 / ADR-0166. `host_emit_wide_event` is a second guest-authored +/// telemetry record with guest-chosen field names — the same untrusted channel as +/// the span APIs, and it reaches the backend directly. +#[test] +fn guest_wide_event_fields_drop_llm_content_for_non_opted_in_tenant() { + use super::redact_guest_wide_event_fields; + use serde_json::{Value, json}; + use std::collections::BTreeMap; + + let mut tags = BTreeMap::new(); + tags.insert("gen_ai.prompt".to_string(), "SECRET PROMPT".to_string()); + tags.insert( + "gen_ai.request.model".to_string(), + "claude-opus-4-8".to_string(), + ); + tags.insert("app.route".to_string(), "checkout".to_string()); + + let mut attributes = BTreeMap::new(); + attributes.insert("gen_ai.completion".to_string(), json!("SECRET COMPLETION")); + attributes.insert( + "gen_ai.response.text".to_string(), + json!("SECRET COMPLETION"), + ); + attributes.insert("gen_ai.usage.input_tokens".to_string(), json!(42)); + attributes.insert("order.id".to_string(), json!("A-17")); + + redact_guest_wide_event_fields(&mut tags, &mut attributes, false); + + assert_eq!( + tags.get("gen_ai.prompt"), + None, + "prompt tag must not export" + ); + assert_eq!( + tags.get("gen_ai.request.model"), + Some(&"claude-opus-4-8".to_string()) + ); + assert_eq!(tags.get("app.route"), Some(&"checkout".to_string())); + assert_eq!(attributes.get("gen_ai.completion"), None); + assert_eq!( + attributes.get("gen_ai.response.text"), + None, + "an unrecognized gen_ai.* key must not export just because it is unlisted" + ); + assert_eq!( + attributes.get("gen_ai.usage.input_tokens"), + Some(&json!(42)) + ); + assert_eq!(attributes.get("order.id"), Some(&json!("A-17"))); + + // Values inside recognized keys are bounded, so the prompt cannot ride along. + let mut smuggle = BTreeMap::new(); + smuggle.insert("gen_ai.request.model".to_string(), json!("M".repeat(4096))); + let mut no_tags = BTreeMap::new(); + redact_guest_wide_event_fields(&mut no_tags, &mut smuggle, false); + let Some(Value::String(model)) = smuggle.get("gen_ai.request.model") else { + panic!("metadata key should survive, bounded"); + }; + assert!(model.len() <= 256, "got {} bytes", model.len()); +} + +/// An opted-in tenant is unaffected by the wide-event filter. +#[test] +fn guest_wide_event_fields_are_untouched_when_opted_in() { + use super::redact_guest_wide_event_fields; + use serde_json::json; + use std::collections::BTreeMap; + + let mut tags = BTreeMap::new(); + tags.insert("gen_ai.prompt".to_string(), "PROMPT".to_string()); + let mut attributes = BTreeMap::new(); + attributes.insert("gen_ai.completion".to_string(), json!("COMPLETION")); + + redact_guest_wide_event_fields(&mut tags, &mut attributes, true); + + assert_eq!(tags.get("gen_ai.prompt"), Some(&"PROMPT".to_string())); + assert_eq!( + attributes.get("gen_ai.completion"), + Some(&json!("COMPLETION")) + ); +} + +/// ARN-243 / ADR-0166. `host_emit_metric` is a fifth guest-authored telemetry +/// channel: the guest chooses tag names *and* string values, and they reach both +/// the OTel meter and a span event. Found by adversarial review after the first +/// four channels were closed. +#[test] +fn guest_metric_tags_drop_llm_content_for_non_opted_in_tenant() { + use super::redact_guest_string_tags; + use std::collections::BTreeMap; + + let mut tags = BTreeMap::new(); + tags.insert( + "gen_ai.completion".to_string(), + "SECRET COMPLETION".to_string(), + ); + tags.insert( + "gen_ai.response.text".to_string(), + "SECRET COMPLETION".to_string(), + ); + tags.insert("GEN_AI.prompt".to_string(), "SECRET PROMPT".to_string()); + tags.insert("gen_ai.request.model".to_string(), "M".repeat(4096)); + tags.insert("app.route".to_string(), "checkout".to_string()); + + redact_guest_string_tags(&mut tags, false); + + assert_eq!(tags.get("gen_ai.completion"), None); + assert_eq!( + tags.get("gen_ai.response.text"), + None, + "an unrecognized gen_ai.* key must not export just because it is unlisted" + ); + assert_eq!( + tags.get("GEN_AI.prompt"), + None, + "the namespace test must normalize the guest-supplied key" + ); + assert!( + tags.get("gen_ai.request.model") + .is_some_and(|v| v.len() <= 256), + "recognized metadata survives, clamped" + ); + assert_eq!(tags.get("app.route"), Some(&"checkout".to_string())); + + // Opted-in tenants are untouched. + let mut exported = BTreeMap::new(); + exported.insert("gen_ai.completion".to_string(), "COMPLETION".to_string()); + redact_guest_string_tags(&mut exported, true); + assert_eq!( + exported.get("gen_ai.completion"), + Some(&"COMPLETION".to_string()) + ); +} diff --git a/crates/temper-wasm/src/host_trait/internal_http.rs b/crates/temper-wasm/src/host_trait/internal_http.rs new file mode 100644 index 000000000..e557bc356 --- /dev/null +++ b/crates/temper-wasm/src/host_trait/internal_http.rs @@ -0,0 +1,163 @@ +//! Internal Temper HTTP request authentication and header isolation. + +use std::sync::Arc; + +/// One server-issued bearer capability for an exact internal HTTP request. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct InternalHttpCapability { + bearer_token: String, + tenant: String, +} + +impl InternalHttpCapability { + /// Construct a capability returned by a trusted server-side issuer. + pub fn new(bearer_token: String, tenant: String) -> Result { + if bearer_token.is_empty() { + return Err("internal HTTP capability token must not be empty".to_string()); + } + if tenant.is_empty() { + return Err("internal HTTP capability tenant must not be empty".to_string()); + } + Ok(Self { + bearer_token, + tenant, + }) + } + + /// Opaque bearer value to send to the internal authentication edge. + pub fn bearer_token(&self) -> &str { + &self.bearer_token + } + + /// Tenant bound into the capability. + pub fn tenant(&self) -> &str { + &self.tenant + } +} + +/// Callback that issues a fresh capability for one HTTP method and URL. +pub type InternalHttpCapabilityIssuerFn = + Arc Result + Send + Sync>; + +/// Return whether `url` is inside the exact configured internal API origin and +/// path boundary. +pub(super) fn is_internal_url<'a>(url: &str, mut base_urls: impl Iterator) -> bool { + let Ok(request) = reqwest::Url::parse(url) else { + return false; + }; + if !request.username().is_empty() || request.password().is_some() { + return false; + } + + base_urls.any(|base| { + let Ok(base) = reqwest::Url::parse(base) else { + return false; + }; + if request.scheme() != base.scheme() + || request.host_str() != base.host_str() + || request.port_or_known_default() != base.port_or_known_default() + { + return false; + } + + let base_path = base.path().trim_end_matches('/'); + base_path.is_empty() + || request.path() == base_path + || request + .path() + .strip_prefix(base_path) + .is_some_and(|suffix| suffix.starts_with('/')) + }) +} + +/// Strip every guest-controlled authority input before internal re-entry. +/// +/// Only ordinary transport/application headers and the explicitly +/// correlation-only Temper namespaces survive. The server then installs a +/// fresh bearer capability and its bound tenant. +pub(super) fn sanitize_internal_headers(headers: &[(String, String)]) -> Vec<(String, String)> { + headers + .iter() + .filter(|(name, _)| internal_header_allowed(name)) + .cloned() + .collect() +} + +fn internal_header_allowed(name: &str) -> bool { + let name = name.to_ascii_lowercase(); + if name.starts_with("x-temper-") { + return name.starts_with("x-temper-workflow-") || name.starts_with("x-temper-observe-"); + } + !matches!( + name.as_str(), + "authorization" + | "proxy-authorization" + | "cookie" + | "host" + | "forwarded" + | "x-forwarded-for" + | "x-forwarded-host" + | "x-forwarded-proto" + | "x-api-key" + | "x-tenant-id" + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn internal_url_matching_uses_origin_and_path_boundaries() { + let bases = ["https://temper.example/api"]; + for url in [ + "https://temper.example/api", + "https://temper.example/api/tdata?x=1", + ] { + assert!(is_internal_url(url, bases.iter().copied()), "{url}"); + } + for url in [ + "http://temper.example/api", + "https://temper.example.evil/api", + "https://temper.example:444/api", + "https://user@temper.example/api", + "https://temper.example/apix", + ] { + assert!(!is_internal_url(url, bases.iter().copied()), "{url}"); + } + } + + #[test] + fn sanitization_preserves_only_non_authority_and_correlation_headers() { + let headers = vec![ + ("Authorization".to_string(), "Bearer guest".to_string()), + ("X-Tenant-Id".to_string(), "victim".to_string()), + ("Host".to_string(), "attacker".to_string()), + ("X-Forwarded-For".to_string(), "127.0.0.1".to_string()), + ("X-Temper-Principal-Kind".to_string(), "admin".to_string()), + ("X-Temper-Attr-Limit".to_string(), "999".to_string()), + ("X-Temper-Workflow-Run-Id".to_string(), "run-1".to_string()), + ( + "X-Temper-Observe-Session-Id".to_string(), + "session-1".to_string(), + ), + ("Content-Type".to_string(), "application/json".to_string()), + ("Idempotency-Key".to_string(), "request-1".to_string()), + ("traceparent".to_string(), "00-trace-span-01".to_string()), + ]; + + assert_eq!( + sanitize_internal_headers(&headers), + vec![ + ("X-Temper-Workflow-Run-Id".to_string(), "run-1".to_string()), + ( + "X-Temper-Observe-Session-Id".to_string(), + "session-1".to_string() + ), + ("Content-Type".to_string(), "application/json".to_string()), + ("Idempotency-Key".to_string(), "request-1".to_string()), + ("traceparent".to_string(), "00-trace-span-01".to_string()), + ] + ); + } +} diff --git a/crates/temper-wasm/src/host_trait/span_hints.rs b/crates/temper-wasm/src/host_trait/span_hints.rs index e5cb7e653..f5e60fa9d 100644 --- a/crates/temper-wasm/src/host_trait/span_hints.rs +++ b/crates/temper-wasm/src/host_trait/span_hints.rs @@ -100,6 +100,187 @@ pub(crate) fn span_hint_otel_name<'a>( .unwrap_or(Cow::Borrowed(default_name)) } +/// Longest a guest-supplied `gen_ai.*` metadata value may be when the tenant has +/// not opted into content export. Every legitimate metadata value is short — a +/// model id, a provider name, a token count, a finish reason — so this bound +/// costs nothing real while stopping a module that hides a prompt inside +/// `gen_ai.request.model`. Key names alone cannot make an untrusted value +/// metadata; the bound is what makes the allowlist mean something. +pub const MAX_REDACTED_LLM_METADATA_VALUE_BYTES: usize = 256; + +/// The rule applied wherever untrusted guest telemetry can name a `gen_ai.*` +/// attribute: inside that namespace, a non-opted-in tenant keeps recognised +/// metadata keys and nothing else. Keys outside the namespace are not judged +/// here — general-purpose guest telemetry is the caller's decision, since the +/// `gen_ai.*` namespace is the part with agreed semantics that dashboards and +/// LLM Observability read. +pub(crate) fn llm_namespace_attr_allowed(key: &str) -> bool { + let key = normalize_llm_attr_key(key); + if !key.starts_with("gen_ai.") { + return true; + } + is_llm_metadata_attr(&key) +} + +/// Whether a guest-supplied key lands in the `gen_ai.*` namespace, normalized. +/// +/// Every namespace test goes through this rather than `starts_with` on the raw +/// key. Normalizing in one place and matching raw in another is how the clamp and +/// the allowlist end up disagreeing: `GEN_AI.request.model` passes a normalized +/// allowlist as recognised metadata, then misses a raw-key clamp and carries a +/// whole prompt. +pub(crate) fn is_llm_namespace_key(key: &str) -> bool { + normalize_llm_attr_key(key).starts_with("gen_ai.") +} + +/// Normalize a guest-supplied attribute key before the namespace test. The +/// span-hint parser already lowercases and trims, but the guest-span and +/// wide-event payloads hand keys over verbatim, so ` GEN_AI.prompt ` would +/// otherwise miss the `gen_ai.` prefix test and pass through as ordinary +/// application telemetry. +pub(crate) fn normalize_llm_attr_key(key: &str) -> String { + key.trim().to_ascii_lowercase() +} + +/// Bound a `gen_ai.*` metadata value of any JSON shape. +/// +/// Strings are clamped. Numbers and booleans pass — they are bounded by their own +/// type and are what real metadata looks like (token counts, temperature). +/// Arrays and objects are **dropped**: no metadata key in the allowlist is +/// structured, and their serialized form is unbounded, so `gen_ai.request.model = +/// [""]` would otherwise ride straight through a clamp that only +/// matched strings. Adversarial review found exactly that hole. +pub(crate) fn clamp_llm_metadata_json(value: &serde_json::Value) -> Option { + match value { + serde_json::Value::String(text) => Some( + clamp_redacted_metadata_value(text) + .map_or_else(|| value.clone(), serde_json::Value::String), + ), + serde_json::Value::Number(_) | serde_json::Value::Bool(_) | serde_json::Value::Null => { + Some(value.clone()) + } + serde_json::Value::Array(_) | serde_json::Value::Object(_) => None, + } +} + +/// Clamp a guest-supplied metadata value to [`MAX_REDACTED_LLM_METADATA_VALUE_BYTES`], +/// on a char boundary. Returns `None` when the value is unchanged. +pub fn clamp_redacted_metadata_value(value: &str) -> Option { + if value.len() <= MAX_REDACTED_LLM_METADATA_VALUE_BYTES { + return None; + } + let mut end = MAX_REDACTED_LLM_METADATA_VALUE_BYTES; + while end > 0 && !value.is_char_boundary(end) { + end -= 1; + } + Some(value[..end].to_string()) +} + +/// Whether a key names LLM *content* (prompt, completion, system instructions, +/// tool arguments/results) under the `gen_ai.*` semantic conventions. +/// +/// This is **not** the gate — enumerating content keys cannot stop a guest that +/// picks another name, which is why [`llm_namespace_attr_allowed`] allowlists +/// metadata instead. It is kept as the explicit statement of which keys are +/// content, used by the contract test that pins them against the Datadog-visible +/// field set and by the test that proves the content and metadata sets are +/// disjoint — which is why it is `#[cfg(test)]`: it documents and verifies the +/// classification, it does not enforce it. See ADR-0166. +#[cfg(test)] +pub(crate) fn is_sensitive_llm_content_attr(attr_key: &str) -> bool { + matches!( + attr_key, + "gen_ai.input.messages" + | "gen_ai.prompt" + | "gen_ai.system_instructions" + | "gen_ai.output.messages" + | "gen_ai.completion" + | "gen_ai.tool.call.arguments" + | "gen_ai.tool.call.result" + ) +} + +/// `gen_ai.*` attributes that describe a call rather than carry its content: +/// model, provider, token counts, ids, finish reasons. Everything outside this +/// set is treated as potential content for a tenant that has not opted in. +fn is_llm_metadata_attr(attr_key: &str) -> bool { + matches!( + attr_key, + "gen_ai.operation.name" + | "gen_ai.system" + | "gen_ai.provider.name" + | "gen_ai.conversation.id" + | "gen_ai.request.model" + | "gen_ai.request.max_tokens" + | "gen_ai.request.temperature" + | "gen_ai.response.model" + | "gen_ai.response.finish_reasons" + | "gen_ai.tool.name" + | "gen_ai.tool.call.id" + | "gen_ai.usage.input_tokens" + | "gen_ai.usage.output_tokens" + | "gen_ai.usage.cache_creation_input_tokens" + | "gen_ai.usage.cache_read_input_tokens" + ) +} + +/// Redact LLM content from host-captured span hints unless the tenant has opted +/// into content export. No-op when `export_content` is true. For a tenant that +/// has not opted in this: +/// +/// - drops every `gen_ai.*` attribute that is not recognised metadata +/// ([`llm_namespace_attr_allowed`]), keeping attributes outside that namespace; +/// - clamps surviving `gen_ai.*` values and the guest-supplied span name to +/// [`MAX_REDACTED_LLM_METADATA_VALUE_BYTES`]; +/// - clears **all** response captures, including ones named as metadata — a +/// capture's value comes from a guest-supplied JSON pointer into the response +/// body, so its name cannot establish what it holds. +/// +/// See ADR-0166. +pub(crate) fn redact_llm_content_hints(hints: &mut SpanHints, export_content: bool) { + if export_content { + return; + } + // One rule, applied wherever an untrusted guest names telemetry (span hints, + // guest spans, guest wide events): inside the `gen_ai.*` namespace only + // recognised metadata keys survive, with bounded values. Names are chosen by + // the guest, so enumerating content keys stops only a module that uses the + // canonical ones — `llm.response.text` would carry the same completion. + // + // Attributes *outside* the namespace are left alone. This channel is the + // generic span-hint ABI, not an LLM-only one: `provider.request_id`, + // `rpc.method` and similar diagnostics ride on it, and dropping them would + // remove working observability for every tenant to no security gain — a guest + // that puts a completion in a header value it wrote itself is the free-form + // channel named as residual in ADR-0166, the same as `host_log`. + hints + .attributes + .retain(|(key, _)| llm_namespace_attr_allowed(key)); + + // Names survive; values still have to be bounded, or the prompt simply + // travels as `gen_ai.request.model`. + for (key, value) in hints.attributes.iter_mut() { + if is_llm_namespace_key(key) + && let Some(clamped) = clamp_redacted_metadata_value(value) + { + *value = clamped; + } + } + // The span *name* is guest-supplied free text on the same channel; bound it + // for the same reason. + if let Some(name) = hints.span_name.as_mut() + && let Some(clamped) = clamp_redacted_metadata_value(name) + { + *name = clamped; + } + // Response captures are different in kind and are dropped in full: the value + // is lifted out of the LLM provider's response body by a guest-supplied JSON + // pointer, so the attribute name says nothing about what the value is. There + // is no name that makes a body-derived value safe — `gen_ai.usage.input_tokens` + // pointed at `/content/0/text` is a completion. + hints.response_captures.clear(); +} + pub(crate) fn datadog_visible_span_hint_field(attr_key: &str) -> Option<&'static str> { match attr_key { "tenant" => Some("tenant"), @@ -230,3 +411,7 @@ pub(crate) fn truncate_for_span_attr(value: &str) -> String { out.push_str(MAX_RESPONSE_CAPTURE_TRUNCATION_SUFFIX); out } + +#[cfg(test)] +#[path = "span_hints_test.rs"] +mod redaction_tests; diff --git a/crates/temper-wasm/src/host_trait/span_hints_test.rs b/crates/temper-wasm/src/host_trait/span_hints_test.rs new file mode 100644 index 000000000..1aa65ef7e --- /dev/null +++ b/crates/temper-wasm/src/host_trait/span_hints_test.rs @@ -0,0 +1,352 @@ +//! Redaction contract for span hints (ARN-243 / ADR-0166). +use super::*; + +fn content_hints() -> SpanHints { + SpanHints { + span_name: None, + attributes: vec![ + ("gen_ai.request.model".to_string(), "claude".to_string()), + ( + "gen_ai.input.messages".to_string(), + "SECRET PROMPT".to_string(), + ), + ( + "gen_ai.system_instructions".to_string(), + "SECRET SYSTEM".to_string(), + ), + ("tenant".to_string(), "acme".to_string()), + ], + response_captures: vec![ + ( + "gen_ai.completion".to_string(), + "/content/0/text".to_string(), + ), + ("http.response.body.size".to_string(), "/size".to_string()), + ], + } +} + +#[test] +fn classifies_content_vs_metadata_attrs() { + assert!(is_sensitive_llm_content_attr("gen_ai.input.messages")); + assert!(is_sensitive_llm_content_attr("gen_ai.completion")); + assert!(is_sensitive_llm_content_attr("gen_ai.tool.call.arguments")); + assert!(!is_sensitive_llm_content_attr("gen_ai.request.model")); + assert!(!is_sensitive_llm_content_attr("gen_ai.usage.input_tokens")); + assert!(!is_sensitive_llm_content_attr("tenant")); +} + +/// Guards the content-key list against the open export surface (`apply_span_hints` +/// records every surviving hint). Every content-bearing span field must be +/// denied; every metadata field Datadog shows must survive. A denylist gap +/// here is a leak; a false positive is over-redaction. See ADR-0166. +#[test] +fn every_content_key_is_a_recognized_datadog_visible_field() { + // Raw LLM content that reaches a span. The first five are also in the + // `datadog_visible_span_hint_field` allowlist; the two tool fields reach + // Datadog via the unconditional `set_attribute` path. All must be denied. + let content_fields = [ + "gen_ai.system_instructions", + "gen_ai.input.messages", + "gen_ai.prompt", + "gen_ai.output.messages", + "gen_ai.completion", + "gen_ai.tool.call.arguments", + "gen_ai.tool.call.result", + ]; + for field in content_fields { + assert!( + is_sensitive_llm_content_attr(field), + "content field `{field}` must be redacted (denylist gap = leak)" + ); + } + + // Datadog-visible metadata must never be redacted (no over-redaction), + // and each must be a real recognized field (guards typos in this list). + let metadata_fields = [ + "gen_ai.request.model", + "gen_ai.response.model", + "gen_ai.provider.name", + "gen_ai.system", + "gen_ai.operation.name", + "gen_ai.request.temperature", + "gen_ai.request.max_tokens", + "gen_ai.conversation.id", + "gen_ai.response.finish_reasons", + "gen_ai.usage.input_tokens", + "gen_ai.usage.output_tokens", + "gen_ai.usage.cache_read_input_tokens", + "gen_ai.usage.cache_creation_input_tokens", + "tenant", + "session_id", + "agent_id", + ]; + for field in metadata_fields { + assert!( + !is_sensitive_llm_content_attr(field), + "metadata field `{field}` must NOT be redacted (over-redaction)" + ); + assert!( + datadog_visible_span_hint_field(field).is_some(), + "metadata field `{field}` should be a recognized Datadog-visible field" + ); + } + + // Every Datadog-visible content field must also be denied. + for field in [ + "gen_ai.system_instructions", + "gen_ai.input.messages", + "gen_ai.prompt", + "gen_ai.output.messages", + "gen_ai.completion", + ] { + assert!( + datadog_visible_span_hint_field(field).is_some(), + "content field `{field}` should be a recognized Datadog-visible field" + ); + } +} + +#[test] +fn redacts_sensitive_content_hints_when_not_opted_in() { + let mut hints = content_hints(); + redact_llm_content_hints(&mut hints, false); + + // Content attributes stripped. + assert!( + hints + .attributes + .iter() + .all(|(k, _)| k != "gen_ai.input.messages"), + "prompt attr must be redacted" + ); + assert!( + hints + .attributes + .iter() + .all(|(k, _)| k != "gen_ai.system_instructions"), + "system instructions attr must be redacted" + ); + // Metadata attributes preserved. + assert!( + hints + .attributes + .iter() + .any(|(k, _)| k == "gen_ai.request.model"), + "model metadata must survive" + ); + assert!(hints.attributes.iter().any(|(k, _)| k == "tenant")); + + // Every response capture is dropped, whatever it is named. A capture is + // `(attribute_name, json_pointer)` and the value is lifted straight out of + // the provider's response body, so the name is a guest-chosen label with no + // bearing on whether the value is content. + assert!( + hints.response_captures.is_empty(), + "no body-derived capture may survive for a non-opted-in tenant, got {:?}", + hints.response_captures + ); +} + +/// The content and metadata sets must stay disjoint: a key in both would make +/// the allowlist export content, and the doc on `is_sensitive_llm_content_attr` +/// claims they are separate. +#[test] +fn content_and_metadata_key_sets_are_disjoint() { + for key in [ + "gen_ai.input.messages", + "gen_ai.prompt", + "gen_ai.system_instructions", + "gen_ai.output.messages", + "gen_ai.completion", + "gen_ai.tool.call.arguments", + "gen_ai.tool.call.result", + ] { + assert!(is_sensitive_llm_content_attr(key)); + assert!( + !is_llm_metadata_attr(key), + "`{key}` is content and must not also be metadata" + ); + assert!( + !llm_namespace_attr_allowed(key), + "`{key}` must not pass the namespace allowlist" + ); + } +} + +/// Guest-span and wide-event payloads hand keys over verbatim, so the +/// namespace test has to normalize or `GEN_AI.prompt` is read as ordinary +/// application telemetry and passes straight through. +#[test] +fn namespace_test_normalizes_guest_supplied_keys() { + for key in [ + "GEN_AI.prompt", + " gen_ai.prompt ", + "Gen_Ai.Completion", + "\tGEN_AI.INPUT.MESSAGES", + ] { + assert!( + !llm_namespace_attr_allowed(key), + "`{key}` must be recognized as an unlisted gen_ai.* key" + ); + } + assert!(llm_namespace_attr_allowed("GEN_AI.request.model")); + assert!(llm_namespace_attr_allowed("order.id")); +} + +/// The gate must not be bypassable inside the namespace it protects. Guest +/// modules choose the attribute names (`X-Temper-Span-Attr-*`) and the capture +/// pointers, so a denylist of canonical `gen_ai.*` content keys is defeated by +/// picking another name in that namespace for the same value. +#[test] +fn redaction_is_not_bypassable_by_guest_chosen_names() { + let mut hints = SpanHints::default(); + // Same completion text, under gen_ai.* names no denylist would carry. + hints.attributes.push(( + "gen_ai.response.text".to_string(), + "SECRET COMPLETION".to_string(), + )); + hints + .attributes + .push(("gen_ai.debug.dump".to_string(), "SECRET PROMPT".to_string())); + // A capture labelled as innocuous metadata, pointed at the completion body. + hints.response_captures.push(( + "gen_ai.usage.input_tokens".to_string(), + "/content/0/text".to_string(), + )); + + redact_llm_content_hints(&mut hints, false); + + assert!( + hints.attributes.is_empty(), + "an unrecognized gen_ai.* name must not survive on the strength of \ + being unlisted, got {:?}", + hints.attributes + ); + assert!( + hints.response_captures.is_empty(), + "a body-derived capture must not survive on the strength of its name" + ); +} + +/// The span-hint channel is the generic observability ABI, not an LLM-only +/// one. Redacting must not quietly delete a module's ordinary diagnostics for +/// every tenant — the opt-in list is empty by default, so over-redaction here +/// would be the default behavior for everyone. +#[test] +fn redaction_keeps_non_llm_diagnostics() { + let mut hints = SpanHints::default(); + for (key, value) in [ + ("provider.request_id", "req_01ABC"), + ("rpc.method", "Complete"), + ("http.response.status_code", "200"), + ("tenant", "acme"), + ] { + hints.attributes.push((key.to_string(), value.to_string())); + } + + redact_llm_content_hints(&mut hints, false); + + for key in [ + "provider.request_id", + "rpc.method", + "http.response.status_code", + "tenant", + ] { + assert!( + hints.attributes.iter().any(|(k, _)| k == key), + "non-LLM diagnostic `{key}` must keep working for a non-opted-in \ + tenant; got {:?}", + hints.attributes + ); + } +} + +/// Names pass the allowlist; values must still be bounded, or the prompt +/// simply travels as `gen_ai.request.model`. The guest-supplied span name is +/// free text on the same channel and is bounded for the same reason. +#[test] +fn metadata_values_and_span_name_are_bounded_when_not_opted_in() { + let mut hints = SpanHints { + span_name: Some("N".repeat(4096)), + ..SpanHints::default() + }; + hints + .attributes + .push(("gen_ai.request.model".to_string(), "M".repeat(4096))); + + redact_llm_content_hints(&mut hints, false); + + let (_, model) = hints + .attributes + .iter() + .find(|(k, _)| k == "gen_ai.request.model") + .expect("recognized metadata survives"); + assert!( + model.len() <= MAX_REDACTED_LLM_METADATA_VALUE_BYTES, + "metadata value must be clamped, got {} bytes", + model.len() + ); + assert!( + hints.span_name.as_deref().unwrap_or_default().len() + <= MAX_REDACTED_LLM_METADATA_VALUE_BYTES, + "span name must be clamped" + ); +} + +/// Clamping must not split a multi-byte character. +#[test] +fn clamping_respects_char_boundaries() { + let value = "é".repeat(1024); + let clamped = clamp_redacted_metadata_value(&value).expect("should clamp"); + assert!(clamped.len() <= MAX_REDACTED_LLM_METADATA_VALUE_BYTES); + assert!(value.starts_with(&clamped), "clamp must be a prefix"); + assert!(clamp_redacted_metadata_value("short").is_none()); +} + +#[test] +fn keeps_content_hints_when_opted_in() { + let mut hints = content_hints(); + redact_llm_content_hints(&mut hints, true); + assert!( + hints + .attributes + .iter() + .any(|(k, _)| k == "gen_ai.input.messages"), + "opted-in tenant keeps prompt" + ); + assert!( + hints + .response_captures + .iter() + .any(|(k, _)| k == "gen_ai.completion"), + "opted-in tenant keeps completion" + ); +} + +/// The clamp and the allowlist must agree on what "in the namespace" means. +/// If the allowlist normalizes the key but the clamp matches the raw one, +/// `GEN_AI.request.model` passes as recognised metadata and then skips the +/// clamp — carrying an entire prompt under a metadata name. +#[test] +fn namespace_clamp_and_allowlist_agree_on_unnormalized_keys() { + let prompt = "P".repeat(4096); + let mut hints = SpanHints::default(); + hints + .attributes + .push(("GEN_AI.request.model".to_string(), prompt.clone())); + hints + .attributes + .push((" gen_ai.request.model ".to_string(), prompt.clone())); + + redact_llm_content_hints(&mut hints, false); + + for (key, value) in &hints.attributes { + assert!( + value.len() <= MAX_REDACTED_LLM_METADATA_VALUE_BYTES, + "`{key}` escaped the clamp with {} bytes — the allowlist and the clamp \ + disagree about the gen_ai.* namespace", + value.len() + ); + } +} diff --git a/crates/temper-wasm/src/lib.rs b/crates/temper-wasm/src/lib.rs index 8cf9dbef5..29351a608 100644 --- a/crates/temper-wasm/src/lib.rs +++ b/crates/temper-wasm/src/lib.rs @@ -17,8 +17,9 @@ mod workflow_headers; pub use authorized_host::{AuthorizedWasmHost, WasmAuthzDecision, WasmAuthzGate, extract_domain}; pub use engine::{WasmEngine, WasmError}; pub use host_trait::{ - BinaryHttpInterceptorFn, ProductionWasmHost, ProgressEmitterFn, SecretResolverFn, SimWasmHost, - SpecEvaluatorFn, TextHttpInterceptorFn, WasmHost, parse_connect_frames, + BinaryHttpInterceptorFn, InternalHttpCapability, InternalHttpCapabilityIssuerFn, + ProductionWasmHost, ProgressEmitterFn, SecretResolverFn, SimWasmHost, SpecEvaluatorFn, + TextHttpInterceptorFn, WasmHost, parse_connect_frames, }; pub use stream::{StreamRegistry, StreamRegistryConfig}; pub use types::{ diff --git a/crates/temper-wasm/tests/guest_span_redaction_contract.rs b/crates/temper-wasm/tests/guest_span_redaction_contract.rs new file mode 100644 index 000000000..306b6f7da --- /dev/null +++ b/crates/temper-wasm/tests/guest_span_redaction_contract.rs @@ -0,0 +1,110 @@ +//! ARN-243 wiring contract for the guest manual-span API. +//! +//! The unit tests for this channel call `allowed_attributes` directly, so they +//! stay green if a call site stops calling it — the same "delete the call" +//! hole adversarial review found on the wide-event path. Every entry point that +//! accepts guest attributes must route them through the filter before they reach +//! a span or the manual-export snapshot. See ADR-0166. + +const GUEST_SPANS_SOURCE: &str = include_str!("../src/engine/guest_spans.rs"); + +/// Blank out `//` line comments so a commented-out call cannot satisfy the +/// contract. Byte offsets are preserved (comment bodies become spaces) so +/// ordering assertions still compare positions in the original file. String +/// literals are tracked so a `"http://…"` is not mistaken for a comment. +/// +/// A heuristic, not a lexer: char literals and raw strings are not modelled. +/// Every assertion here asserts *presence*, so a mis-scan fails loudly. +fn strip_line_comments(src: &str) -> String { + let mut out = String::with_capacity(src.len()); + for line in src.split_inclusive('\n') { + let bytes = line.as_bytes(); + let mut in_string = false; + let mut escaped = false; + let mut comment_at = None; + for (i, &b) in bytes.iter().enumerate() { + if escaped { + escaped = false; + continue; + } + match b { + b'\\' if in_string => escaped = true, + b'"' => in_string = !in_string, + b'/' if !in_string && bytes.get(i + 1) == Some(&b'/') => { + comment_at = Some(i); + break; + } + _ => {} + } + } + match comment_at { + Some(at) => { + out.push_str(&line[..at]); + for ch in line[at..].chars() { + out.push(if ch == '\n' { '\n' } else { ' ' }); + } + } + None => out.push_str(line), + } + } + out +} + +/// Every function that takes a guest payload and puts its attributes on a span +/// must filter them first, and must do it before any consumer reads them. +#[test] +fn every_guest_span_entry_point_filters_attributes_before_use() { + let src = &strip_line_comments(GUEST_SPANS_SOURCE); + let entry_points = [ + "pub(crate) fn start_span(", + "pub(crate) fn add_span_event(", + "pub(crate) fn set_span_attributes(", + "pub(crate) fn end_span(", + ]; + + for entry in entry_points { + let start = src + .find(entry) + .unwrap_or_else(|| panic!("guest span entry point `{entry}` not found")); + let end = src[start + entry.len()..] + .find("\n pub(crate) fn ") + .map_or(src.len(), |rel| start + entry.len() + rel); + let body = &src[start..end]; + + let filter_at = body + .find("allowed_attributes(&payload.attributes") + .unwrap_or_else(|| { + panic!( + "`{entry}` must pass guest attributes through `allowed_attributes` \ + before recording them; without it a module publishes prompts and \ + completions on its own span for a non-opted-in tenant (ARN-243)" + ) + }); + assert!( + body[filter_at..].contains("export_llm_content"), + "`{entry}` must filter with the per-tenant policy, not a constant" + ); + + // Nothing may consume the raw payload attributes after that point. + for consumer in ["apply_attributes(", "manual_span_attributes("] { + let mut scan = 0; + while let Some(rel) = body[scan..].find(consumer) { + let at = scan + rel; + let args_end = body[at..] + .find(");") + .map_or(body.len(), |rel| (at + rel).min(body.len())); + let args = &body[at..args_end]; + assert!( + !args.contains("payload.attributes"), + "`{entry}` passes the raw guest attributes to `{consumer}`; it must \ + pass the filtered map (ARN-243)" + ); + assert!( + filter_at < at, + "`{entry}` calls `{consumer}` before filtering the guest attributes" + ); + scan = at + consumer.len(); + } + } + } +} diff --git a/crates/temper-wasm/tests/http_stream_outbound.rs b/crates/temper-wasm/tests/http_stream_outbound.rs index fca665ea1..28897db3c 100644 --- a/crates/temper-wasm/tests/http_stream_outbound.rs +++ b/crates/temper-wasm/tests/http_stream_outbound.rs @@ -18,7 +18,7 @@ use futures_util::StreamExt; use serde_json::Value; use temper_wasm::WasmHost; -use temper_wasm::host_trait::ProductionWasmHost; +use temper_wasm::host_trait::{InternalHttpCapability, ProductionWasmHost}; use temper_wasm::http_stream::{HttpRequestHead, StreamError}; use temper_wasm::types::WasmInvocationContext; @@ -85,35 +85,44 @@ async fn spawn_echo_server() -> String { async fn outbound_streaming_injects_internal_auth_headers() { let base = spawn_echo_server().await; - let mut secrets = BTreeMap::new(); - secrets.insert("temper_api_url".to_string(), base.clone()); - secrets.insert("temper_api_key".to_string(), "secret123".to_string()); - - let host = Arc::new(ProductionWasmHost::new(secrets).with_invocation_context( - WasmInvocationContext { - tenant: "default".to_string(), - entity_type: "Workspace".to_string(), - entity_id: "ws-1".to_string(), - trigger_action: "CreateFile".to_string(), - wasm_module: Some("blob_adapter".to_string()), - trigger_params: Value::Null, - entity_state: Value::Null, - agent_id: Some("operator".to_string()), - session_id: None, - integration_config: BTreeMap::new(), - trace_id: String::new(), - workflow_root_entity_type: Some("CurationQuery".to_string()), - workflow_root_entity_id: Some("cq-1".to_string()), - workflow_run_id: Some("CurationQuery:cq-1".to_string()), - http_request: None, - }, - )); + let host = Arc::new( + ProductionWasmHost::new(BTreeMap::new()) + .with_internal_api_base_url(Some(base.clone())) + .with_internal_capability_issuer(Arc::new(|method, url| { + assert_eq!(method, "PUT"); + assert!(url.ends_with("/internal")); + InternalHttpCapability::new("stream-capability".to_string(), "tenant-a".to_string()) + })) + .with_invocation_context(WasmInvocationContext { + tenant: "default".to_string(), + entity_type: "Workspace".to_string(), + entity_id: "ws-1".to_string(), + trigger_action: "CreateFile".to_string(), + wasm_module: Some("blob_adapter".to_string()), + trigger_params: Value::Null, + entity_state: Value::Null, + agent_id: Some("operator".to_string()), + session_id: None, + integration_config: BTreeMap::new(), + trace_id: String::new(), + workflow_root_entity_type: Some("CurationQuery".to_string()), + workflow_root_entity_id: Some("cq-1".to_string()), + workflow_run_id: Some("CurationQuery:cq-1".to_string()), + http_request: None, + }), + ); let handles = host .http_stream_begin_outbound(HttpRequestHead { method: "PUT".into(), url: format!("{base}/internal"), - headers: vec![("content-type".into(), "image/png".into())], + headers: vec![ + ("content-type".into(), "image/png".into()), + ("authorization".into(), "Bearer guest-root".into()), + ("x-tenant-id".into(), "victim".into()), + ("x-temper-principal-kind".into(), "admin".into()), + ("x-temper-principal-id".into(), "attacker".into()), + ], }) .await .unwrap(); @@ -146,10 +155,12 @@ async fn outbound_streaming_injects_internal_auth_headers() { } let body = String::from_utf8(body).unwrap(); - assert!(body.contains("authorization=Bearer secret123")); - assert!(body.contains("tenant=default")); - assert!(body.contains("principal_kind=agent")); - assert!(body.contains("principal_id=operator")); + assert!(body.contains("authorization=Bearer stream-capability")); + assert!(body.contains("tenant=tenant-a")); + assert!(body.contains("principal_kind=;principal_id=;")); + assert!(!body.contains("guest-root")); + assert!(!body.contains("victim")); + assert!(!body.contains("attacker")); assert!(body.contains("workflow_type=CurationQuery")); assert!(body.contains("body=raw-image-bytes")); } diff --git a/crates/temper-wasm/tests/span_hint_redaction_behavior.rs b/crates/temper-wasm/tests/span_hint_redaction_behavior.rs new file mode 100644 index 000000000..3c5b1db43 --- /dev/null +++ b/crates/temper-wasm/tests/span_hint_redaction_behavior.rs @@ -0,0 +1,168 @@ +//! Behavioral proof for ARN-243 / ADR-0166. +//! +//! The other tests for this gate are unit tests on the redaction helper plus +//! source-text contracts on the call sites. Neither observes what actually +//! reaches an exporter, so both would survive a change that redacts a copy of +//! the hints while the original is still recorded. This test drives a real +//! `ProductionWasmHost::http_call` with guest span-hint headers, exports the +//! resulting span through the OpenTelemetry SDK, and inspects the attributes +//! that came out the other end. +//! +//! The HTTP request itself is expected to fail — the span, its hints, and the +//! export are all complete before the socket is touched, and the assertions are +//! about the span, not the response. + +use std::collections::BTreeMap; + +use opentelemetry::trace::TracerProvider as _; +use opentelemetry_sdk::trace::{InMemorySpanExporter, SdkTracerProvider}; +use temper_wasm::host_trait::{ProductionWasmHost, WasmHost}; +use tracing_subscriber::layer::SubscriberExt as _; + +/// A port that nothing listens on, so `http_call` fails fast after the span has +/// been created and annotated. +const UNROUTABLE_URL: &str = "http://127.0.0.1:1/v1/messages"; + +const PROMPT: &str = "SECRET PROMPT: the user's private message"; +const COMPLETION: &str = "SECRET COMPLETION: the model's private reply"; + +/// Guest-supplied span-hint headers carrying both content and metadata, plus a +/// content value hidden under a metadata name and a capture pointed at the body. +fn guest_hint_headers() -> Vec<(String, String)> { + vec![ + ("content-type".to_string(), "application/json".to_string()), + ( + "X-Temper-Span-Name".to_string(), + "tool.llm_call".to_string(), + ), + ( + "X-Temper-Span-Attr-gen_ai.input.messages".to_string(), + PROMPT.to_string(), + ), + ( + "X-Temper-Span-Attr-gen_ai.completion".to_string(), + COMPLETION.to_string(), + ), + // A gen_ai.* name no denylist enumerates. + ( + "X-Temper-Span-Attr-gen_ai.response.text".to_string(), + COMPLETION.to_string(), + ), + // Legitimate metadata, must survive. + ( + "X-Temper-Span-Attr-gen_ai.request.model".to_string(), + "claude-opus-4-8".to_string(), + ), + // Non-LLM diagnostics, must survive. + ( + "X-Temper-Span-Attr-provider.request_id".to_string(), + "req_01ABC".to_string(), + ), + // A capture labelled as metadata but pointed at the completion body. + ( + "X-Temper-Span-Capture-Response-gen_ai.usage.input_tokens".to_string(), + "/content/0/text".to_string(), + ), + ] +} + +/// Run one `http_call` under a subscriber that exports to memory, and return +/// every attribute key/value recorded on the exported spans. +fn exported_span_attributes(export_llm_content: bool) -> Vec<(String, String)> { + let exporter = InMemorySpanExporter::default(); + let provider = SdkTracerProvider::builder() + .with_simple_exporter(exporter.clone()) + .build(); + let tracer = provider.tracer("arn-243-behavior"); + let subscriber = + tracing_subscriber::registry().with(tracing_opentelemetry::layer().with_tracer(tracer)); + + tracing::subscriber::with_default(subscriber, || { + let host = + ProductionWasmHost::new(BTreeMap::new()).with_llm_content_export(export_llm_content); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("build runtime"); + // The request fails (nothing listens on port 1); the span is already + // created and annotated by then, which is what this test inspects. + let _ = + runtime.block_on(host.http_call("POST", UNROUTABLE_URL, &guest_hint_headers(), "{}")); + }); + + provider.force_flush().ok(); + exporter + .get_finished_spans() + .expect("exported spans") + .into_iter() + .flat_map(|span| { + span.attributes + .into_iter() + .map(|kv| (kv.key.to_string(), kv.value.to_string())) + }) + .collect() +} + +#[test] +fn non_opted_in_tenant_exports_no_llm_content_on_a_real_span() { + let attributes = exported_span_attributes(false); + assert!( + !attributes.is_empty(), + "expected the http_call span to be exported with attributes" + ); + + for (key, value) in &attributes { + assert!( + !value.contains("SECRET PROMPT") && !value.contains("SECRET COMPLETION"), + "LLM content reached the exporter as `{key}` = {value:?}" + ); + } + for content_key in [ + "gen_ai.input.messages", + "gen_ai.completion", + "gen_ai.response.text", + ] { + assert!( + !attributes.iter().any(|(key, _)| key == content_key), + "content attribute `{content_key}` must not be exported; got {attributes:?}" + ); + } + + // Metadata and non-LLM diagnostics still work. + assert!( + attributes + .iter() + .any(|(key, value)| key == "gen_ai.request.model" && value == "claude-opus-4-8"), + "model metadata must survive; got {attributes:?}" + ); + // Documented, and deliberately asserted rather than assumed: a hint attribute + // whose name is not in `datadog_visible_span_hint_field` does not reach the + // exporter at all. `apply_span_hints` records those only via + // `otel_span.set_attribute` on a context-derived handle, which does not + // propagate back to the span the layer exports — so `provider.request_id` is + // dropped regardless of any redaction. Two consequences worth stating: the + // gate's real reach is narrower than the code shape suggests (it can only + // matter for names the export path keeps), and guest diagnostics outside that + // set are being silently lost today. The latter is an observability bug, not a + // security one, and is tracked as ARN-350. This assertion pins the current + // behavior so that fixing ARN-350 forces a deliberate revisit of this test. + assert!( + !attributes + .iter() + .any(|(key, _)| key == "provider.request_id"), + "unexpected: a non-Datadog-visible hint attribute reached the exporter. If \ + ARN-350 was fixed, re-check that the redaction still covers this path; got \ + {attributes:?}" + ); +} + +#[test] +fn opted_in_tenant_still_exports_llm_content_on_a_real_span() { + let attributes = exported_span_attributes(true); + assert!( + attributes + .iter() + .any(|(key, value)| key == "gen_ai.input.messages" && value.contains("SECRET PROMPT")), + "an opted-in tenant must still export content; got {attributes:?}" + ); +} diff --git a/crates/temper-wasm/tests/span_hint_redaction_contract.rs b/crates/temper-wasm/tests/span_hint_redaction_contract.rs new file mode 100644 index 000000000..9e153a7e1 --- /dev/null +++ b/crates/temper-wasm/tests/span_hint_redaction_contract.rs @@ -0,0 +1,209 @@ +//! ARN-243 wiring contract: every host HTTP-call site that parses guest span +//! hints must filter LLM content before applying them, or host-captured +//! prompts/completions would bypass the per-tenant export gate. Unit tests on +//! `redact_llm_content_hints` cannot see whether each call site actually calls +//! it; this source-level contract does. See ADR-0166. + +const HOST_TRAIT_SOURCE: &str = include_str!("../src/host_trait.rs"); + +/// Blank out `//` line comments so a commented-out call cannot satisfy a source +/// contract. Byte offsets are preserved (comment bodies become spaces) so every +/// ordering assertion still compares positions in the original file. String +/// literals are tracked, so a `"http://..."` in code is not mistaken for a +/// comment and does not blank the rest of its line. +/// +/// This is a heuristic, not a lexer: char literals (`'/'`) and raw strings +/// (`r#"..."#`) are not modelled. Both scanned files are free of them today, and +/// every assertion built on this asserts *presence*, so a mis-scan produces a +/// loud failure rather than a silent pass. +fn strip_line_comments(src: &str) -> String { + let mut out = String::with_capacity(src.len()); + for line in src.split_inclusive('\n') { + let bytes = line.as_bytes(); + let mut in_string = false; + let mut escaped = false; + let mut comment_at = None; + for (i, &b) in bytes.iter().enumerate() { + if escaped { + escaped = false; + continue; + } + match b { + b'\\' if in_string => escaped = true, + b'"' => in_string = !in_string, + b'/' if !in_string && bytes.get(i + 1) == Some(&b'/') => { + comment_at = Some(i); + break; + } + _ => {} + } + } + match comment_at { + Some(at) => { + out.push_str(&line[..at]); + for ch in line[at..].chars() { + out.push(if ch == '\n' { '\n' } else { ' ' }); + } + } + None => out.push_str(line), + } + } + out +} + +#[test] +fn every_span_hint_split_is_followed_by_redaction() { + let src = &strip_line_comments(HOST_TRAIT_SOURCE); + let split_marker = "= split_span_hint_headers("; + let redact_marker = "redact_llm_content_hints(&mut span_hints"; + + // Collect every split-site offset first, then require the redact call to + // appear in the interstitial code before the next split site (or EOF). The + // window is exactly the code between two sites, so the guard holds no matter + // how much code is inserted between the split and its redaction — no magic + // fixed-size window that could silently exclude a displaced redact call. + let mut sites = Vec::new(); + let mut from = 0; + while let Some(rel) = src[from..].find(split_marker) { + let at = from + rel; + sites.push(at); + from = at + split_marker.len(); + } + assert!( + sites.len() >= 4, + "expected at least 4 span-hint split sites to guard, found {}", + sites.len() + ); + for (i, &at) in sites.iter().enumerate() { + let end = sites.get(i + 1).copied().unwrap_or(src.len()); + let window = &src[at..end]; + let redact_at = window.find(redact_marker).unwrap_or_else(|| { + panic!( + "split_span_hint_headers call at byte {at} is not followed by \ + redact_llm_content_hints before the next split site; host-captured LLM \ + content would bypass the per-tenant export gate (ARN-243)" + ) + }); + + // Presence is not enough: the redaction has to run *before* the hints are + // applied. Moving the redact call below `apply_span_hints` leaves the + // markers all present while exporting every content attribute, so pin the + // order rather than the existence. + for apply_marker in ["apply_span_hints(", "apply_response_captures("] { + let mut scan = 0; + while let Some(rel) = window[scan..].find(apply_marker) { + let apply_at = scan + rel; + assert!( + redact_at < apply_at, + "in the span-hint block at byte {at}, `{apply_marker}` (offset \ + {apply_at}) runs before redact_llm_content_hints (offset \ + {redact_at}); the hints would be applied unredacted (ARN-243)" + ); + scan = apply_at + apply_marker.len(); + } + } + } +} + +/// The gate is only as good as the value handed to it. A call site that passes a +/// literal `true` — or any expression that is not the host's per-tenant decision +/// — exports content for every tenant while every ordering and unit test stays +/// green. This is the mutation an ordering contract cannot see. +#[test] +fn span_hint_redaction_uses_the_per_tenant_policy_not_a_literal() { + let src = &strip_line_comments(HOST_TRAIT_SOURCE); + let expected = "redact_llm_content_hints(&mut span_hints, self.export_llm_content);"; + let calls = src + .matches("redact_llm_content_hints(&mut span_hints") + .count(); + let policy_calls = src.matches(expected).count(); + assert_eq!( + calls, + policy_calls, + "every span-hint redaction must pass `self.export_llm_content`; {} of {calls} \ + call sites pass something else (a hardcoded `true` would leak every tenant)", + calls - policy_calls + ); + assert!( + policy_calls >= 4, + "expected at least 4 guarded sites, found {policy_calls}" + ); +} + +/// The guest metric channel (`host_emit_metric`) has two sinks that read the +/// guest's tags — a span event and the OTel meter. Unit tests on the helper +/// cannot see whether `emit_metric` calls it, nor whether it calls it early +/// enough; this pins both. +#[test] +fn guest_metric_tags_are_redacted_before_either_sink() { + let src = &strip_line_comments(HOST_TRAIT_SOURCE); + let start = src + .find("fn emit_metric(&self, metric_json: &str)") + .expect("ProductionWasmHost must implement emit_metric"); + // Scope to this function: the trait's default impl and SimWasmHost's also + // define `emit_metric`, and only the production one carries guest tags. + let end = src[start..] + .find("\n fn ") + .map_or(src.len(), |rel| start + rel); + let body = &src[start..end]; + + let redact_at = body + .find("redact_guest_string_tags(&mut payload.tags") + .expect( + "emit_metric must redact guest tags: they are guest-named and \ + guest-valued strings that reach OTel metrics and a span event (ARN-243)", + ); + for sink in [ + "record_guest_metric_span_event(", + "payload\n .tags", + ] { + if let Some(sink_at) = body.find(sink) { + assert!( + redact_at < sink_at, + "guest metric tags must be redacted before `{sink}` reads them; \ + redact at {redact_at}, sink at {sink_at}" + ); + } + } + assert!( + body.contains("self.export_llm_content"), + "the metric redaction must be driven by the per-tenant policy, not a constant" + ); +} + +/// `build_guest_wide_event` is the other guest-authored telemetry record. Its +/// unit tests call the helper directly, so deleting the *call* here left them all +/// green — this pins the wiring, and that the guest half is judged before any +/// host-derived field is merged in. +#[test] +fn guest_wide_event_fields_are_redacted_at_the_call_site() { + let src = &strip_line_comments(HOST_TRAIT_SOURCE); + let start = src + .find("fn build_guest_wide_event(&self, event_json: &str)") + .expect("ProductionWasmHost must build guest wide events"); + let end = src[start..] + .find("\n fn ") + .map_or(src.len(), |rel| start + rel); + let body = &src[start..end]; + + let redact_at = body + .find("redact_guest_wide_event_fields(&mut tags, &mut attributes") + .expect( + "build_guest_wide_event must redact the guest-supplied tags and \ + attributes: a wide event carries guest-chosen names and values \ + straight to the backend (ARN-243)", + ); + assert!( + body[redact_at..].contains("self.export_llm_content"), + "the wide-event redaction must be driven by the per-tenant policy, not a constant" + ); + // The host merges its own fields (tenant, entity_type, trigger_action) in + // after this point; judging them would be wrong, so the redaction has to come + // first — which is also what keeps the guest half from being judged twice. + if let Some(merge_at) = body.find(".entry(\"tenant\".into())") { + assert!( + redact_at < merge_at, + "guest fields must be redacted before host-derived fields are merged in" + ); + } +} diff --git a/docs/adrs/0157-class-a-auth-edge.md b/docs/adrs/0157-class-a-auth-edge.md new file mode 100644 index 000000000..247643469 --- /dev/null +++ b/docs/adrs/0157-class-a-auth-edge.md @@ -0,0 +1,682 @@ +# ADR-0157: Credential-Bound Class A Authentication Edge + +- Status: Accepted +- Date: 2026-07-06 +- Revised: 2026-07-11 after adversarial validation of PR #343 +- Deciders: Temper core maintainers +- Related: ADR-0033, ADR-0043, ARN-165, ARN-166, ARN-167, ARN-170, ARN-187, ARN-192, ARN-219, ARN-231, ARN-233 + +## Context + +Temper historically reconstructed Cedar authority from request headers. That +made `x-temper-principal-*`, `x-temper-agent-*`, `x-temper-attr-*`, and +`x-tenant-id` part of the security boundary even though an HTTP client or WASM +guest could supply them. + +The first version of this ADR attempted to make those headers safe by stripping +them at one router edge, adding an internal "trusted" marker, and materializing a +credential-derived principal back into headers. Validation showed that this was +still a header trust model with several independent bypasses: + +1. A WASM guest could overwrite the inherited principal and tenant on the + in-process local-TData path. The host then added the trusted marker, turning + the guest's values into Admin authority. A focused Cedar/OData test changed a + denied Customer create into a successful cross-tenant Admin create. +2. The exported kernel `build_router` did not install the strip layer. A caller + that supplied the marker could still reach Admin-only behavior. There is no + current production direct embed, but the API made a future unsafe embed easy. +3. When `TEMPER_API_KEY` was absent, the network server passed requests through + as `Customer::anonymous`. Shipped any-principal policies grant that principal + real read and mutation capabilities, so "unprivileged" was not fail-closed. +4. Every `ProductionWasmHost` received the deployment-wide API key. A permitted + guest could select another tenant and use that ambient credential as a + cross-tenant Admin without knowing the key. +5. The internal blob HTTP handlers ignored the authenticated tenant and always + addressed `default`, while the WASM Cedar gate treated any loopback URL + containing `/_internal/blobs` as trusted. A non-default credential could + therefore address default-tenant storage, and a near-match URL on another + local service could inherit the blob exception. +6. Management authorization was split and, in several cases, absent. The admin + router accepted only in-process Admin/System kinds even though network + credentials intentionally resolve to Agent, while tenant, Genesis install, + REPL, and server-local spec-directory routes allowed any authenticated + credential to reach destructive or cross-tenant operations. A duplicate + `/observe/tenants/{id}` deletion route also bypassed the path-tenant guard. +7. Inline spec submission joined caller filenames onto a predictable temporary + directory and could write through `..` or absolute paths. Its optional + `cedar_policies` field activated arbitrary policy text after only + `submit_specs`, bypassing `manage_policies`, durable policy rows, and the + approval transaction. +8. Tenant-binding the compatibility blob route was insufficient: any valid + tenant credential could still read or write an arbitrary object key without + a Cedar decision. +9. Native CLI adapters minted permanent `AgentCredential` rows with no expiry or + revocation. A token captured from the child environment remained valid after + the invocation, and a missing minted credential let the child inherit the + server's deployment-wide `TEMPER_API_KEY`. This is ARN-231's permanent + adapter-credential path and part of ARN-170's Class A boundary. +10. The isolated spec-verification subprocess inherited the server's complete + environment, including deployment, database, and provider credentials, and + its timeout dropped the wait future without configuring the child to die. + +Loopback transport, header order, and possession of a deployment secret are not +identity. Authority must be derived once from a credential and remain typed and +tenant-bound until the operation is authorized. + +## Decision + +### 1. Authority is an immutable typed request context + +Introduce `AuthenticatedRequestContext`, containing a `TenantId` and the exact +`SecurityContext` produced by credential resolution. Its fields are private and +it is carried as an axum request extension or a direct in-process argument. + +Handlers consume this context; they do not reconstruct authority from headers. +`SecurityContext::from_headers` remains only for explicitly untrusted/anonymous +compatibility paths and cannot produce Customer, Agent, Admin, System, scopes, +roles, ABAC attributes, or action provenance from caller input. + +The edge still removes the complete `x-temper-*` authority namespace as defense +in depth. Only the `x-temper-observe-*` and `x-temper-workflow-*` correlation +namespaces survive, and those values are never read as Cedar authority. There is +no trusted-principal header, materialization middleware, or +`PreAuthenticatedRequest` bypass. + +### 2. Bearer authentication is tenant-scoped and fails closed + +For every protected network request, middleware: + +1. validates the requested tenant, +2. resolves the bearer token in that tenant's `AgentCredential` registry, +3. constructs one `AuthenticatedRequestContext`, and +4. rejects the request with 401 when any step fails. + +The deployment `TEMPER_API_KEY` is bootstrap material, not ambient Admin +authority. Bootstrap may register it as the verified `operator` AgentType in an +explicit tenant. The normal tenant resolver then handles it exactly like any +other credential. Registering the same operator in another tenant is an explicit +administrative act; a match in one tenant never authorizes another. + +There is no no-key network pass-through. The only unauthenticated routes are the +exact liveness/service-discovery endpoints and the credential-resolution +bootstrap endpoint. Local development that needs protected routes must create a +credential or opt into a separately named loopback-only development server; it +must not silently become `Customer::anonymous`. + +An active `AgentCredential` with a non-empty `expires_at` is valid only when the +field is well-formed RFC3339 and strictly later than the injected current time. +Malformed and expired values fail closed. Successful identity resolutions are +not cached: every protected request validates the credential, validates its +linked `AgentType`, then re-reads the credential and requires the same sequence, +status, and fields. That stability check prevents a mixed-time identity assembled +from two authority states that were never active together. When an event journal is configured, resolution ignores +snapshots and strictly replays each complete durable journal; read failures, +sequence gaps, malformed or actor-misbound events, envelope/payload action +mismatches, transitions incompatible with the active spec, contradictory +tombstones, and history after a terminal tombstone deny the request. In-memory +deployments read their single local actor state. This makes credential +revocation, generic credential deletion, and AgentType deprecation effective on +the next request, including when another replica performed the durable mutation. + +This deliberately adds three authoritative state reads to each protected request. +For a persistent deployment those reads are full journal replays, each bounded +by the existing 10,000-event replay budget. Identity journals are expected to be +short, but the added latency is accepted in exchange for eliminating a +revocation window. A later optimization must provide a durable, replica-visible +version or revocation generation with equivalent fail-closed semantics; a TTL +positive cache is not compatible with this boundary. + +Tenant authorization also fails closed when no tenant policy set is active. +`ServerState` starts with the default-deny engine, and +`authorize_for_tenant` never falls back to a process-global compatibility +policy. A missing, corrupt, or failed policy load therefore denies requests +until that exact tenant's last-known-good policy is restored; it cannot become +permit-all or borrow another scope (ARN-230). + +Cedar resource identity is server-derived as well. Shared resource builders +copy ordinary domain fields first, discard the reserved `id`/`Id`, +`status`/`Status`, `has_spec`, and `ctx_*_status` namespace, then install the +canonical entity ID, spec-defined lifecycle state, governance bit, and resolved +context-entity statuses. Collection creates reject conflicting identity aliases +and any caller lifecycle value that differs from the spec initial state. The +runtime publishes both `id`/`Id` and `status`/`Status` for compatibility with +existing lowercase and OData-style schemas, but strips all caller values first +and derives every alias from the same entity ID and lifecycle state. +Content-addressed creates use the same builder with their predeclared digest ID. +A caller cannot therefore select a permitted Cedar resource while persisting or +targeting a different entity. + +The reserved-field contract is shared with the spec parser and entity mutation +boundary. Specs cannot declare runtime-owned field names as state variables or +action parameters. Runtime action parameters, direct field updates, initial +state, snapshots, and replay are sanitized through one helper before they can +become fields or durable event parameters; identity and lifecycle are then +reinstalled from `EntityState`. This protects non-OData ingress and legacy +persisted input without maintaining a second local denylist. + +Bound actions, PATCH, PUT, and DELETE bind each Cedar decision to a canonical digest of +the exact local actor sequence, lifecycle status, and fields that were +authorized. The actor compares that digest inside its mailbox immediately +before mutation. A stale decision returns 409 and changes nothing. Including +fields as well as the durable sequence is required because in-memory/no-journal +actors can mutate while their sequence remains zero. Field writes use one ask +attempt: retrying the same compare-and-set after a lost reply could misreport a +committed write as a conflict, so clients must re-read and re-authorize. Actions +retain their idempotency key; a retry of an already-applied action returns the +cached/durable result before evaluating the now-stale digest. + +That local compare-and-set does not by itself prove a Cedar `ctx_*_status` +derived from another entity stream. A second read or a process-local mutex would +still race across replicas. Context-dependent authorization therefore requires +the ARN-192 event-store primitive to atomically compare the referenced stream +sequences and append the target field event in one backend transaction. PR #343 +must remain open until that guarded append is integrated; this ADR explicitly +rejects shipping a local-only check as a complete cross-entity fix. + +### 3. The kernel router enforces the boundary itself + +`build_router` installs the authority-header strip and a protected-route guard, +so an embedder cannot accidentally expose handlers that trust raw headers. +Protected handlers require `AuthenticatedRequestContext`. Tests and trusted +in-process callers install an explicit typed context; they do not forge HTTP +headers. Webhook ingress is governed by its separate Class B admission boundary +and static/liveness routes remain deliberately public. + +`$hints` is not static service metadata. Hints can be enriched from runtime +trajectory analysis, so the endpoint requires a typed credential and reads a +tenant-keyed bounded map. One tenant cannot observe another tenant's learned +operational guidance. + +Tenant selection after authentication comes from the typed context. A caller's +`x-tenant-id` is only an input to credential resolution and must equal the +context tenant downstream. + +### 4. Internal WASM calls use capabilities, never the root key + +The local-TData optimization passes the invocation's typed context and fixed +tenant directly to OData. Guest-supplied authority and tenant headers are +discarded. The guest may supply ordinary content, tracing, and idempotency +headers only. + +HTTP fallthrough that must re-enter the server uses a server-issued internal +invocation credential with all of these properties: + +- opaque, high-entropy, and stored only as a digest; +- bound to the exact tenant, HTTP method, canonical path/query, and typed + `SecurityContext`; +- short-lived and single-use; +- held in a bounded `BTreeMap` with deterministic eviction order and explicit + expiry; +- consumed by the normal bearer edge before any handler runs. + +Only the server-owned canonical API base URL classifies a destination as +internal. Tenant secrets and integration configuration cannot reclassify an +external origin and cause a capability to be sent there. + +The local blob fast path has a separate, narrower capability. It is constructed +only after Cedar authorizes the tenant's `blob_endpoint` bootstrap secret, and +it binds the exact parsed loopback scheme, host, explicit port, and +`/_internal/blobs` path. Only `GET` and `PUT` below that exact path are exempted +from the ordinary outbound-HTTP policy. Other ports, userinfo, query strings, +path substrings, and sibling paths fall through to Cedar default-deny. The fast +path and its authorization gate share the same parsed endpoint type so their +classifiers cannot drift. + +Issuance uses an injected cryptographic token source in production and an +injected seeded source in simulation. Redirects are disabled for capability +requests. A wrong tenant, method, path, replay, or expired credential returns +401 and never falls back to another bearer interpretation. + +`SecurityContext::system()` is not capability-delegable. Both issuance and +consumption reject it, because an opaque lookup would still reconstitute System +authority across an HTTP bearer boundary. System-owned work must use a direct +typed kernel API. + +`ProductionWasmHost` no longer reads or receives `TEMPER_API_KEY`, and the root +key is not exposed as a WASM secret. Native adapters declare whether they need a +tenant-scoped platform credential; only the Claude Code and Codex CLI adapters +do. Before either child starts, `env_clear` removes the complete server process +environment. A small non-authority runtime allowlist (`PATH`, temporary/locale/ +terminal values, and certificate-bundle paths) is copied back. Provider +credentials come only from the current tenant's named secrets and only for the +selected adapter; `HOME`, `CODEX_HOME`, `CLAUDE_CONFIG_DIR`, and provider base +URLs must be explicit integration configuration. The newly minted invocation +credential is then installed. Database, Turso, deployment, webhook, proxy, +unrelated-provider, and arbitrary tenant secrets cannot cross by ambient +inheritance. If required credential or explicit CLI configuration is absent, +the child fails normally instead of recovering ambient server authority. + +CLI stdin is closed and stdout/stderr are captured through one shared bounded +runner, not `Command::output`. Each stream has a 4 MiB retention budget. Both +pipes are read concurrently; crossing either budget immediately kills and reaps +the child and fails the invocation, preventing an adapter process from growing +the server heap without bound. + +Adapter credentials have a deterministic 61-minute expiry derived from +`sim_now()`, paired with a 60-minute invocation budget. One detached task owns +both adapter execution and cleanup, so cancellation of the request awaiting it +does not cancel cleanup. It catches adapter errors and panics, terminates CLI +children on timeout, and applies the normal spec-governed `Revoke` transition +through the entity actor with a stable idempotency key and three bounded retry +attempts before returning. The durable expiry is the fail-closed backstop for a +process-wide crash that prevents cleanup from running. + +Minting is itself Cedar-authorized as `Issue` on the prospective +`AgentCredential`, using the original invocation security context and resource +attributes that include the target `agent_type_id`, instance ID, expiry, and +server-generated credential ID. Permission to run the source entity action is +not treated as implicit permission to mint any AgentType identity. Missing +authority context or a denied delegation aborts before an `AgentCredential` +actor is created. + +Credential plaintext is a domain-separated SHA-256 derivation over two +independent scheduler-provided UUIDv7 values. This retains deterministic DST +injection while exceeding the 128-bit entropy bar and hiding UUID timestamp and +version structure from the bearer representation. + +The plaintext credential exists only in the invocation context and child +environment: serde omits it, debug formatting exposes only a presence bit, and +adapter results/errors are recursively scrubbed of both the invocation bearer +and every tenant secret before they can be logged, returned, or persisted as +callback parameters. Longer secret values are scrubbed first so overlapping +values cannot leave a suffix. Durable credential events carry only the digest, +non-secret prefix, and expiry. + +The spec-verification subprocess receives an empty environment and has +kill-on-drop enabled before it parses untrusted IOA input. Its 30-second timeout +therefore terminates the child instead of leaving an orphan holding inherited +server secrets. + +### 5. Authorization consumes the resolved context unchanged + +OData, Observe, API, REPL, OTS, and tenant-access middleware all use the same +typed context. Session/workflow metadata may enrich tracing, but cannot replace +the principal, tenant, scopes, role, agent type, verification status, or action +context. In-process `SecurityContext::system()` remains available only to code +that directly calls a typed kernel API; it is never serializable into HTTP +authority. + +Management routes authorize the exact operation and resource rather than +hard-coding a network-unreachable principal kind: + +- admission and profiling use `manage_admission` on + `AdmissionControl::` and `capture_profile` on the exact + `Profiler` mode; +- REPL execution uses `execute_repl` on `Repl::`; +- server-local directory loading requires a distinct + `load_specs_from_directory` permission on the canonical + `SpecDirectory::`, after the credential tenant matches the target; +- tenant membership and lifecycle routes use exact Tenant/TenantUser resources, + while deployment-wide create/list operations additionally require a + default-tenant control-plane credential; and +- Genesis installation binds the body target to the credential tenant and + authorizes the pinned `App` resource before materialization. Follow-update + reads are authorization-gated and filtered to that tenant. + +The duplicate Observe tenant-deletion route is removed. Shared helpers perform +these checks; there is no second header- or route-specific identity model. + +Server-local spec directories are canonicalized and must be real directories, +not symlinks. Model, invariant, and IOA inputs must be regular files and are +subject to per-file, file-count, directory-entry, and aggregate-byte budgets. +Inline specs stage in a unique process-private `TempDir`; every filename is a +normalized relative path below the one exact `model.csdl.xml` directory. +Absolute paths, `.`/`..`, suffix-confused model names, sibling roots, and +oversized components fail before a host write. Non-empty bundled Cedar policy +text is rejected and must use the separately authorized policy API. + +### 6. Tenant ownership is enforced by durable evolution APIs + +Authentication at a handler is not data isolation. Every trajectory analysis, +feature request, evolution record, record-chain traversal, and live evolution +event carries the credential-bound tenant into the durable store operation. +Shared Postgres tables include `tenant` in every predicate and write. Turso +tables retain the tenant column even when a database is tenant-routed, so a +misrouted store cannot turn into cross-tenant access. Filters are applied in SQL +before ordering or limiting; handlers never fetch a global result and postfilter +it. + +Existing Turso databases receive an idempotent tenant-column migration. Legacy +rows are assigned to `default`, preserving their historical ownership without +making them visible to newly created tenants. Evolution broadcasts retain the +tenant on the event and subscribers filter before serialization. + +### 7. Governance approval is durable before it becomes authority + +Approving a pending authorization decision follows one ordered state change: + +1. validate the candidate combined Cedar policy without activating it; +2. durably persist the policy and approved decision; +3. activate the exact persisted candidate in the tenant engine. + +Persistence errors are returned to the caller and never collapsed into a +best-effort boolean. If runtime activation fails after durable writes, the +operation compensates both durable records to their previous values before +returning an error. A decision is never reported approved while its policy is +absent, and a policy is never left active while its decision remains pending. + +### 8. Raw content-addressed ingest admits the resource before reading bytes + +`POST /tdata/Blobs/Temper.IngestRaw` preserves the existing 2 GiB object +contract without materializing the raw body, canonical bytes, or either base64 +field on the heap. The client must supply `X-Expected-Object-Id`, the lowercase +40-character SHA-1 of `blob \0`. Requiring the expected ID +is a deliberate protocol change: without it, a content-derived resource ID is +unknowable until after an attacker-controlled body has already been consumed, +so exact Cedar and quota admission cannot happen at the security boundary. + +Before polling the request body, the handler: + +1. validates the typed principal, tenant, repository ID, expected object ID, + and declared length; +2. validates the declaration against the object and process staging capacities, + then acquires one tenant-fair upload slot, one bounded global concurrency + slot, and one fixed staging unit; +3. verifies Cedar create authority for that exact Blob ID, repository, and + declared size; +4. runs repository and account admission, then reserves the exact owner bytes + in the shared commons storage-cap ledger; and +5. verifies that a durable object-store backend and staging path are available. + +The body is then copied through the shared blob-store I/O boundary into a +temporary staging object while its canonical SHA-1 is computed. Short, long, +failed, cancelled, and wrong-digest streams delete their staging object and +cannot create an entity. The staging budget is injected into `ServerState` and +uses deterministic defaults. Its byte permits grow in fixed units with bytes +actually accepted by the staging file, not with the attacker-controlled +declaration. Aggregate staged bytes therefore cannot exceed the configured +capacity, while a stalled 2 GiB declaration holds one staging unit rather than +reserving gigabytes. An upload that would cross the actual-byte budget fails +and releases its temporary file and permits. + +The owner-byte reservation is RAII and all ordinary commons writes include +pending reservations in their cap calculation. The coarse commons mutation +lock is held only while a reservation is created and while final metadata is +persisted, never while an attacker-controlled body or remote object-store +request is pending. Cancellation removes the reservation; successful metadata +persistence clears the storage projection cache and releases the reservation +before unlocking final publication. + +Raw-upload admission is tenant-fair rather than a single global slowloris +queue. A tenant may occupy at most one raw-upload slot at a time, while a +separate bounded global concurrency budget allows other tenants to progress. +The tenant-slot registry itself is bounded and deterministic. Every slot, +incremental staging reservation, and owner-byte reservation is RAII, including +when the request future is cancelled. + +The staging copy has three independent progress bounds: a maximum interval +without a non-empty body chunk, a total upload deadline, and a minimum average +throughput after a short grace period. A client cannot keep a global upload +slot indefinitely or grow its actual-byte staging reservation by stalling or +sending one byte just before the idle deadline. Independent total-operation +boundaries cover staging and each subsequent local or S3 object-store stream. +S3 clients use explicit connect and request timeouts; local staging, flush, +sync, and publish operations use bounded async I/O waits. These are production +I/O deadlines and do not enter deterministic simulation state. + +After digest verification, the blob-store boundary reads staging in bounded +chunks and streams two standard field-overflow JSON representations: base64 +content and base64 canonical bytes. Their keys use the existing +`field-overflow/sha256/*.json` namespace and their entity fields use the +existing `__temper_blob_ref` envelope. This keeps OData and Genesis hydration +on one representation instead of introducing a second raw-blob schema. Both +overflow objects must be durable before the small metadata entity is created. +Content-addressed orphan objects after a later failure are harmless and may be +swept; a partially created Blob entity is not permitted. + +The create response is metadata-only for the two binary fields. Hydrating +multi-gigabyte fields merely to echo an accepted upload would recreate the +memory-amplification vulnerability. Existing read paths remain the explicit +place to request field content. + +### 9. Blob hydration is aggregate-bounded and large fields are media streams + +An overflow envelope is a media descriptor, not permission to allocate its +declared size. Generic OData entity/list/expand reads, Observe reads, query +materialization, and action responses share a 1 MiB aggregate inline hydration +budget for the complete response. No individual overflow value above the +normal 128 KiB inline ceiling is fetched for those JSON responses. Once either +budget is exhausted, the original bounded `__temper_blob_ref`, size, and +encoding descriptor remains in the response. Blob-store reads check the +authoritative object size before buffering, so a forged or stale small size in +an envelope cannot bypass the budget. A shared 64-attempt I/O budget also +bounds missing, corrupt, or repeatedly referenced objects that consume no byte +budget; failed keys are remembered for the response and are not retried. + +Blob-store resolution adds a deterministic tenant namespace below every +non-default local root and remote bucket prefix. A descriptor copied or guessed +from another tenant therefore resolves in the caller's namespace, not the +source tenant's. The historical database blob table has no tenant column, so +its legacy fallback and shadow writes are restricted to the `default` tenant; +non-default tenants never fall back into that global keyspace. + +The compatibility `/_internal/blobs/{key}` transport also consumes +`AuthenticatedRequestContext` and selects that exact credential tenant. It +never substitutes `default`, and it requires `read_blob_object` or +`write_blob_object` on the exact `BlobObject::` before touching storage. +This closes the overlap with ARN-219's file helper authorization audit and +ARN-233's tenant-to-storage-key audit without claiming those broader sink +reviews are complete. + +WASM dispatch uses separate hard aggregate budgets for both inline hydration +and its deferred cache. The deferred map is never an unbounded alternate heap: +an entry that exceeds its individual limit or the invocation's remaining +aggregate cache budget stays a descriptor and is not fetched. These limits are +below the default guest memory budget and are enforced before the invocation +thread is created. + +Large Git Blob fields remain fully available through authenticated primitive +media endpoints: + +- `GET /tdata/Blobs('')/Content/$value` streams the decoded raw blob body. +- `GET /tdata/Blobs('')/CanonicalBytes/$value` streams the decoded + `blob \0` representation. + +Both paths require the same typed request context and exact Blob read +authorization as the entity endpoint. They load and authorize only bounded +metadata, validate that the requested property is one of the two supported +binary fields, require a canonical SHA-256 field-overflow key and JSON encoding, +check both encoded and decoded lengths, and verify the serialized object +against its content-addressed key while forwarding bounded chunks from the +object store through an incremental JSON-string/base64 decoder. A separate +stream-concurrency budget prevents slow media consumers from consuming the +ordinary blob-I/O budget. The paths never build a JSON string or a decoded +body-sized `Vec`. + +Genesis materialization consumes that same representation incrementally. +Blob `Content` is decoded directly into an RAII temporary file and atomically +published at the destination. App identity components are validated before +joining cache paths, cache-root names include a digest of the complete pinned +reference, and complete trees are staged beside the destination before a +rollback-safe directory replacement. Remote bundle bodies, manifests, file +counts, individual files, and aggregate exported bytes have explicit budgets; +bundle traversal rejects symbolic links. Before object reads or file creation, +tree materialization now consumes closure-wide app, tree-object, tree-entry, +depth, canonical-tree-byte, file-count, per-file, and aggregate-file-byte +budgets. Conflicting versions of one dependency and different owners that map +to the same cache directory fail closed instead of selecting traversal order. +The materializer no longer performs +`Vec -> serde_json::Value/String -> base64 Vec` for large Blob content, and a +failed, truncated, malformed, or over-budget stream cannot leave a partially +published application file. + +Pinned Git identity is not yet independently recomputed across Commit, Tree, +and Blob objects at this boundary. That remains a release blocker for treating +the Genesis registry itself as hostile rather than an authenticated registry. + +### 10. Buffered text batches have positional and byte budgets + +The authenticated file-text batch endpoints remain buffered JSON APIs, so they +accept at most 100 distinct, non-empty identifiers. Duplicate identifiers are +rejected instead of being authorized once and fetched repeatedly. Each text +item is limited to the same 2 MiB boundary as the public buffered File +`$value` write path, and the aggregate text response is limited to 16 MiB. +Larger files use the authenticated streaming `$value` endpoint. The state-layer +reader enforces the same contract as the HTTP handler and consumes the response +budget before retaining each result, so an internal caller cannot bypass it. + +### 11. Fresh audit findings not falsely claimed as closed + +The admission endpoint's authentication is corrected here, but its runtime +override is not effective. `override_caps(entity)` writes a process-global +controller entry, while dispatch always re-reads tenant spec caps and passes +them to `try_acquire_with_caps`; that call replaces the registered entry. The +endpoint can therefore report success without changing effective admission. +An ignored red regression records the required precedence. A separate ARN must +redesign overrides as tenant-keyed dispatch inputs; this ADR does not describe +the current no-op as fixed. + +The same fresh audit also found a broader exact-row authorization gap in +Observe handlers that pass no resource ID to Cedar before fetching a selected +entity/spec/module/record, plus process-global health/metrics/refresh surfaces +that cannot be made tenant-safe until the underlying metrics state is tenant +dimensioned. Those are ARN-219-class sink audits and require dedicated lanes; +the credential edge and management routes fixed here do not imply those +broader surfaces are complete. + +## Consequences + +### Positive + +- One credential resolution determines both tenant and principal. +- Raw headers, loopback origin, and a shared root secret cannot become authority. +- Local and network WASM calls preserve least privilege without duplicate auth + implementations. +- Direct router embeds fail safe by default. +- Repeated batch identifiers cannot multiply one authorized blob read into an + unbounded buffered response. +- The same foundation can be reused by ARN-166, ARN-167, and ARN-187. + +### Negative + +- Existing tests and internal callers that asserted principal headers must move + to typed contexts. +- Operator access must be registered per tenant instead of implicitly spanning + the deployment. +- Internal HTTP re-entry requires a bounded credential store and explicit token + source rather than one environment-variable lookup. +- Raw Blob clients must compute and send the canonical object ID before upload. + The server performs extra sequential disk reads to keep memory bounded, and + the configured staging-byte budget limits aggregate actual in-flight disk use + in fixed-size accounting units. +- Large Blob fields remain descriptors in JSON reads. Clients that need their + bytes must use the authenticated property `$value` media endpoint. +- A tenant can run only one raw ingest at a time. This is deliberate fair-share + admission; independent tenants retain progress even when one sender stalls. +- Protected requests perform two authoritative identity-state reads. Persistent + deployments pay bounded full-journal replay latency until a durable, + replica-visible version primitive is available. +- Native adapter invocations have a one-hour execution budget. Work that needs + more time must checkpoint and resume under a newly minted credential. + +## Verification + +The change is accepted only with end-to-end tests proving: + +- forged principal, scope, role, ABAC, action-context, and tenant headers do not + affect the typed principal; +- missing, malformed, wrong-tenant, expired, replayed, wrong-method, and + wrong-path credentials are rejected before persistence; +- an already-used resolver rejects a directly revoked credential and a + credential linked to a newly deprecated AgentType on its next call; generic + OData deletion cannot retain authority, and a second `ServerState` sharing the + durable store observes a revocation without process-local invalidation; +- the no-key network server rejects protected reads and writes; +- the exported kernel router rejects protected requests without a typed context; +- `$hints` rejects anonymous requests and returns only the authenticated + tenant's bounded hint set; +- local TData and real `ProductionWasmHost` fallthrough preserve the exact + invocation principal and tenant; +- internal blob writes and reads remain isolated for identical keys in two + tenants, sibling keys require separate Cedar authority, and near-match + loopback origins/paths receive no blob capability; +- admin, REPL, tenant-management, user-membership, and Genesis installation + routes reject missing, wrong-tenant, wrong-resource, and policy-less typed + principals before mutation; the duplicate Observe deletion path is absent; +- server-local spec loading rejects cross-tenant credentials, unapproved + canonical directories, symlinked files/directories, and all configured + count/byte budgets before registration; inline paths cannot escape their + unique staging directory and bundled Cedar text cannot become authority; +- the deployment key is never selected as an Agent/Admin fallback; and +- CLI adapter children never inherit the deployment key; captured invocation + credentials are rejected by a second `ServerState` immediately after success, + adapter error, and caller cancellation, and are rejected after deterministic + expiry even when cleanup does not run; plaintext is absent from serialized + contexts, returned results/errors, debug output, and durable journal events; + credential-shape tests prove both independent UUID sources contribute to an + opaque 256-bit digest representation; real child-process sentinel tests prove + database, Turso, webhook, cloud, deployment-root, unrelated-provider, and + arbitrary tenant secrets are absent while the selected tenant provider key, + explicit config paths, runtime `PATH`, and invocation bearer remain; +- the isolated verifier child receives no parent environment and is killed when + its execution budget expires; +- evolution list/get/update/chain/stream operations cannot observe another + tenant even when record identifiers collide; +- failed policy or decision persistence leaves runtime authority unchanged, and + failed activation restores the prior durable policy and decision; and +- raw Blob authorization, storage-cap rejection, and staging-budget rejection + do not poll the body; wrong digests, short/long streams, cancellation, and + object-store failures leave no entity or staging file; concurrent uploads + cannot exceed the actual staged-byte budget; a stalled or trickled upload loses + all admission reservations at its progress/deadline bound, and another tenant + can upload while it is in flight; +- collection and content-addressed creates overwrite all trusted Cedar aliases + from server state, reject conflicting `id`/`Id` and lifecycle values, and + never persist caller-supplied context-status or governance attributes; + action events, direct PATCH/PUT field updates, initial state, and restored + snapshots likewise retain only server-derived, mutually equal + `id`/`Id`/`status`/`Status` aliases, and specs declaring those reserved names + fail validation; +- generic entity/list/expand/Observe reads never inline more than the aggregate + hydration budget, a lying overflow size cannot cause an oversized read, and + WASM deferred caches remain below their hard aggregate budget; +- both Blob property `$value` endpoints reject unauthenticated/unauthorized + callers and stream multi-chunk decoded bytes without hydrating the entity; +- Genesis admits dependency closure, tree traversal, and output bytes against + aggregate budgets before materialization; it writes large Blob content + directly to a temporary file with bounded memory and removes the temporary + output on malformed or failed input; + and +- legitimate tenant credentials, workflow correlation, and public health checks + continue to work. + +## DST Compliance + +The identity resolver retains no positive authority state and uses injected +simulation time for expiry checks. Strict durable replay consumes events in +sequence order under the existing replay budget. The internal single-use +capability store uses `BTreeMap`, bounded capacity, injected time, and an injected +token source. Simulation never calls OS randomness or wall-clock time. Local +dispatch continues through the production OData authorization path with a typed +context rather than a parallel test implementation. Adapter credential expiry +also uses `sim_now`; the detached task and wall-clock timeout wrap external CLI +side effects only, outside simulated transition semantics. Revocation still +uses the production entity actor and event journal path. + +## Rejected Alternatives + +1. **Strip headers only.** In-process and future direct-embed paths do not + necessarily cross that strip layer. +2. **A trusted header marker.** It merely creates a second forgeable authority + input and requires perfect middleware ordering. +3. **Treat no-key mode as anonymous Customer.** Shipped policies give anonymous + principals useful capabilities, so this is not fail-closed. +4. **Continue passing the root API key to WASM.** Tenant checks after a + deployment-wide Admin credential cannot recover least privilege. +5. **Sign forwarded identity headers with one long-lived HMAC key.** This still + serializes authority into replayable bearer data and duplicates credential + lifecycle logic; bounded single-use capabilities are narrower and auditable. +6. **Keep full hydration but rely on the 2 GiB object limit.** One object can + expand into several body-sized allocations, and a list or `$expand` can + multiply that again. An aggregate response budget must precede storage I/O. +7. **Make only the upload idle timeout finite.** A one-byte trickle can satisfy + an idle timer forever. Idle, total, and sustained-throughput bounds are all + required, together with per-tenant fair-share admission. +8. **Keep a short identity TTL plus mutation-path invalidation.** Any positive + TTL extends authority after a missed mutation. HTTP middleware cannot observe + internal actions, generic entity mutations, or writes performed by another + replica, so enumerating invalidation paths is not an authority boundary. +9. **Let adapter credentials inherit the process key or remain permanent.** An + inherited root credential destroys tenant least privilege; a permanent child + credential turns one invocation into durable authority. Explicit environment + removal, bounded minting, durable revocation, and expiry are all required. diff --git a/docs/adrs/0157-journaled-field-updates.md b/docs/adrs/0157-journaled-field-updates.md new file mode 100644 index 000000000..c4783076c --- /dev/null +++ b/docs/adrs/0157-journaled-field-updates.md @@ -0,0 +1,156 @@ +# ADR-0157: Journaled PATCH/PUT Field Updates + +## Status + +Accepted (2026-07-12) + +## Context + +OData PATCH and PUT reach the entity actor as `EntityMsg::UpdateFields`, which +mutated `state.fields` in memory and replied success without appending anything +to the event journal (ARN-189). Entity state is rebuilt exclusively from the +journal (snapshot + event replay) on actor eviction, server restart, and query +projection backfill — so every PATCH/PUT was silently lost the moment any of +those ran. The adjacent `Delete` handler already journals fail-closed +(persist first, mutate on success), making the gap an inconsistency rather +than a design choice. + +## Decision + +1. **Two new journal event types**, emitted by the `UpdateFields` handler: + `FieldsUpdated` (PATCH merge) and `FieldsReplaced` (PUT replacement), with + the update payload carried in `params` and `from_status == to_status`. + They live outside the spec's action vocabulary, like the existing + `Deleted` event. +2. **Fail-closed acknowledgment.** The handler applies the update, appends + the event (co-committing key/vector index rows derived from the NEW + fields, per ADR-0153/0155), and only then replies success. On append + failure the in-memory fields are rolled back and the reply is an error — + an update that is not durable is not acknowledged. +3. **One shared application function.** `apply_field_update` in + `entity_actor::effects` implements merge/replace semantics (PUT preserves + `Id` and `Status`) and is called by both the live handler and journal + replay, so a rehydrated entity reaches exactly the live post-update state. + Replay handles the two event types explicitly: the generic param-sync path + can only merge and would resurrect keys a PUT dropped. +4. **Duplicate events are acceptable; conflicting appends fail safe.** The + real duplicate path is a dispatch-layer ask timeout after a fully + successful handle: `ask_with_backoff` re-sends `UpdateFields`, and the + actor appends a second event. Both event types are idempotent in effect + (replaying the same merge or replacement twice converges), so duplicates + cost one journal row, never correctness. An actor-level retry after a + persisted-but-unacknowledged append cannot double-append: every store + enforces `expected_sequence`, so that retry hits a sequence conflict, which + is now recovered rather than surfaced (see Consequences). +5. **Field updates consume the event budget.** The handler enforces the same + `MAX_EVENTS_SINCE_SNAPSHOT` gate as spec actions, rejecting before + mutating. Without it, sustained PATCH traffic while the snapshot path is + stalled (queue full, stalled writer, save errors — all soft failures) + would grow the snapshot replay tail past the budget and make the entity + permanently unhydratable. + +## Consequences + +- PATCH/PUT survive actor eviction, server restart, and projection backfill; + the backfill previously rebuilt projections without the patched fields even + when the live projection had them. +- Entities without configured persistence keep the previous in-memory-only + behavior (the append is skipped, as in every other handler). +- `FieldsUpdated`/`FieldsReplaced` are reserved action names, and the reservation + is **enforced**, not conventional: replay dispatches those names to + `apply_field_update` before the generic action path, so a spec action of the + same name would be hijacked on rehydration — its params merged into fields, its + transition never replayed. The `Action` arm refuses both names. +- Journals written by older builds simply lack the new events; replay of old + journals is unchanged. +- A sequence conflict on the append (concurrent writer, or a crashed ack whose + append landed) is **recovered**, not surfaced: the arm rolls back the + speculative merge, replays to the authoritative sequence, re-applies onto the + caught-up state, and retries, for the same 1 + 2 attempt budget the `Action` + arm uses under ADR-0046. Without it a single conflict wedged every later update + until the actor happened to rehydrate. The refusals checked before the first + attempt — deletion and the event budget — are rechecked after each replay, + because the race may have deleted the entity or spent the budget. Beyond the + retry budget the arm fails closed and rolls back to the caught-up state. + + Two properties of that loop are load-bearing and easy to lose: + - **Catching up rebuilds from a fresh initial state**, as + `recover_entity_state_from_store` does. `replay_events` applies onto whatever + state it is handed and never resets it, so replaying onto the live state + re-applies every event on top of its own effects — the events deque grows, + `total_event_count` / `events_since_snapshot` climb, and non-idempotent + effects (counter increments) fire twice. That corruption would be returned to + the caller, upserted into the query projection, and made durable by the next + snapshot. + - **A conflict under a live `expected_precondition` is refused, not retried.** + That precondition is a compare-and-set: the caller authorized this write + against one exact digest. A conflict proves the journal held state the actor's + memory did not, so replaying and committing would apply the write to state the + caller never saw and Cedar never evaluated. `entity_ops` already caps + preconditioned asks at a single attempt for the same reason. + +- A field-update event whose payload no longer deserializes is skipped under a + lenient replay policy (so hydration survives spec evolution) and **fails** under + a strict one, matching the tombstone and generic-event arms. Strict replay backs + authoritative state resolution, where identity and authority are read: silently + dropping a `FieldsReplaced` there could preserve exactly the authority it was + written to revoke. Lenient skips are counted + (`temper_entity_field_update_replay_skipped_total`). + +- **Commit ambiguity is not resolved.** If the store commits the append but returns + a generic error (a client timeout after commit), the arm rolls back and reports + failure while the journal holds the update. A quiet entity then serves + pre-update fields from memory until it next rehydrates, at which point the + "failed" write appears. Inherent to a non-idempotent append over an unreliable + channel; called out so it is not discovered as a surprise. + +- **A rolling deploy can transiently mis-replay a PUT.** An older build replaying a + `FieldsReplaced` event has no arm for it, so it falls to the generic param-sync + path, which only merges — keys the PUT dropped reappear until a new-build actor + hydrates the entity. + +- **Reserved names are enforced at invocation, not at deployment.** The `Action` + arm refuses `FieldsUpdated` / `FieldsReplaced`, which stops new collisions. It + does not help a tenant whose spec already declared such an action and whose + journal already holds those events; separating them needs a discriminator on the + event envelope, which is a migration, not a guard. +- ADR numbering: 0156 is used by the concurrently open ARN-179 change + (`docs/adrs/0156-pg-actor-runtime-effect-vocabulary.md` on PR #370). + +## Invariants this arm must keep + +Stated as invariants rather than as a changelog, because each one was violated by +at least one implementation of this fix and none of them is obvious from reading +the happy path. + +1. **A non-object payload never reaches state or the journal.** `parse_json_body_or_400` + accepts any valid JSON, so a `PUT` body of `[1,2,3]` arrives here. With + `replace` it would set `fields` to the array, `canonicalize_entity_fields` + could not restore `Id`/`Status` (no object to insert into), and the append + would co-commit zero key and zero vector rows — purging the entity's index. + Journaling is what makes that permanent, so the guard is a precondition of + this decision, not an extra. +2. **Live and replay run the same transformation.** `apply_field_update` is + shared, sanitizes runtime-owned fields, and canonicalizes identity/lifecycle. + Any step applied on only one of the two paths silently rewrites the entity at + the next rehydration. It returns whether it applied, so a caller cannot treat + a declined update as a successful one. +3. **The event is sanitized before it is written**, not only on the way into + state, so the journal never records a second claimed truth for identity or + lifecycle. +4. **Catching up after a conflict rebuilds from a fresh initial state.** + `replay_events` applies onto whatever state it is given and never resets it. +5. **A conflict under a live `expected_precondition` is refused, not retried.** +6. **Every refusal checked before the first attempt is rechecked after a replay** — + deletion and the event budget — because the race may have caused either. +7. **A dropped update is counted, never silent** — malformed payload or + non-object payload, under a lenient policy; under a strict policy it fails. + +## Alternatives Considered + +- **Journal a synthetic spec action.** Would push PATCH/PUT through guard + evaluation and effect derivation that field updates deliberately bypass, + and would collide with real spec vocabularies. +- **Snapshot-only durability.** Snapshots are throttled (`maybe_save_snapshot`) + and best-effort; relying on them would leave a loss window and break the + journal-as-source-of-truth invariant that replay and backfill assume. diff --git a/docs/adrs/0158-repl-host-op-isolation.md b/docs/adrs/0158-repl-host-op-isolation.md new file mode 100644 index 000000000..bd52e1515 --- /dev/null +++ b/docs/adrs/0158-repl-host-op-isolation.md @@ -0,0 +1,69 @@ +# ADR-0158: REPL host-operation isolation + +**Status:** Accepted +**Date:** 2026-08-14 +**Issue:** ARN-166 (kernel `/api/repl` unauthenticated → arbitrary host file-read + RCE) + +## Context + +`temper.*` methods are dispatched from Python REPL code through +`temper_sandbox::dispatch::dispatch_temper_method`. Two of those methods, +`upload_wasm` and `compile_wasm`, are **host-process operations**: `upload_wasm` +reads a WASM file from the local filesystem, and `compile_wasm` writes a crate +to disk and spawns `cargo build`. + +The same dispatch path runs in two very different host processes: + +1. **Local stdio MCP server** — runs on the *developer's own machine*. Reading a + local file and running `cargo` there is exactly what the developer intends. +2. **Server-hosted REPL** (`POST /api/repl`) — runs *inside the Temper server + process*. There, `upload_wasm` reads the **server's** filesystem and + `compile_wasm` runs `cargo` **as the server user**. + +ARN-170 (merged) already made `/api/repl` require an authenticated credential and +a Cedar `execute_repl` permit. But authentication alone does not close ARN-166: +an *authorized* caller could still invoke `compile_wasm` and achieve arbitrary +file read + code execution as the server user. Authorization answers "may this +principal use the REPL"; it does not answer "may the REPL reach the host." + +## Decision + +Make host reachability an explicit capability of the dispatch context, not an +ambient property of the method. + +- `DispatchContext` gains `allow_host_ops: bool`. +- `dispatch_temper_method` rejects `upload_wasm`/`compile_wasm` **before touching + the filesystem** when `allow_host_ops` is false, with a message that names + where those ops belong (the local MCP server). +- The server-hosted REPL sets `allow_host_ops = false` (via `ReplConfig`). +- The local stdio MCP server sets `allow_host_ops = true`. + +The gate is the capability flag, not a hardcoded block: with the flag set, the +same two methods proceed normally — verified by a test that exercises both arms. + +## Consequences + +- The RCE/file-read vector on the server-hosted REPL is closed at the dispatch + boundary, independent of authorization. Even a fully authorized `execute_repl` + caller cannot drive host ops in the server process. +- The denial is surfaced to the operator channel: the gate emits a + `tracing::warn!` (`target: temper.repl.host_op`, with method/tenant/agent) before + returning the error. The Monty runtime collapses a dispatch error into a `null` + program result, so the HTTP caller sees no error field — a broader observability + gap tracked separately — but the refusal is now visible in logs/Datadog rather + than indistinguishable from a no-op. +- All other `temper.*` methods (entity CRUD, specs, governance, evolution, + non-host WASM registry reads) remain available in the server REPL — the fix + removes only the two host-process sinks, not the REPL's usefulness. +- Local developer workflows (`temper.upload_wasm`, `temper.compile_wasm` via the + MCP server) are unchanged. + +## Alternatives considered + +- **Rely on the ARN-170 auth gate alone.** Rejected: authorization is not + isolation; an authorized principal would still get RCE as the server user. +- **Remove `upload_wasm`/`compile_wasm` from dispatch entirely.** Rejected: + removes a legitimate local developer capability; the two contexts have + genuinely different trust, so the capability, not the method, is what varies. +- **Sandbox the server-side `cargo` invocation.** Larger surface, still leaves + local file reads; the capability gate is the smaller, complete fix. diff --git a/docs/adrs/0159-wasmtime-36-security-update.md b/docs/adrs/0159-wasmtime-36-security-update.md new file mode 100644 index 000000000..44912dd2e --- /dev/null +++ b/docs/adrs/0159-wasmtime-36-security-update.md @@ -0,0 +1,78 @@ +# ADR-0159: Update Wasmtime 29 → 36 (RUSTSEC-2026-0096) + +- Status: Accepted +- Date: 2026-08-14 +- Issue: ARN-169 +- Related: `crates/temper-wasm` (the only crate that depends on Wasmtime) + +## Context + +The workspace pinned `wasmtime = "29"` / `wasmtime-wasi = "29"`. Wasmtime 29 is +affected by **RUSTSEC-2026-0096** (CVSS 9.0). Temper compiles and runs untrusted +guest WASM modules (`temper-wasm`), so a vulnerability in the WASM runtime is +directly reachable by guest input and must be closed. + +## Decision + +Update to `wasmtime = "36.0.13"` / `wasmtime-wasi = "36.0.13"` — a maintained 36.x +train that carries the fix (36.0.7 patches RUSTSEC-2026-0096), chosen over the +newest (46.x) to minimize API churn. The floor is `36.0.13` specifically, not +`36.0.12`, because 36.0.13 also fixes RUSTSEC-2026-0222, so an earlier 36.x patch +must not be admitted. `cargo audit` confirms zero wasmtime advisories on the +resolved graph, and RUSTSEC-2026-0096 present on `main` / gone on this branch. + +The one import the API move requires: +`wasmtime_wasi::pipe::MemoryOutputPipe` → `wasmtime_wasi::p2::pipe::MemoryOutputPipe`. + +**Pin the WASM feature surface so the bump does not widen the sandbox.** wasmtime +30+ turns several proposals on by default that 29 rejected; inheriting the 36 +defaults would silently expand what a guest can do relative to the reviewed 29 +surface. The engine now sets these explicitly: +- `wasm_memory64(false)` — 29 rejected 64-bit memories at compile, and memory64 is + a prerequisite for RUSTSEC-2026-0096; keep it rejected. +- `wasm_threads(false)` — rejects shared memories, which a guest could otherwise + grow outside the per-memory `max_memory` limiter. +- `wasm_multi_memory(false)` — the `max_memory` limiter caps each memory + individually; a single memory per module keeps the per-invocation budget meaningful. +- `MemoryLimiter` now bounds table and memory host allocation on two axes: + `table_growing` denies growth past `MAX_TABLE_ELEMENTS` (1,000,000) per table, + and `tables()`/`memories()` cap the store's table/memory *counts* (`MAX_TABLES` + = 8, `MAX_MEMORIES` = 1). wasmtime's default limiter allows 10,000 of each, so + the per-table element cap alone was not a store-wide budget — a guest could + declare thousands of tables at the cap. Together these bound total table host + memory to `MAX_TABLES * MAX_TABLE_ELEMENTS`. + +Temper's guests are single-memory wasm32 modules and use none of the disabled +features; the `temper-wasm` engine suite passes with the surface pinned. + +## Consequences + +- RUSTSEC-2026-0096 (and RUSTSEC-2026-0222, via the 36.0.13 floor) is closed for + the guest-WASM execution path. +- The reviewed 29 sandbox surface is *preserved*, not merely inherited: memory64, + shared memory, and multiple memories are rejected; the linear-memory limiter and + the new table cap deny growth past budget; fuel/epoch timeouts fire; traps stay + isolated. Regression tests cover each: `memory_growth_denied_by_limiter` (now + non-vacuous — the guest traps if the grow unexpectedly succeeds), + `initial_memory_over_budget_is_rejected`, `table_growth_denied_past_cap`, + `too_many_tables_is_rejected`, `memory64_module_is_rejected`, + `multi_memory_module_is_rejected`, `shared_memory_module_is_rejected`, and + `wasi_preview1_module_invokes_end_to_end`. + +## Follow-ups (pre-existing, tracked separately) + +- **WASI host-side allocation in preview1 host calls.** `random_get` (and peers) + allocate a host buffer sized by the guest-supplied length before the guest-memory + bounds check; the `ResourceLimiter` governs guest memory/tables, not host-side + allocations inside host functions, so a guest can force a large transient host + allocation. This predates the bump (present on 29) and its clean fix needs either + a wasmtime train with the embedder allocation controls (42+) or a custom WASI + host wrapper — a separate hardening effort, not this security patch. + +## Alternatives considered + +- **Jump straight to 46.x (latest).** Rejected for now: a larger API delta for no + additional security benefit over 36.x for this advisory. A later routine bump can + move further once the surface is re-reviewed. +- **Backport a patch onto 29.** Not offered upstream; the fix ships in the newer + release trains. diff --git a/docs/adrs/0160-clear-rustsec-advisories.md b/docs/adrs/0160-clear-rustsec-advisories.md new file mode 100644 index 000000000..df06cddf7 --- /dev/null +++ b/docs/adrs/0160-clear-rustsec-advisories.md @@ -0,0 +1,119 @@ +# ADR-0160: Clear reachable RUSTSEC advisories in transitive deps (ARN-169) + +- Status: Accepted +- Date: 2026-08-15 +- Issue: ARN-169 (follow-up to the wasmtime bump in ADR-0159) + +## Context + +`cargo audit` on `main` reported 18 advisories. ADR-0159 closed the critical +wasmtime one. This pass clears the remaining advisories that are reachable without +a cascading parent upgrade. Cleared here (crate → advisories): + +- `postgres-protocol` 0.6.11 → 0.6.12 — RUSTSEC-2026-0179, -0180 +- `tokio-postgres` 0.7.17 → 0.7.18 — RUSTSEC-2026-0178 (DoS panic) +- `quinn-proto` 0.11.14 → 0.11.16 — RUSTSEC-2026-0185 +- `quick-xml` 0.37.5 → 0.41 — RUSTSEC-2026-0194, -0195 (DoS) +- `rustls-webpki` (0.103 line) 0.103.9 → 0.103.13 — RUSTSEC-2026-0049, -0098, -0099, -0104 +- `crossbeam-epoch` 0.9.18 → 0.9.20 — RUSTSEC-2026-0204 +- `protobuf` 2.28.0 — RUSTSEC-2024-0437, removed from the tree entirely (see below) + +## Decision + +Most are plain `cargo update` (same major, no source change). Two entries need +source changes: the `quick-xml` bump below, and the `protobuf` removal described in +Consequences (a `pprof` feature swap plus its one-line serializer call). + +The first is `quick-xml` 0.37 → 0.41, which is pinned by our own `temper-spec` +(0.41.0 is the *minimum* version patching RUSTSEC-2026-0194/-0195, so the API break +was unavoidable — no smaller bump exists): + +- 0.41's `Reader::read_text` returns a `Result` instead of an owned + `String`. The Collection annotation path (`csdl/parser/elements.rs`) + now calls `.decode()` on it. **Behavior is preserved exactly**: 0.37's + `read_text` also only charset-decoded and did *not* unescape XML entities (its + rustdoc: "does not unescape read data"), so `A & B` stays literal under + both versions, consistent with the inline-attribute path (`attr_str`). A + regression test (`test_parse_collection_annotation_preserved_across_quick_xml_bump`) + locks the preserved behavior in. +- All other `temper-spec` quick-xml uses (`read_event_into`, `element.name()`, + raw attribute bytes) are version-stable. + +Not changed here (deliberately, to keep this a behavior-neutral dep bump): the CSDL +parser does not unescape XML entities in annotation text — neither before nor after +this bump — and the parse→emit path double-escapes such values. Adding proper +entity handling is a separate, pre-existing correctness fix that needs its own +sign-off, out of scope for an advisory-clearance bump. + +One behavior difference exists, and only for **ill-formed** XML: 0.41 is stricter +about a raw, un-escaped `&` in element text (its text scan is `memchr2(b'<', b'&')` +where 0.37's was `memchr(b'<')`), so such an annotation item is dropped where 0.37 +kept it verbatim. Well-formed CSDL is unaffected (our emitter always escapes, and no +fixture contains a raw `&`), and the surrounding document still parses. Stricter +rejection of invalid XML is the desirable direction. + +That the item is *silently* dropped rather than raising `CsdlParseError` is a +pre-existing shape of this code that the bump makes reachable; it is tracked as +**ARN-347** rather than fixed here, so this stays a dependency bump. + +## Consequences + +- `cargo audit` drops from 18 to 6 (12 advisory instances cleared). temper-spec's + suite passes and the workspace builds clean; a server built on the bumped stack + boots and parses seeded CSDL through the new quick-xml path without error. +- Who is actually affected, so a future reader audits the right crate: + - `postgres-protocol` / `tokio-postgres` reach us through `deadpool-postgres` + (`temper-actor-runtime`, `temper-cli`, `temper-agents`) — **not** + `temper-store-postgres`, which uses `sqlx`. 0.6.12 rejects SCRAM iteration + counts above 100_000 (Postgres defaults to 4096) and turns hstore / short + `DataRow` panics into errors. + - the `rustls-webpki` 0.103 line serves `reqwest` / `bollard` / `testcontainers` / + `quinn`. It is **not** the Turso TLS path — that still resolves 0.102.8 via + `libsql`, which is why the residual below is a real cascade rather than an + oversight. + - `quick-xml` affects only the CSDL parser in `temper-spec`. +- **`protobuf` needed no upgrade at all.** It was in the tree only because + `temper-server` asked `pprof` for its `protobuf-codec` feature, which pins + protobuf 2.x — a line with no fix for RUSTSEC-2024-0437. The same `pprof` version + offers `prost-codec`, so switching the feature (and `Profile::write_to_vec` → + prost's `Message::encode`) drops the vulnerable crate outright. Both codecs emit + the same pprof protobuf wire format, so uploaded profiles are unchanged. +- The lockfile also moves crates beyond the upgrades above, which is expected resolver + dedup rather than part of the security change: `hyper-util` / `quinn` / + `quinn-udp` re-point from `socket2` 0.6.3 to 0.5.10, and several dependents from + `windows-sys` 0.61.2 down to already-present 0.60/0.59/0.52/0.48 lines. Both + `socket2` lines already coexisted on `main`, every requirement is a range that + admits 0.5.10 (`hyper-util` wants `>=0.5.9, <0.7`; `quinn`/`quinn-udp` the broader + `>=0.5, <0.7`), no advisory covers the selected versions, and builds, clippy + and tests are clean. Called out explicitly because a networking-crate *downgrade* + inside a security PR is exactly the kind of thing a later audit should not have to + re-derive. The `prost-codec` swap likewise adds five lock entries — `prost-build` + and `prost-types` 0.12.6, `petgraph` 0.6.5, `fixedbitset` 0.4.2, `multimap` + 0.10.1 — all **build-time only** via pprof's `[build-dependencies]`; `prost` + itself was already in the tree via `libsql-hrana`, so no new runtime dependency + is introduced. + +## Residual (out of scope here, tracked on ARN-169) + +After this pass `cargo audit` reports 6. Each remaining finding needs a cascading +parent upgrade, has no fixed release, or would cost a capability we actually use: + +- `rustls-webpki` 0.102.8 (RUSTSEC-2026-0049/-0098/-0099/-0104) — pulled by + `libsql` 0.9.x → `hyper-rustls` 0.25 → `rustls` 0.22. Two routes exist and both + are rejected deliberately, not by omission: + - *Upgrade:* needs `libsql` on `rustls` 0.23 — a cascade into the Turso storage + client with real breakage risk. + - *Feature toggle:* `hyper-rustls` is optional in libsql behind its default-on + `tls` feature, so this is structurally the same shape as the `protobuf-codec` + swap above. It is rejected because `temper-store-turso` calls + `Builder::new_remote` (`store/mod.rs`, the `libsql://` branch of the production + constructor), and with `tls` off libsql's `connector()` is an outright + `panic!("The \`tls\` feature is disabled, ...")` — so dropping it breaks remote + Turso at runtime, not just in theory. Removing a working capability to clear an + advisory needs sign-off, so it stays residual. +- `rsa` 0.9.10 (RUSTSEC-2023-0071, Marvin timing) and `tokio-tar` 0.3.1 + (RUSTSEC-2025-0111) — no fixed release available; accept/monitor. +- **temperpaw** carries the same dependency set and needs the equivalent pass in + its own repo. + +Consider adding `cargo audit` to CI so new advisories surface automatically. diff --git a/docs/adrs/0164-wasm-guest-read-bounds-before-alloc.md b/docs/adrs/0164-wasm-guest-read-bounds-before-alloc.md new file mode 100644 index 000000000..b27a93d51 --- /dev/null +++ b/docs/adrs/0164-wasm-guest-read-bounds-before-alloc.md @@ -0,0 +1,126 @@ +# ADR-0164: Bounds-check guest reads before allocating + +- Status: Accepted +- Date: 2026-07-12 +- Deciders: Temper core maintainers +- Related: + - `crates/temper-wasm/src/engine/host_functions.rs` (guest memory reads) + - `crates/temper-wasm/src/engine/mod.rs` (post-invocation result read) + - ARN-226 (security finding) + +## Context + +WASM host functions read caller-supplied `(ptr, len)` operands out of the guest's +linear memory. The read helpers allocate the destination buffer **before** the +bounds check runs (`host_functions.rs`): + +```rust +fn read_guest_bytes(caller, memory, ptr, len, …) -> Result, ()> { + if ptr < 0 || len < 0 { … return Err(()); } + let mut buf = vec![0u8; len as usize]; // allocation happens first + memory.read(caller, ptr as usize, &mut buf)?; // bounds check happens here + … +} +``` + +`len` is an `i32` chosen by the guest, so a single call with `len = i32::MAX` +forces the host to allocate ~2 GiB **before** `memory.read` rejects the +out-of-bounds range. A guest can repeat this cheaply, so it is an unauthenticated +host-side memory-exhaustion / DoS: the allocation is driven entirely by an +attacker-controlled length that is never validated against the guest's actual +memory size. `read_guest_string` has the same shape. + +## Decision + +Validate that the `[ptr, ptr + len)` range lies within the guest's current linear +memory **before** allocating. `guest_read_bounds_ok(mem_size, ptr, len)` returns +true only when `ptr + len` does not overflow `usize` and is `<= mem_size` +(`memory.data_size(store)`). The read helpers call it first and return the ABI +error sentinel on failure, so no buffer is allocated for an out-of-bounds length. + +Each allocation is therefore bounded by the guest's linear memory size, which is +itself capped by `WasmResourceLimits::max_memory`: an oversized `len` is rejected +before any allocation, so no single read can allocate more than the guest could +legitimately hold. + +This is a **per-allocation** bound, not a per-invocation one. A host function that +performs several reads (`host_evaluate_spec` holds four) still peaks at a multiple +of the guest's memory, and the outbound stream channel is bounded in chunks rather +than bytes. An aggregate copy budget is the right next tightening and is tracked +separately (see Follow-ups); it is out of scope here because it is a functionality +cap, not a fix for the disclosed vector. + +The guard is applied at every guest-length-driven allocation, not only the two +helpers: + +- `read_guest_string` / `read_guest_bytes` check internally, and the ten previously + inline `vec![0u8; len]` sites (the `host_emit_*` family, `host_read_field`, + `host_evaluate_spec`, `host_http_stream_send_response_head`) are refactored onto + them. Four of those had no negative-length check at all, so `len = -1` requested a + `usize::MAX`-sized allocation. +- The **post-invocation result read** (`engine/mod.rs`) is a separate vector: the + guest returns a pointer whose preceding four bytes are a `u32` length prefix, so a + forged prefix could drive a ~4 GiB allocation. It calls `guest_read_bounds_ok` + directly before allocating and fails with a distinct + `"result length exceeds guest linear memory"`. + +Tests cover the predicate *and* the wiring, because a returned error sentinel alone +cannot distinguish "rejected before allocating" from "allocated, then failed": + +- a unit test pins the predicate's boundaries (`end == mem_size` allowed; overflow + and out-of-range rejected); +- two end-to-end tests drive a guest calling a helper-backed host function with a + 64 MiB length against a 64 KiB page — one per helper (`host_emit_progress` for + `read_guest_string`, `host_cache_contains` for `read_guest_bytes`) — and a counting + allocator in the test binary asserts **no allocation at or above 32 MiB happened**. + This is the only way to catch a deleted helper guard: the guest sees the same + result either way (`-1`, and for the cache ABI a plain `0`), so only the + allocation itself distinguishes "refused" from "allocated, then failed"; +- a second end-to-end test returns a forged 64 MiB result-length prefix and asserts + both the distinct error and the absence of a large allocation, so moving the + allocation above the guard is caught as well; +- a third returns a length that is small on its own but whose `ptr + len` runs past + the end of memory, pinning the check to the whole range rather than the length. + +Each of those five mutations — deleting either helper guard, deleting the result +guard, moving the result allocation above its guard, and weakening the check to +`len` alone — was applied and confirmed to turn the suite red. + +A sixth test guards the *class* rather than the instances: it asserts that guest +memory is read in exactly two places in `host_functions.rs` (inside the two +bounds-checked helpers). The original vulnerability existed because the raw +allocate-then-read shape had been copied to ten call sites; this fails if an +eleventh appears, instead of waiting for someone to notice. + +## Consequences + +### Positive +- A guest-supplied `len` can no longer force a host allocation larger than the + guest's own (already bounded) memory. Host-function reads fail fast with the same + ABI error sentinel the out-of-bounds `memory.read` would have produced, minus the + allocation; the post-invocation result path deliberately reports a distinct + `"result length exceeds guest linear memory"` so its guard is observable in tests. + +### Behavior +- Legitimate in-bounds reads are unchanged (the extra check is a single + comparison). Out-of-bounds reads already failed; they now fail before rather + than after allocating. + +### DST Compliance +- Pure integer arithmetic (`checked_add`, comparison); no wall clock, no threads, + no `HashMap`, no ambient I/O. Deterministic. + +## Non-Goals / Follow-ups +- **Aggregate host-copy budget — tracked as ARN-348.** This ADR bounds each + allocation by the guest's own memory; it does not bound their *sum*. In-bounds + paths still amplify: `host_evaluate_spec` holds four reads at once, the outbound + stream channel is bounded in chunks rather than bytes (so large in-bounds writes + can pin far more than the intended ~1 MiB), and `max_response_bytes` is still not + enforced on the HTTP body path. Fuel and `max_duration` bound execution, not + resident host bytes, so they are not substitutes. That work is a functionality + cap as much as a security one and needs its own sign-off. + +## Alternatives Considered +1. **Clamp `len` to a fixed constant.** Rejected: a fixed cap either breaks + legitimate large reads or is still larger than needed; bounding by the guest's + own memory size is exact and self-adjusting. diff --git a/docs/adrs/0165-mcp-trajectory-tenant-and-bounds.md b/docs/adrs/0165-mcp-trajectory-tenant-and-bounds.md new file mode 100644 index 000000000..59e2a8bfd --- /dev/null +++ b/docs/adrs/0165-mcp-trajectory-tenant-and-bounds.md @@ -0,0 +1,125 @@ +# ADR-0165: MCP trajectory is owned by the session identity and bounded + +- Status: Accepted +- Date: 2026-08-15 +- Deciders: Temper core maintainers +- Related: + - `crates/temper-mcp/src/runtime.rs` (MCP trajectory capture + upload) + - ARN-222 (security finding) + +> This is the landed version for ARN-222. It is based on Fable's arena entry +> (winner of the head-to-head), with one addition ported from the competing entry +> (#365): a bound on stdio frame size. #365 raised that gap but bounded it only +> *after* reading the whole line; this version bounds the allocation during the +> read (Sub-Decision 3). + +## Context + +The MCP client captures an OTS trajectory of each session's `execute` turns and +uploads it to the server (`runtime.rs`). Two problems (ARN-222): + +1. **Tenant mixing.** The trajectory is uploaded with + `X-Tenant-Id: self.primary_tenant()`, and `primary_tenant()` returns the + **most-referenced tenant in the executed code** (`tenants_seen`, populated from + `extract_temper_call_metadata(code)`), falling back to the session identity only + when no tenant appears in the code. The executed code is attacker-controlled, so + a session authenticated as tenant A can inject `temper` calls referencing tenant + B and cause its trajectory — containing A's session code and results — to be + filed under **tenant B**. Trajectory storage is thus keyed by code content rather + than by the authenticated session identity, mixing tenants. + +2. **Unbounded code/results.** `record_execute_turn` records the full submitted + `code` and the full execution `result` verbatim (`OTSMessageContent::text`) with + no size cap, across an unbounded number of turns. A large or runaway session + accumulates the whole thing in memory and uploads it, an unauthenticated + memory/storage-exhaustion vector. + +## Decision + +### Sub-Decision 1: The trajectory belongs to the authenticated identity + +The trajectory upload is keyed by `self.identity_tenant` — the session's +authenticated tenant — not by any code-derived tenant. Code content can never move +a trajectory into another tenant's storage. `tenants_seen` is retained only as an +observability signal: if the session's code referenced a tenant other than the +identity, that is logged (a cross-tenant-activity signal), but it does not +determine storage. + +### Sub-Decision 2: Bounded capture + +Every guest-controlled channel that feeds the trajectory is bounded, so total size +is bounded regardless of session input: +- recorded `code` / `result` text → truncated to `MAX_TRAJECTORY_TEXT_BYTES` (on a + UTF-8 char boundary, marked); +- the decision's `error_type` → the same truncated text (not the raw error); +- the embedded `trajectory_actions` → capped in count (`MAX_TRAJECTORY_ACTIONS`) and + collapsed to a summary when serialized size exceeds the text budget; +- the number of recorded turns → capped (`MAX_TRAJECTORY_TURNS`), further turns + dropped with a warning; +- the **total** recorded text across the trajectory → capped at + `MAX_TRAJECTORY_TOTAL_BYTES` (1.8 MB of metered serialized cost), kept under the + server's 2 MiB ingest limit so a + session that is within the per-turn and turn-count caps still cannot produce a + trajectory the server rejects with 413 (which the client treats as non-retryable + and would silently drop, suppressing the audit trail); +- the per-session `tenants_seen` map → capped in both distinct keys + (`MAX_SEEN_KEYS`) **and** per-key byte length (`MAX_SEEN_KEY_BYTES`); an oversized + key is dropped rather than retained, so 256 near-1-MiB keys cannot retain hundreds + of MiB. The turn/byte budgets reset on re-`initialize`. + +No code-derived value is placed in a request header. The previous, code-derived +`X-Entity-Type` header (which had no server-side reader) is removed: an illegal byte +such as `\n` in an attacker-controlled entity type would otherwise make the HTTP +client reject the whole upload and silently lose the trajectory. Only the session's +authenticated tenant and startup-config agent/session ids remain as headers. + +The bounds and stdio framing live in `trajectory_bounds.rs` so they (and their +tests) are auditable in one place. + +### Sub-Decision 3: Bounded stdio frames + +`run_stdio_server` read JSON-RPC frames with `BufReader::lines()`, which buffers a +whole line into one allocation — a peer that never sends a newline could exhaust +memory before any parse. Frames are now read through `read_stdio_frame`, which caps +each frame at `MAX_STDIO_LINE_BYTES` (1 MiB) **during** the read: it never +allocates more than the budget plus one byte, drains an oversized frame to the next +newline in bounded chunks, drops it with a warning, and resynchronizes on the +following frame. Invalid UTF-8 frames are dropped rather than aborting the session. + +## Consequences + +### Positive +- A trajectory can only ever be stored under the session's authenticated tenant, so + code content cannot cross tenant boundaries. Capture size is bounded, closing the + memory/storage-exhaustion vector. + +### Behavior +- Legitimate sessions are unaffected: their identity tenant is where their + trajectory already belonged, and normal turn sizes are well under the caps. + +### DST Compliance +- `temper-mcp` is not simulation-visible. The new logic is pure (string truncation, + identity selection, a counter); no wall clock, threads, or ambient I/O added. + +## Non-Goals / Follow-ups +- Redaction of secrets that a guest may print into a result is a separate content + concern, tracked elsewhere. This ADR closes the tenant-attribution and unbounded + size vectors. +- **Server-side tenant binding is already in place.** The + `POST /api/ots/trajectories` handler keys storage on the typed + `AuthenticatedRequestContext::tenant()` (`observe/evolution/trajectories.rs`), not + the raw `X-Tenant-Id` header, and the bearer edge resolves the credential within + the requested tenant (ADR-0157/ARN-187). So the storage boundary is enforced on + both sides: this fix removes the client-side code-derived-tenant vector, and the + server independently ignores an untrusted tenant header. The only residual is a + bearer token that is valid in multiple tenants — orthogonal to ARN-222. +- Peak *transient* processing memory (an error string materialized by the sandbox + before truncation, or actions/metadata parsed from a frame before the caps apply) + is bounded by the 1 MiB stdio frame cap and the sandbox's own memory budget, then + truncated before retention. Retained trajectory size is what this ADR bounds; + reducing transient peaks further is not required to close the vector. + +## Alternatives Considered +1. **Reject the session when code references another tenant.** Rejected: legitimate + cross-tenant reads may be authorized server-side; the fix is to store under the + authenticated identity and log the cross-tenant signal, not to block execution. diff --git a/docs/adrs/0166-llm-content-export-tenant-redaction.md b/docs/adrs/0166-llm-content-export-tenant-redaction.md new file mode 100644 index 000000000..16dd13641 --- /dev/null +++ b/docs/adrs/0166-llm-content-export-tenant-redaction.md @@ -0,0 +1,267 @@ +# ADR-0166: Per-Tenant Redaction of LLM Observability Content + +- Status: Accepted +- Date: 2026-07-13 +- Deciders: Temper core maintainers +- Related: + - ADR-0037: `X-Temper-Span-*` header hints (host HTTP capture path) + - `crates/temper-server/src/state/dispatch/wasm.rs` (LLM dispatch recording) + - `crates/temper-wasm/src/host_trait/span_hints.rs` (host HTTP capture) + - `crates/temper-observe/src/wide_event/agent.rs` (wide-event builders) + +## Context + +When an entity integration is marked `llm = true`, the WASM dispatch path records +the model's **prompt, completion, and system instructions** onto the OpenTelemetry +span, into the WideEvent, and submits them to Datadog LLM Observability. A second, +independent path lets WASM modules (e.g. `llm_caller`) capture request/response +**content** onto the host `wasm.host.http_call` span via `X-Temper-Span-Attr-*` +and `X-Temper-Span-Capture-Response-*` headers (ADR-0037). + +Both paths export raw LLM content to the telemetry backend **unconditionally** — +there is no per-tenant control. A tenant whose prompts or completions contain +PII, secrets, or regulated data has no way to keep that content out of Datadog. +This is the ARN-243 finding: LLM observability leaks full prompts with no tenant +opt-out. + +The existing `strip_private_observability_params` helper removes `_gen_ai_*` +content keys **before persistence** (so they are not echoed back to the guest or +stored in entity state), but the telemetry sinks read those keys *before* that +strip runs — so the content still reaches Datadog. + +## Decision + +Gate all LLM content export on a per-tenant policy that **defaults to redact**. +Content is only exported for tenants that explicitly opt in. Metadata +(token counts, model, provider, finish reason, trace-linking IDs) is always +exported — only prompt/completion/system/tool content is redacted. + +### Sub-Decision 1: Per-tenant opt-in resolved on `ServerState` + +`ServerState` gains an immutable `llm_content_export_tenants: Arc>` +loaded once at startup from `TEMPER_LLM_CONTENT_EXPORT_TENANTS` (comma-separated +tenant ids; `*` opts in every tenant). The resolver is a pure set lookup: + +```rust +pub fn export_llm_content(&self, tenant: &str) -> bool { + self.llm_content_export_tenants.contains("*") + || self.llm_content_export_tenants.contains(tenant) +} +``` + +An empty set (the default) means every tenant is redacted. + +**Why this approach**: it is redact-by-default, per-tenant, deterministic (read +once at startup, then a pure lookup on the hot path), and requires no new storage +dependency or per-dispatch async. It mirrors the existing `local_tdata_hosts` +env-loaded allowlist. A future enhancement can move the opt-in into per-tenant +Cedar policy or the secrets vault without changing the redaction sites. + +### Sub-Decision 2: Dispatch path — strip content keys from callback params + +Four telemetry sinks (span record, `llm_call_wide_event`, `submit_llmobs_llm_span`, +`submit_llmobs_tool_spans`) all read content from the same `result.callback_params` +map. A single strip of the content keys — `_gen_ai_input_messages`, +`_gen_ai_output_messages`, `_gen_ai_system_instructions`, `_dd_llmobs_tool_spans` — +before the sinks run redacts all four at once. Metadata keys (`_gen_ai_provider`, +`_gen_ai_model`, `_gen_ai_finish_reason`, `_gen_ai_*_span_id`, token counts) are +preserved. + +**Why this approach**: one choke point covers span, WideEvent, and both LLM-Obs +submissions. It cannot be bypassed by adding a new sink downstream, because the +content is already gone from the map. + +### Sub-Decision 3: Host capture path — filter span hints before applying + +The host HTTP capture path builds a `SpanHints` struct from guest headers, then +applies its attributes and response captures onto the span. When the tenant is +not opted in, the hints are filtered immediately after parsing, before they are +recorded onto either the tracing span or the raw OTel span. The decision is +propagated into `ProductionWasmHost` via a new `with_llm_content_export(bool)` +builder that **defaults to `false` (redact)**, set from +`ServerState::export_llm_content` at every server-side host construction site. + +This filter is an **allowlist, not a denylist**, because the thing it gates is +the thing that names the data. Both halves of a span hint are guest-controlled: +the attribute name arrives as an `X-Temper-Span-Attr-*` header, and a response +capture is a `(name, json_pointer)` pair whose value is lifted out of the +provider's response body by that pointer. Enumerating the canonical `gen_ai.*` +content keys therefore stops only a module that uses the canonical names — a +module that sends `X-Temper-Span-Capture-Response-llm.response.text: +/content/0/text` exports the same completion under a name no denylist carries. + +So, for a tenant that has not opted in, one rule applies at every point an +untrusted guest names telemetry — span hints, guest spans, guest span events, +guest wide events, and guest metric tags: + +Every namespace test runs on a *normalized* key (`is_llm_namespace_key`: +trimmed, lowercased). Normalizing in the allowlist but matching raw in the clamp +is how the two end up disagreeing — `GEN_AI.request.model` passes as recognised +metadata and then misses the clamp, carrying a whole prompt. Adversarial review +found exactly that. + +- **Attributes inside the `gen_ai.*` namespace** survive only if the key is a + recognised metadata key (model, provider, token counts, finish reasons, + conversation/tool ids, temperature, max tokens), and their values are clamped + to 256 bytes. The clamp is what makes the allowlist mean anything: a key name + cannot turn an untrusted value into metadata, so without a bound a module sends + the whole prompt as `gen_ai.request.model`. Every legitimate metadata value is + far shorter than the bound. +- **Attributes outside the namespace** pass through unchanged. These channels are + the generic observability ABI, not LLM-only ones — see Scope below for why + redacting them would cost working capability without buying protection. +- **Response captures** are dropped in full. Every capture is by construction a + value read out of the response body via a guest-supplied pointer, so the + attribute name says nothing about whether the value is content. +- **Guest-supplied span names** — both `X-Temper-Span-Name` on the hint path and + the name passed to `host_start_span` — are clamped to the same 256 bytes, since + they are free text on the same channel. + +The decision reaches the guest-facing APIs through `WasmHost::exports_llm_content` +(default `false`), which the engine reads when it builds the guest span registry. + +Content attrs (exported only on opt-in): `gen_ai.input.messages`, +`gen_ai.prompt`, `gen_ai.system_instructions`, `gen_ai.output.messages`, +`gen_ai.completion`, `gen_ai.tool.call.arguments`, `gen_ai.tool.call.result`. + +The cost of the allowlist is that a new legitimate metadata attribute is redacted +for non-opted-in tenants until it is added to `is_llm_metadata_attr`. That is the +correct direction to fail: a missing dashboard field is recoverable, an exported +prompt is not. + +### Sub-Decision 4: Test the policy, not only the filter + +The filter is only as good as the predicate that opens it, so +`export_llm_content` is split into pure functions +(`parse_llm_content_export_tenants`, `tenant_exports_llm_content`) and tested +directly: unset/blank input opts in nobody, listed tenants match exactly (no +prefix, no case folding), and `*` opts in everyone. Each was mutation-checked — +default-allow, prefix matching, and keeping blank entries each turn the suite +red. Without these, a policy flipped to default-allow would leak every tenant's +prompts while every redaction test stayed green. + +## Consequences + +### Positive +- For a tenant that has not opted in, no LLM content reaches the telemetry + backend **under the `gen_ai.*` semantic-convention names** — the names LLM + Observability, the GenAI dashboards, and the wide-event pipeline actually read. + That holds across all five channels an untrusted guest can reach: the host HTTP + span-hint path, the callback params, the guest manual-span API + (`host_start_span` / `host_set_span_attributes` / `host_add_span_event`), + `host_emit_wide_event`, and `host_emit_metric` — whose tags are guest-named + *and* guest-valued strings that reach both the OTel meter and a span event. +- Redaction is fail-safe: `WasmHost::exports_llm_content` defaults to `false`, so + a host that never answers redacts, and the builder default matches. +- Metadata-based dashboards (tokens, latency, model, provider) keep working for + every tenant, and non-LLM guest diagnostics (`provider.request_id`, + `rpc.method`, application attributes) are untouched. + +### Measured reach of the span-hint gate + +A behavioral test (`crates/temper-wasm/tests/span_hint_redaction_behavior.rs`) +drives a real `http_call` and reads the span back out of an in-memory OTel +exporter, rather than trusting the code shape. It showed something the unit tests +could not: `apply_span_hints` sets non-Datadog-visible attributes only via +`otel_span.set_attribute` on a context-derived handle, which does not propagate +to the exported span. So on that channel, only names in +`datadog_visible_span_hint_field` reach the backend at all today. + +Two consequences, both stated rather than assumed. The gate's real reach on the +span-hint path is narrower than its code suggests — it can only matter for names +the export path keeps. And guest diagnostics outside that set are being silently +dropped, which is an observability bug (tracked as **ARN-350**), not a security +one. The redaction is written for the wider surface deliberately: ARN-350 will +widen what reaches the exporter, and the gate must already be correct when it +does. The behavioral test asserts today's narrower behavior explicitly so that +fixing ARN-350 forces a revisit here instead of quietly changing the surface. + +### Scope — what this does *not* claim + +Stated plainly, because an overstated gate is worse than a narrow one: this does +not make it impossible for a determined guest module to move bytes into +telemetry. A module already holds its own prompt, and several channels carry +free-form text that carries no agreed meaning to redact against: + +- `host_log` and `log_structured` attach the guest's message to span events and + tracing events verbatim; a module that logs its prompt exports it. +- Guest-supplied error strings reach `error.message` / `exception.message`. +- Attributes outside the `gen_ai.*` namespace — on span hints, guest spans, and + wide events — are the module's own application telemetry and pass through. + +Closing those would mean disabling guest logging and guest observability +outright, which removes working capability for every tenant to stop a guest from +exporting data it already owns. The line drawn here is: **the semantic-convention +namespace that downstream systems interpret as LLM content is governed; free-form +guest text is not.** Tracked as ARN-349 so the choice is revisited deliberately +rather than forgotten. + +### Negative +- Tenants currently relying on content in Datadog must be added to + `TEMPER_LLM_CONTENT_EXPORT_TENANTS` to keep it. This is the intended, + deliberate default flip. +- Response captures (`X-Temper-Span-Capture-Response-*`) are dropped entirely for + non-opted-in tenants, including one that points at a metadata field such as + token counts. This is a real, if narrow, capability change: a capture's value + comes from a guest-supplied JSON pointer into the provider's response body, so + the attribute name cannot establish that the value is metadata — a capture + named `gen_ai.usage.input_tokens` pointed at `/content/0/text` is a completion. + Metadata that used to arrive by capture should arrive as a span attribute + instead, which the module can set from its own parsed response. +- Tool-call observability (`_dd_llmobs_tool_spans`) bundles tool arguments and + results (content) together with tool name and duration (metadata) in a single + array. For non-opted-in tenants the whole array is suppressed rather than + field-redacted, so tool-level metadata is lost too. This errs toward less + export, which is the safe direction for a security default; a future + refinement can field-redact within each tool span if the metadata proves + valuable on its own. + +### Risks +- A future LLM export path built through an untouched host would redact for + opted-in tenants (a false redaction, never a leak) — acceptable, and the + trait default makes new paths secure automatically. Note this cuts both ways. + Production builds a three-layer host — + `AuthorizedWasmHost(LocalTDataWasmHost(ProductionWasmHost))` — and only the + innermost layer holds the flag, so **every** wrapper has to forward + `exports_llm_content`. A wrapper that does not makes the engine read the trait + default and redact even for an opted-in tenant: fail-safe, but the opt-in is + inert on that channel. Both wrappers forward, and the test asserts on the real + three-layer stack — an earlier version wrapped `ProductionWasmHost` directly, + a composition production never builds, and it passed while the real chain + dropped the decision. +- The `gen_ai.*` metadata allowlist is a list, and lists go stale. A new + legitimate metadata key is redacted for non-opted-in tenants until it is added + to `is_llm_metadata_attr`. That is the correct direction to fail — a missing + dashboard field is recoverable, an exported prompt is not — but it will look + like a bug to whoever adds the key. + +### DST Compliance +- `TEMPER_LLM_CONTENT_EXPORT_TENANTS` is read once at `ServerState` + construction (`// determinism-ok: read once at startup`), matching the + existing `env_local_tdata_hosts` pattern. `export_llm_content` is a pure + set lookup. The strip and hint-filter helpers are pure, order-independent + transforms over `BTreeSet`/`Vec` — no wall clock, RNG, or ambient I/O. + +## Non-Goals +- Field-level or regex-based redaction of *parts* of a prompt. This is an + all-or-nothing content gate per tenant. +- A runtime admin API to toggle a tenant's opt-in. Startup env config only for now. + +## Alternatives Considered + +1. **Cedar policy per tenant** — Model export as a Cedar action evaluated per + dispatch. Rejected for now: heavier (new action/resource, per-call evaluation + on a hot path) than the finding requires. Left as a documented follow-up. +2. **OTel SpanProcessor that strips content at export** — One central place, but + requires the tenant on every span and non-trivial processor plumbing in the + telemetry bootstrap. It is, however, the only shape that would cover the + free-form channels named under Scope, and is the natural next step if the + residual there proves unacceptable. +3. **Export by default, opt-out** — Rejected: the finding is that content leaks + by default. Only redact-by-default closes it. + +## Rollback Policy + +Set `TEMPER_LLM_CONTENT_EXPORT_TENANTS=*` to restore the previous +export-everything behavior for all tenants, or revert this change set — the +redaction helpers are additive and self-contained. diff --git a/docs/adrs/0172-operator-bootstrap-manage-policies.md b/docs/adrs/0172-operator-bootstrap-manage-policies.md new file mode 100644 index 000000000..b6ebcafca --- /dev/null +++ b/docs/adrs/0172-operator-bootstrap-manage-policies.md @@ -0,0 +1,202 @@ +# ADR-0172: Seed a Narrow Operator `manage_policies` Permit at Credential Bootstrap + +- Status: Accepted +- Date: 2026-08-20 +- Deciders: Temper core maintainers +- Related: + - ADR-0014: Governance gap closure (`manage_policies` on `PolicySet`) + - ADR-0032: Granular Cedar policy storage + - ADR-0033: Platform-assigned agent identity + - ADR-0144: Idempotent Cedar merges + - ADR-0157: Credential-bound Class A authentication edge + - `crates/temper-platform/src/bootstrap.rs` (`bootstrap_operator_credential`) + - `crates/temper-server/src/api/decisions.rs` (approve / deny) + - `crates/temper-server/src/api/decisions_access.rs` + - `crates/temper-server/src/authz/policy_persistence.rs` + - `crates/temper-server/tests/policy_authorization.rs` + +## Context + +A new tenant can boot and run installed apps. `temper serve --app` and +`install_os_app` are process load, not Cedar. After boot, Cedar is +default-deny. + +Approving a denied action (`POST /api/tenants/{tenant}/decisions/{id}/approve`, +`temper decide`) and adding Cedar (the policy API) both require action +`manage_policies` on resource `PolicySet`. OS-app Cedar does not grant that. + +`bootstrap_operator_credential` creates the operator `AgentType` and +`AgentCredential` so `TEMPER_API_KEY` resolves as a verified identity. It does +not persist or activate a `manage_policies` permit. On a virgin store the +operator therefore cannot approve a denial or add Cedar. That is the +governance loop, not "apps cannot install" and not "named actions the app +already permits." + +This is ARN-389. + +Closing that door is not enough. An agent who was denied must not be able +to approve or deny that same decision, even if they somehow have +`manage_policies`. "They lack the permit" is not the control. + +## Decision + +Two halves, both required: + +1. When the operator credential is bootstrapped for a tenant, persist and + activate a **narrow** Cedar permit through the same door as every other + policy: validate, merge into the tenant's live Cedar, write a granular + `policies` row, survive restart. Not a code bypass. Not permit-all. +2. On the approve and deny HTTP paths, reject the request when the + caller's principal id equals `PendingDecision.agent_id`. 403. This is + independent of Cedar. + +### Sub-Decision 1: Seed inside `bootstrap_operator_credential` + +The permit is created in `bootstrap_operator_credential` for **that** +tenant. Every caller that bootstraps an operator credential gets the door, +not only `"default"`. CLI Phase 8 today still registers the deployment key +in `default` only (ADR-0157); this ADR does not change that scope. + +**Why this approach**: the missing permit is a property of the operator +identity, not of a particular tenant name. Wiring at the credential +function keeps the seed and the identity on the same path. + +### Sub-Decision 2: Exact permit shape + +Use the statement already proven in `policy_authorization.rs`: + +``` +permit( + principal is Agent, + action == Action::"manage_policies", + resource == PolicySet::"{tenant}" +) when { + principal.agent_type == "operator" && + principal.agentTypeVerified == true +}; +``` + +`{tenant}` is the tenant being bootstrapped. Unverified principals and +non-operator agent types remain default-deny for `manage_policies`. + +**Why this approach**: it is the smallest Cedar that closes the approval +loop, matches the existing HTTP authorization test, and does not grant +entity actions or `create_tenant`. + +### Sub-Decision 3: Merge live Cedar; persist a stable granular row + +1. Read the tenant's current live policy text. +2. Append the permit only if that statement is not already present + (ADR-0144-style idempotent merge). Do not replace app Cedar. +3. Reload the tenant Cedar engine with the merged text. +4. Persist the **isolated** statement as granular policy id + `operator-bootstrap-manage-policies` via `persist_and_activate_policy` + (`created_by = "bootstrap"`). Hash-gated writes make re-bootstrap a + no-op once the row exists. + +Restart recovery (`recover_cedar_policies`) already concatenates granular +rows, so the permit survives reboot without a special case. + +**Why this approach**: OS-app install already merges into live Cedar and +persists per-file rows. Reusing that pattern keeps operator bootstrap on +the same storage and activation path. Persisting only the statement (not +the whole live blob as `primary`) avoids wiping or duplicating app policy +rows. + +### Sub-Decision 4: `create_tenant` stays a separate door + +`create_tenant` is not required to list policies, add a Cedar rule, or +approve a denial in the bootstrapped tenant. This ADR does not grant it. +If a later loop is blocked on tenant provisioning, that is a different +permit and a different decision. + +### Sub-Decision 5: Denied principal cannot resolve their own decision + +After `manage_policies` succeeds, `POST .../decisions/{id}/approve` and +`POST .../decisions/{id}/deny` compare `security_context.principal.id` to +`PendingDecision.agent_id`. On match, return 403 and do not change the +decision or Cedar. + +This is not Cedar. An agent who was granted `manage_policies` still cannot +approve themselves into power. A verified operator who is **not** the +denied principal can approve. + +**Why this approach**: the missing operator door and the self-approval +ban are different failures. Cedar grants the governance door; the +approve/deny handlers refuse to let the subject of the denial walk +through it for that decision. + +## Rollout Plan + +1. **Phase 0 (this PR)** — ADR, seed in `bootstrap_operator_credential`, + self-resolution reject on approve/deny, red-green tests, live local + `temper serve` on an isolated virgin store. +2. **Phase 1** — None required. Existing tenants pick up the row on the + next process that runs `bootstrap_operator_credential`. + +## Consequences + +### Positive + +- A verified operator can approve denials and manage Cedar on a virgin + store without a hand-seeded policy file. +- The permit is ordinary Cedar: visible, persistable, and disableable + like any other granular row. +- App Cedar remains intact across bootstrap, install, and restart. +- The denied agent cannot close their own governance loop, even with + `manage_policies`. + +### Negative + +- A stolen `TEMPER_API_KEY` that resolves as the verified operator can + manage policies for that tenant. That is the same trust already placed + in the bootstrap key for identity; this ADR only makes the intended + governance door reachable. + +### Risks + +- Re-bootstrap must not append forever. Mitigated by exact-statement + merge plus hash-gated `save_policy` on a stable policy id. +- Loading only persisted rows after seed (and before app rows exist) + would drop in-memory app Cedar. Mitigated by merging into live text + and never calling `load_and_activate_tenant_policies` as the seed + path. + +### DST Compliance + +- `persist_and_activate_policy` already uses `sim_now()` for trajectory + timestamps. +- No new `HashMap`/`HashSet`, threads, or wall-clock in simulation-visible + crates beyond existing persistence helpers. + +## Non-Goals + +- Skipping Cedar on approve or policy writes. +- Seeding `permit(principal, action, resource);`. +- Treating boot-time Cedar skip / `--app` process load as the fix. +- Granting `create_tenant`. +- Auto-registering the deployment key in every tenant (ADR-0157). +- Changing OS-app Cedar or named-action permits. + +## Alternatives Considered + +1. **Code bypass for `manage_policies` when principal is operator** — + Rejected. Authority would not be Cedar, would not persist as a policy + row, and would be invisible to the policy API. +2. **Permit-all for the operator** — Rejected. The bug is the governance + door, not missing entity actions. OS-app Cedar already covers named + actions it intends to allow. +3. **Document that operators must load a policy file** — Rejected. A + virgin store cannot approve the first denial; the operator cannot + write the file through the API that the file is meant to unlock. +4. **Grant `create_tenant` in the same seed** — Rejected. Separate door; + not required for approve / add Cedar in the bootstrapped tenant. +5. **Rely on default-deny so the denied agent cannot approve** — + Rejected. That fails as soon as they obtain `manage_policies`. The + ban is on the approve/deny path. + +## Rollback Policy + +Delete or disable the `operator-bootstrap-manage-policies` row and restart, +or stop calling `bootstrap_operator_credential`. The credential entities +remain; only the Cedar door is removed. diff --git a/docs/efforts/ARN-237/decisions.md b/docs/efforts/ARN-237/decisions.md new file mode 100644 index 000000000..0879efe6a --- /dev/null +++ b/docs/efforts/ARN-237/decisions.md @@ -0,0 +1,13 @@ +# Decisions & Tradeoffs + +## Decision + +Narrow the parser fix to entity decoding failures. + +Came up because PR #409 changes raw values to unescape_value().ok(), which silently defaults a present malformed optional attribute. + +Options: defer that new defaulting behavior; rewrite all strict parsing; propagate the decoding error alone. + +Chose targeted error propagation over broad strict parsing because it resolves the new behavior while preserving the separately planned truncated-schema/startup recovery work. + +Where: PR #409, crates/temper-spec/src/csdl/parser/xml.rs. diff --git a/docs/efforts/ARN-237/intent.md b/docs/efforts/ARN-237/intent.md new file mode 100644 index 000000000..81775e7c6 --- /dev/null +++ b/docs/efforts/ARN-237/intent.md @@ -0,0 +1,9 @@ +# ARN-237: CSDL attribute escaping merge + +The user authorized completing and merging existing Temper PR #409 on 2026-09-09. + +Preserve the PR's scoped outcome: emitted XML attribute values round-trip without injection or normalization loss, and parsed attribute entities decode correctly. Review the reported malformed-entity/defaulting behavior and resolve defects introduced by this change without expanding into the deferred truncated-schema and startup-recovery work. + +The broader ARN-237 remains open until its separately sequenced strict-parser requirements are complete. The implementation belongs to Temper's generic specification kernel. + +Success: scoped verification and review pass on the final PR head; the existing PR merges through the required workflow, followed by applicable kernel deployment verification. diff --git a/docs/efforts/ARN-237/plan.md b/docs/efforts/ARN-237/plan.md new file mode 100644 index 000000000..162373b22 --- /dev/null +++ b/docs/efforts/ARN-237/plan.md @@ -0,0 +1,3 @@ +# ARN-237: plan + +Inspect decoding callers; add a regression for an invalid entity in an optional attribute; make decoding failure explicit; run CSDL tests and relevant metadata flow checks, then review the final diff. Keep deferred strict-parser requirements on ARN-237. Merge through recorded proof/review and verify the applicable kernel deployment. diff --git a/docs/efforts/ARN-237/spec.md b/docs/efforts/ARN-237/spec.md new file mode 100644 index 000000000..057503644 --- /dev/null +++ b/docs/efforts/ARN-237/spec.md @@ -0,0 +1,3 @@ +# ARN-237: spec + +CSDL emission escapes every attribute and preserves values across a parse round trip. Valid entities decode. Invalid entity references must not turn a present optional attribute into an absent/defaulted value. This correction does not add mandatory-attribute, truncated-document, numeric-validation, or startup-recovery features. diff --git a/reference-apps/crucible/Cargo.toml b/reference-apps/crucible/Cargo.toml index e38d3fa81..81f188e1a 100644 --- a/reference-apps/crucible/Cargo.toml +++ b/reference-apps/crucible/Cargo.toml @@ -32,6 +32,7 @@ axum = { workspace = true } hyper = { workspace = true } temper-jit = { path = "../../crates/temper-jit" } temper-runtime = { path = "../../crates/temper-runtime" } +temper-authz = { path = "../../crates/temper-authz" } temper-server = { path = "../../crates/temper-server" } temper-verify = { path = "../../crates/temper-verify" } temper-spec = { path = "../../crates/temper-spec" } diff --git a/reference-apps/crucible/tests/crucible_agents_validation.rs b/reference-apps/crucible/tests/crucible_agents_validation.rs index 3136764cc..c1a7a604f 100644 --- a/reference-apps/crucible/tests/crucible_agents_validation.rs +++ b/reference-apps/crucible/tests/crucible_agents_validation.rs @@ -44,6 +44,25 @@ const AGENT_VERSION_IOA: &str = include_str!("../specs/agent_version.ioa.toml"); const CALLABLE_AGENT_IOA: &str = include_str!("../specs/callable_agent.ioa.toml"); const CROSS_INVARIANTS_TOML: &str = include_str!("../specs/cross-invariants.toml"); const MODEL_CSDL: &str = include_str!("../specs/model.csdl.xml"); +const CRUCIBLE_AGENT_VALIDATION_POLICY: &str = r#" +permit( + principal, + action in [ + Action::"list", + Action::"read", + Action::"create", + Action::"update", + Action::"delete", + Action::"ArchiveManagedAgent" + ], + resource is ManagedAgent +); +permit(principal, action in [Action::"list", Action::"read", Action::"create", Action::"update", Action::"delete"], resource is AgentMcpServer); +permit(principal, action in [Action::"list", Action::"read", Action::"create", Action::"update", Action::"delete"], resource is AgentSkill); +permit(principal, action in [Action::"list", Action::"read", Action::"create", Action::"update", Action::"delete"], resource is AgentTool); +permit(principal, action in [Action::"list", Action::"read", Action::"create", Action::"update", Action::"delete"], resource is AgentToolConfig); +permit(principal, action in [Action::"list", Action::"read", Action::"create", Action::"update", Action::"delete"], resource is AgentVersion); +"#; /// Build a `ServerState` preloaded with all nine Crucible IOAs (three from /// the Environment slice + six from the ManagedAgent slice) and the extended @@ -106,6 +125,35 @@ fn build_crucible_state() -> ServerState { } } state + .authz + .reload_tenant_policies( + TenantId::default().as_str(), + CRUCIBLE_AGENT_VALIDATION_POLICY, + ) + .expect("install Crucible agent validation policy"); + state +} + +fn authenticate(mut request: Request) -> Request { + let security_context = temper_authz::SecurityContext { + principal: temper_authz::Principal { + id: "crucible-agent-validation".to_string(), + kind: temper_authz::PrincipalKind::Customer, + role: None, + acting_for: None, + agent_type: None, + attributes: Default::default(), + }, + context_attrs: Default::default(), + correlation_id: "crucible-agent-validation".to_string(), + }; + request + .extensions_mut() + .insert(temper_authz::AuthenticatedRequestContext::new( + TenantId::default(), + security_context, + )); + request } async fn send( @@ -115,12 +163,14 @@ async fn send( body: &str, ) -> (StatusCode, serde_json::Value) { let router = build_router(state.clone()); - let req = Request::builder() - .method(method) - .uri(uri) - .header("Content-Type", "application/json") - .body(Body::from(body.to_string())) - .unwrap(); + let req = authenticate( + Request::builder() + .method(method) + .uri(uri) + .header("Content-Type", "application/json") + .body(Body::from(body.to_string())) + .unwrap(), + ); let resp = router.oneshot(req).await.unwrap(); let status = resp.status(); let bytes = axum::body::to_bytes(resp.into_body(), 1_000_000) @@ -251,12 +301,15 @@ async fn create_agent_archived_without_archived_at_is_rejected() { let (status, body_out) = post(&state, "/tdata/ManagedAgents", body).await; assert_eq!( status, - StatusCode::CONFLICT, - "Archived without ArchivedAt must be rejected: {body_out:?}" + StatusCode::BAD_REQUEST, + "creating directly into Archived must be rejected before field invariants: {body_out:?}" ); - assert_eq!( - body_out["error"]["details"]["invariant"].as_str(), - Some("ArchivedRequiresArchivedAt") + assert_eq!(body_out["error"]["code"].as_str(), Some("InvalidBody")); + assert!( + body_out["error"]["message"] + .as_str() + .is_some_and(|message| message.contains("initial state 'Active'")), + "rejection must preserve the spec-defined initial-state contract: {body_out:?}" ); } diff --git a/reference-apps/crucible/tests/crucible_chat_integration.rs b/reference-apps/crucible/tests/crucible_chat_integration.rs index 3d5863194..73c271592 100644 --- a/reference-apps/crucible/tests/crucible_chat_integration.rs +++ b/reference-apps/crucible/tests/crucible_chat_integration.rs @@ -25,7 +25,10 @@ //! //! No network, no API key, no secrets. Everything runs in-process. +use axum::extract::Request; use axum::http::StatusCode; +use axum::middleware::Next; +use axum::response::Response; use crucible_reference::chat::anthropic::MockModel; use crucible_reference::chat::responder::{RespondRequest, respond}; use crucible_reference::chat::seed::{CallableAgentSeedSpec, SeedOptions, seed}; @@ -59,6 +62,41 @@ const CALLABLE_AGENT_IOA: &str = include_str!("../specs/callable_agent.ioa.toml" const SESSION_THREAD_IOA: &str = include_str!("../specs/session_thread.ioa.toml"); const CROSS_INVARIANTS_TOML: &str = include_str!("../specs/cross-invariants.toml"); const MODEL_CSDL: &str = include_str!("../specs/model.csdl.xml"); +const CRUCIBLE_CHAT_INTEGRATION_POLICY: &str = r#" +permit(principal, action in [Action::"list", Action::"read", Action::"create", Action::"update", Action::"delete"], resource is Environment); +permit(principal, action in [Action::"list", Action::"read", Action::"create", Action::"update", Action::"delete"], resource is ManagedAgent); +permit(principal, action in [Action::"list", Action::"read", Action::"create", Action::"update", Action::"delete"], resource is AgentTool); +permit( + principal, + action in [ + Action::"list", + Action::"read", + Action::"create", + Action::"update", + Action::"delete", + Action::"StartSession", + Action::"IdleSession", + Action::"ResumeSession" + ], + resource is Session +); +permit(principal, action in [Action::"list", Action::"read", Action::"create", Action::"update", Action::"delete"], resource is SessionEvent); +permit(principal, action in [Action::"list", Action::"read", Action::"create", Action::"update", Action::"delete"], resource is CallableAgent); +permit( + principal, + action in [ + Action::"list", + Action::"read", + Action::"create", + Action::"update", + Action::"delete", + Action::"IdleThread", + Action::"ResumeThread", + Action::"TerminateThread" + ], + resource is SessionThread +); +"#; /// Build a fully-loaded Crucible `ServerState` with every entity type /// marked as verified. Same pattern as `crucible_sessions_validation.rs`. @@ -127,12 +165,41 @@ fn build_crucible_state() -> ServerState { } } state + .authz + .reload_tenant_policies( + TenantId::default().as_str(), + CRUCIBLE_CHAT_INTEGRATION_POLICY, + ) + .expect("install Crucible chat integration policy"); + state +} + +async fn authenticate_test_request(mut request: Request, next: Next) -> Response { + let security_context = temper_authz::SecurityContext { + principal: temper_authz::Principal { + id: "crucible-chat-integration".to_string(), + kind: temper_authz::PrincipalKind::Customer, + role: None, + acting_for: None, + agent_type: None, + attributes: Default::default(), + }, + context_attrs: Default::default(), + correlation_id: "crucible-chat-integration".to_string(), + }; + request + .extensions_mut() + .insert(temper_authz::AuthenticatedRequestContext::new( + TenantId::default(), + security_context, + )); + next.run(request).await } /// Spawn a real axum::serve on an ephemeral port and return its /// bound address. The task runs until the test exits. async fn spawn_server(state: ServerState) -> SocketAddr { - let router = build_router(state); + let router = build_router(state).layer(axum::middleware::from_fn(authenticate_test_request)); let listener = tokio::net::TcpListener::bind("127.0.0.1:0") .await .expect("ephemeral bind should succeed"); diff --git a/reference-apps/crucible/tests/crucible_sessions_validation.rs b/reference-apps/crucible/tests/crucible_sessions_validation.rs index f9c3ef287..b37219268 100644 --- a/reference-apps/crucible/tests/crucible_sessions_validation.rs +++ b/reference-apps/crucible/tests/crucible_sessions_validation.rs @@ -50,6 +50,66 @@ const CALLABLE_AGENT_IOA: &str = include_str!("../specs/callable_agent.ioa.toml" const SESSION_THREAD_IOA: &str = include_str!("../specs/session_thread.ioa.toml"); const CROSS_INVARIANTS_TOML: &str = include_str!("../specs/cross-invariants.toml"); const MODEL_CSDL: &str = include_str!("../specs/model.csdl.xml"); +const CRUCIBLE_SESSION_VALIDATION_POLICY: &str = r#" +permit( + principal, + action in [ + Action::"list", + Action::"read", + Action::"create", + Action::"update", + Action::"delete", + Action::"ArchiveEnvironment" + ], + resource is Environment +); +permit( + principal, + action in [ + Action::"list", + Action::"read", + Action::"create", + Action::"update", + Action::"delete", + Action::"ArchiveManagedAgent" + ], + resource is ManagedAgent +); +permit( + principal, + action in [ + Action::"list", + Action::"read", + Action::"create", + Action::"update", + Action::"delete", + Action::"StartSession", + Action::"IdleSession", + Action::"ResumeSession", + Action::"RescheduleSession", + Action::"TerminateSession", + Action::"ArchiveSession" + ], + resource is Session +); +permit(principal, action in [Action::"list", Action::"read", Action::"create", Action::"update", Action::"delete"], resource is SessionResource); +permit(principal, action in [Action::"list", Action::"read", Action::"create", Action::"update", Action::"delete"], resource is SessionEvent); +permit(principal, action in [Action::"list", Action::"read", Action::"create", Action::"update", Action::"delete"], resource is CallableAgent); +permit( + principal, + action in [ + Action::"list", + Action::"read", + Action::"create", + Action::"update", + Action::"delete", + Action::"IdleThread", + Action::"ResumeThread", + Action::"TerminateThread" + ], + resource is SessionThread +); +"#; /// Build a `ServerState` preloaded with all twelve Crucible IOAs (three from /// the Environment slice, six from the ManagedAgent slice, three from the @@ -121,6 +181,35 @@ fn build_crucible_state() -> ServerState { } } state + .authz + .reload_tenant_policies( + TenantId::default().as_str(), + CRUCIBLE_SESSION_VALIDATION_POLICY, + ) + .expect("install Crucible session validation policy"); + state +} + +fn authenticate(mut request: Request) -> Request { + let security_context = temper_authz::SecurityContext { + principal: temper_authz::Principal { + id: "crucible-session-validation".to_string(), + kind: temper_authz::PrincipalKind::Customer, + role: None, + acting_for: None, + agent_type: None, + attributes: Default::default(), + }, + context_attrs: Default::default(), + correlation_id: "crucible-session-validation".to_string(), + }; + request + .extensions_mut() + .insert(temper_authz::AuthenticatedRequestContext::new( + TenantId::default(), + security_context, + )); + request } async fn send( @@ -130,12 +219,14 @@ async fn send( body: &str, ) -> (StatusCode, serde_json::Value) { let router = build_router(state.clone()); - let req = Request::builder() - .method(method) - .uri(uri) - .header("Content-Type", "application/json") - .body(Body::from(body.to_string())) - .unwrap(); + let req = authenticate( + Request::builder() + .method(method) + .uri(uri) + .header("Content-Type", "application/json") + .body(Body::from(body.to_string())) + .unwrap(), + ); let resp = router.oneshot(req).await.unwrap(); let status = resp.status(); let bytes = axum::body::to_bytes(resp.into_body(), 1_000_000) @@ -276,10 +367,17 @@ async fn create_session_with_unknown_status_is_rejected() { "UpdatedAt": "2026-04-11T00:00:00Z" }"#; let (status, body_out) = post(&state, "/tdata/Sessions", body).await; - assert_eq!(status, StatusCode::CONFLICT, "{body_out:?}"); assert_eq!( - body_out["error"]["details"]["invariant"].as_str(), - Some("StatusMustBeKnown") + status, + StatusCode::BAD_REQUEST, + "creating directly into an unknown status must be rejected before field invariants: {body_out:?}" + ); + assert_eq!(body_out["error"]["code"].as_str(), Some("InvalidBody")); + assert!( + body_out["error"]["message"] + .as_str() + .is_some_and(|message| message.contains("initial state 'Rescheduling'")), + "rejection must preserve the spec-defined initial-state contract: {body_out:?}" ); } @@ -298,10 +396,17 @@ async fn create_session_terminated_without_terminated_at_is_rejected() { "UpdatedAt": "2026-04-11T00:00:00Z" }"#; let (status, body_out) = post(&state, "/tdata/Sessions", body).await; - assert_eq!(status, StatusCode::CONFLICT, "{body_out:?}"); assert_eq!( - body_out["error"]["details"]["invariant"].as_str(), - Some("TerminatedRequiresTerminatedAt") + status, + StatusCode::BAD_REQUEST, + "creating directly into Terminated must be rejected before field invariants: {body_out:?}" + ); + assert_eq!(body_out["error"]["code"].as_str(), Some("InvalidBody")); + assert!( + body_out["error"]["message"] + .as_str() + .is_some_and(|message| message.contains("initial state 'Rescheduling'")), + "rejection must preserve the spec-defined initial-state contract: {body_out:?}" ); } @@ -320,10 +425,17 @@ async fn create_session_archived_without_archived_at_is_rejected() { "UpdatedAt": "2026-04-11T00:00:00Z" }"#; let (status, body_out) = post(&state, "/tdata/Sessions", body).await; - assert_eq!(status, StatusCode::CONFLICT, "{body_out:?}"); assert_eq!( - body_out["error"]["details"]["invariant"].as_str(), - Some("ArchivedRequiresArchivedAt") + status, + StatusCode::BAD_REQUEST, + "creating directly into Archived must be rejected before field invariants: {body_out:?}" + ); + assert_eq!(body_out["error"]["code"].as_str(), Some("InvalidBody")); + assert!( + body_out["error"]["message"] + .as_str() + .is_some_and(|message| message.contains("initial state 'Rescheduling'")), + "rejection must preserve the spec-defined initial-state contract: {body_out:?}" ); } diff --git a/reference-apps/crucible/tests/crucible_validation.rs b/reference-apps/crucible/tests/crucible_validation.rs index aef6886dd..25cabb8ab 100644 --- a/reference-apps/crucible/tests/crucible_validation.rs +++ b/reference-apps/crucible/tests/crucible_validation.rs @@ -35,6 +35,100 @@ const SESSION_SCHEDULE_IOA: &str = include_str!("../specs/session_schedule.ioa.t const CRUCIBLE_SCHEDULER_IOA: &str = include_str!("../specs/crucible_scheduler.ioa.toml"); const CROSS_INVARIANTS_TOML: &str = include_str!("../specs/cross-invariants.toml"); const MODEL_CSDL: &str = include_str!("../specs/model.csdl.xml"); +const CRUCIBLE_VALIDATION_POLICY: &str = r#" +permit( + principal, + action in [ + Action::"list", + Action::"read", + Action::"create", + Action::"update", + Action::"delete", + Action::"ArchiveEnvironment" + ], + resource is Environment +); +permit(principal, action in [Action::"list", Action::"read", Action::"create", Action::"update", Action::"delete"], resource is EnvironmentAllowedHost); +permit(principal, action in [Action::"list", Action::"read", Action::"create", Action::"update", Action::"delete"], resource is EnvironmentPackage); +permit(principal, action in [Action::"list", Action::"read", Action::"create", Action::"update", Action::"delete"], resource is ManagedAgent); +permit( + principal, + action in [ + Action::"list", + Action::"read", + Action::"create", + Action::"update", + Action::"delete", + Action::"StartSession", + Action::"IdleSession", + Action::"ResumeSession", + Action::"RescheduleSession", + Action::"TerminateSession", + Action::"ArchiveSession" + ], + resource is Session +); +permit(principal, action in [Action::"list", Action::"read", Action::"create", Action::"update", Action::"delete"], resource is SessionResource); +permit( + principal, + action in [ + Action::"list", + Action::"read", + Action::"create", + Action::"update", + Action::"delete", + Action::"ArchiveMemoryStore" + ], + resource is MemoryStore +); +permit(principal, action in [Action::"list", Action::"read", Action::"create", Action::"update", Action::"delete"], resource is Memory); +permit( + principal, + action in [ + Action::"list", + Action::"read", + Action::"create", + Action::"update", + Action::"delete", + Action::"RedactVersion" + ], + resource is MemoryVersion +); +permit( + principal, + action in [ + Action::"list", + Action::"read", + Action::"create", + Action::"update", + Action::"delete", + Action::"ActivateSchedule", + Action::"Trigger", + Action::"TriggerComplete", + Action::"TriggerFailed", + Action::"PauseSchedule", + Action::"ResumeSchedule", + Action::"ExpireSchedule" + ], + resource is SessionSchedule +); +permit( + principal, + action in [ + Action::"list", + Action::"read", + Action::"create", + Action::"update", + Action::"delete", + Action::"Start", + Action::"ScheduledCheck", + Action::"CheckComplete", + Action::"CheckFailed", + Action::"ScheduleFailed" + ], + resource is CrucibleScheduler +); +"#; /// Build a `ServerState` preloaded with Crucible's three IOAs, CSDL, and /// the cross-invariants file. Marks all three entity types as verified so @@ -102,6 +196,32 @@ fn build_crucible_state() -> ServerState { } } state + .authz + .reload_tenant_policies(TenantId::default().as_str(), CRUCIBLE_VALIDATION_POLICY) + .expect("install Crucible validation policy"); + state +} + +fn authenticate(mut request: Request) -> Request { + let security_context = temper_authz::SecurityContext { + principal: temper_authz::Principal { + id: "crucible-validation".to_string(), + kind: temper_authz::PrincipalKind::Customer, + role: None, + acting_for: None, + agent_type: None, + attributes: Default::default(), + }, + context_attrs: Default::default(), + correlation_id: "crucible-validation".to_string(), + }; + request + .extensions_mut() + .insert(temper_authz::AuthenticatedRequestContext::new( + TenantId::default(), + security_context, + )); + request } /// Send an HTTP request through the router and return `(status, body_json)`. @@ -112,12 +232,14 @@ async fn send( body: &str, ) -> (StatusCode, serde_json::Value) { let router = build_router(state.clone()); - let req = Request::builder() - .method(method) - .uri(uri) - .header("Content-Type", "application/json") - .body(Body::from(body.to_string())) - .unwrap(); + let req = authenticate( + Request::builder() + .method(method) + .uri(uri) + .header("Content-Type", "application/json") + .body(Body::from(body.to_string())) + .unwrap(), + ); let resp = router.oneshot(req).await.unwrap(); let status = resp.status(); let bytes = axum::body::to_bytes(resp.into_body(), 1_000_000) @@ -1224,12 +1346,13 @@ async fn session_schedule_status_invariant() { .await; assert_eq!( status, - StatusCode::CONFLICT, + StatusCode::BAD_REQUEST, "bad status rejected: {body:?}" ); + assert_eq!(body["error"]["code"].as_str(), Some("InvalidBody")); assert_eq!( - body["error"]["details"]["invariant"].as_str(), - Some("StatusMustBeKnown") + body["error"]["message"].as_str(), + Some("Status must equal the spec-defined initial state 'Draft'") ); } diff --git a/scripts/check-determinism.sh b/scripts/check-determinism.sh index 230caeaec..9f2a9b706 100755 --- a/scripts/check-determinism.sh +++ b/scripts/check-determinism.sh @@ -35,7 +35,10 @@ scan_nondeterminism() { esac local MATCHES - MATCHES="$(grep -nE "$PATTERN" "$FILE" 2>/dev/null | grep -v '// determinism-ok' | grep -v '^[[:space:]]*//' || true)" + # `grep -n` prefixes every line with `NN:`, so the comment filter + # has to skip that prefix — anchored at `^` it matched nothing and + # every mention in a doc comment counted as a finding. + MATCHES="$(grep -nE "$PATTERN" "$FILE" 2>/dev/null | grep -v '// determinism-ok' | grep -vE '^[0-9]+:[[:space:]]*(//|/\*|\*)' || true)" if [ -n "$MATCHES" ]; then local REL_FILE REL_FILE="$(realpath --relative-to="$WORKSPACE_ROOT" "$FILE" 2>/dev/null || echo "$FILE")" @@ -61,6 +64,14 @@ scan_nondeterminism \ 'HashMap' \ "Use BTreeMap for deterministic iteration order" +# The per-edit hook guard has always scanned for this; this repo-wide audit did +# not, so a HashSet could reach a simulation-visible crate with neither +# catching it. +scan_nondeterminism \ + "HashSet usage" \ + 'HashSet' \ + "Use BTreeSet for deterministic iteration order" + scan_nondeterminism \ "Wall-clock time" \ 'SystemTime::now\(\)|Instant::now\(\)' \ diff --git a/ui/observe/lib/types.ts b/ui/observe/lib/types.ts index 8fbfc432b..4dfa68540 100644 --- a/ui/observe/lib/types.ts +++ b/ui/observe/lib/types.ts @@ -32,6 +32,12 @@ export interface StateVariable { export interface SpecDetail { entity_type: string; + /** + * Content hash of the spec's IOA source as the kernel registered it — the + * digest a conformance check compares a run's recorded spec version + * against. Optional: a server older than the field omits it. + */ + spec_version?: string; states: string[]; initial_state: string; actions: SpecAction[];