From 2bacd5007bec3f0eb0d4a722a43d734817575003 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 05:51:16 +0000 Subject: [PATCH 1/2] Add a Rust port where branches are enum variants Port fuzzymatch to Rust in rust/, keeping the Python package as is. Branches are declared as variants of the caller's own enum (via the `branches!` macro or a hand-written `Branches` impl), and a Matcher returns a `Verdict` to dispatch on with a native `match`. That moves the Python runtime machinery into the type system: - `@otherwise` / `Unmatched` become the non-Matched arms, which the compiler requires (E0004), and a new variant breaks every match until it is routed. - `Verdict::{Matched, LowConfidence, GuardFailed}(B)` and `NoMatch` replace `Outcome` plus a nullable branch, so misses can be handled per branch and per reason. - Model labels are parsed back into `B`; unknown ones are errors. - Configuration is validated once in `build()`. - The Jev adapter speaks the HTTP API directly, with tagged serde enums for questions and answers (unknown answer types ignored). Includes ports of both examples (the shell guard's `Permission` enum has no Deny variant), the matcher test suite, an HTTP round-trip test against a local server, and a CI job for fmt, clippy and tests. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XvSGXaFnctMyew8CowkNRS --- .github/workflows/ci.yml | 18 + .gitignore | 3 + README.md | 5 +- rust/Cargo.lock | 860 ++++++++++++++++++++++++++++++++ rust/Cargo.toml | 37 ++ rust/README.md | 147 ++++++ rust/examples/shell_guard.rs | 186 +++++++ rust/examples/support_router.rs | 104 ++++ rust/rustfmt.toml | 2 + rust/src/branch.rs | 201 ++++++++ rust/src/decider.rs | 105 ++++ rust/src/jev.rs | 319 ++++++++++++ rust/src/lib.rs | 69 +++ rust/src/matcher.rs | 373 ++++++++++++++ rust/src/testing.rs | 78 +++ rust/tests/jev.rs | 147 ++++++ rust/tests/matcher.rs | 286 +++++++++++ 17 files changed, 2939 insertions(+), 1 deletion(-) create mode 100644 rust/Cargo.lock create mode 100644 rust/Cargo.toml create mode 100644 rust/README.md create mode 100644 rust/examples/shell_guard.rs create mode 100644 rust/examples/support_router.rs create mode 100644 rust/rustfmt.toml create mode 100644 rust/src/branch.rs create mode 100644 rust/src/decider.rs create mode 100644 rust/src/jev.rs create mode 100644 rust/src/lib.rs create mode 100644 rust/src/matcher.rs create mode 100644 rust/src/testing.rs create mode 100644 rust/tests/jev.rs create mode 100644 rust/tests/matcher.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8fdf3ab..89642b8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,3 +20,21 @@ jobs: - run: uv run ruff check . - run: uv run ruff format --check . - run: uv run pytest -q + + rust: + runs-on: ubuntu-latest + defaults: + run: + working-directory: rust + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy, rustfmt + - uses: Swatinem/rust-cache@v2 + with: + workspaces: rust + - run: cargo fmt --check + - run: cargo clippy --locked --all-targets -- -D warnings + - run: cargo clippy --locked --no-default-features -- -D warnings + - run: cargo test --locked diff --git a/.gitignore b/.gitignore index 505a3b1..dc4058a 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,6 @@ wheels/ # Virtual environments .venv + +# Rust build output +/rust/target diff --git a/README.md b/README.md index b0e438a..bb093bf 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,9 @@ A Python `match` statement dispatches on the *shape* of a value. `fuzzymatch` di value *means*, and it keeps what makes `match` trustworthy: a fixed set of arms, guards, and a `case _` fallback. +There is also a [Rust port](rust/README.md), where the branches are variants of your own enum and +the compiler checks that every outcome is handled. + > **Status: proof of concept.** The request and response mapping is tested through the real TypeSafe > SDK with only the HTTP layer mocked, but it has not yet been run against the live API. See > [Limits](#limits). @@ -148,7 +151,7 @@ uv run pytest - `async` matchers on `AsyncTypeSafeClient` - an `escalate_to=` decider, so uncertain calls go to a frontier model automatically - a small tool that replays observer logs to suggest per-branch thresholds -- nested matchers for hierarchical routing, and a TypeScript port +- nested matchers for hierarchical routing, and a TypeScript port (a [Rust port](rust/README.md) exists) ## License diff --git a/rust/Cargo.lock b/rust/Cargo.lock new file mode 100644 index 0000000..4740a24 --- /dev/null +++ b/rust/Cargo.lock @@ -0,0 +1,860 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f360145194ee8e21db5ee7f3fcd4fe52210864c75c985dae33218202c8bbe040" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "cookie" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a373e3602691c3cdea496d2f0ee5935151e6168fe87739483c463db1b2f2f87" +dependencies = [ + "percent-encoding", + "time", + "version_check", +] + +[[package]] +name = "cookie_store" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fc4bff745c9b4c7fb1e97b25d13153da2bc7796260141df62378998d070207f" +dependencies = [ + "cookie", + "document-features", + "idna", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "time", + "url", +] + +[[package]] +name = "crc32fast" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "document-features" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" +dependencies = [ + "litrs", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "find-msvc-tools" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" + +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fuzzymatch" +version = "0.1.0" +dependencies = [ + "indexmap", + "serde", + "serde_json", + "thiserror", + "ureq", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "icu_collections" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" + +[[package]] +name = "icu_properties" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" + +[[package]] +name = "icu_provider" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown", + "serde", + "serde_core", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "litrs" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom", + "libc", + "untrusted", + "windows-sys", +] + +[[package]] +name = "rustls" +version = "0.23.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "indexmap", + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "smallvec" +version = "1.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9395f0f0eee849a9b707b2f06bb92a6a422090e2123bb2ef8e87a0e61892a8e" + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "time" +version = "0.3.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9e442fc33d7fdb45aa9bfeb312c095964abdf596f7567261062b2a7107aaabd" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b36ee98fd31ec7426d599183e8fe26932a8dc1fb76ddb6214d05493377d34ca" + +[[package]] +name = "time-macros" +version = "0.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e552d1249bf61ac2a52db88179fd0673def1e1ad8243a00d9ec9ed71fee3dd" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "ureq" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7ac20be9b7726e0bbdbf974c059676d9acb1cd414961f570a4e8231cacd7fc" +dependencies = [ + "base64", + "cookie_store", + "flate2", + "log", + "percent-encoding", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "ureq-proto", + "utf8-zero", + "webpki-roots", +] + +[[package]] +name = "ureq-proto" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f86fd172ccca569e458f61b6bdd6220965a9ef36e672a6852953b51a0e1583be" +dependencies = [ + "base64", + "http", + "httparse", + "log", +] + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8-zero" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "webpki-roots" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33811428bee40dbceb6d545e95754741d17a6aef9a4849f0fd62e2ba4f412a78" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/rust/Cargo.toml b/rust/Cargo.toml new file mode 100644 index 0000000..ca428fc --- /dev/null +++ b/rust/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "fuzzymatch" +version = "0.1.0" +description = "Pattern matching on meaning: classify into your own enum by natural-language description, gated by calibrated confidence." +edition = "2024" +rust-version = "1.85" +license = "MIT" +authors = ["Sam Keen "] +repository = "https://github.com/AlteredCraft/fuzzymatch" +readme = "README.md" +keywords = ["pattern-matching", "dispatch", "classification", "typesafe", "llm"] +categories = ["api-bindings", "text-processing"] + +[features] +default = ["jev"] +# The TypeSafe Jev adapter. Turn it off to bring your own `Decider` without an HTTP client. +jev = ["dep:ureq", "dep:indexmap"] + +[dependencies] +serde = { version = "1", features = ["derive"] } +serde_json = { version = "1", features = ["preserve_order"] } +thiserror = "2" +indexmap = { version = "2", features = ["serde"], optional = true } +ureq = { version = "3", features = ["json"], optional = true } + +[[example]] +name = "support_router" +required-features = ["jev"] +test = true + +[[example]] +name = "shell_guard" +required-features = ["jev"] +test = true + +[dev-dependencies] +ureq = { version = "3", features = ["json"] } diff --git a/rust/README.md b/rust/README.md new file mode 100644 index 0000000..213b639 --- /dev/null +++ b/rust/README.md @@ -0,0 +1,147 @@ +# fuzzymatch (Rust) + +**Pattern matching on meaning, checked by the compiler.** A Rust port of the Python +[`fuzzymatch`](../README.md): classify a value by what it *means* into your own enum, with a +calibrated confidence gating whether a branch matches, then dispatch with an ordinary `match`. + +```rust +use fuzzymatch::{Branch, JevDecider, Matcher, Verdict, branches}; + +branches! { + enum Intent { + Refund => Branch::new("customer wants their money back") + .min_confidence(0.9) + .when("The message includes an order number"), + OrderStatus => Branch::new("customer asks where their order is"), + } +} + +let support = Matcher::::new("What is the customer asking for?", JevDecider::from_env()?)?; + +let m = support.classify("Order #88213 arrived broken. I want a refund.")?; +match m.verdict { + Verdict::Matched(Intent::Refund) => refund(), + Verdict::Matched(Intent::OrderStatus) => look_up_order(), + Verdict::LowConfidence(Intent::Refund) => ask_a_person_to_confirm_refund(&m), + Verdict::GuardFailed(Intent::Refund) => ask_for_order_number(), + Verdict::LowConfidence(_) | Verdict::GuardFailed(_) | Verdict::NoMatch => escalate(&m), +} +``` + +> **Status: proof of concept**, like the Python version. The wire format is tested over real HTTP +> against a local stand-in for the API, not against the live TypeSafe API. + +## What Rust's enums and `match` buy you + +The Python library had to rebuild a small `match` statement at runtime: decorators register +handlers, `@otherwise` is the `case _`, and `Unmatched` is raised if you forgot it. In Rust the +branches are variants of your own enum, and the language's `match` does the dispatching, so that +machinery either moves into the type system or disappears. + +| Python | Rust | +| ------------------------------------------------------- | ------------------------------------------------------------------------ | +| `@matcher.on(...)` registers a handler under a string | `branches!` declares an enum variant with its `Branch` | +| `@matcher.otherwise`, else `Unmatched` at runtime | the non-`Matched` arms of your `match`, required by the compiler | +| `Outcome` plus `branch: str \| None`, `guard: float \| None` | `Verdict`: `NoMatch` carries no branch, the rest carry a `B` | +| one `otherwise` for every kind of miss | match on them separately: `LowConfidence(Refund)` ≠ `GuardFailed(Refund)` | +| `match.ranked()` → `[(str, float)]` | `ranked()` → `[(Option, f64)]`, `None` is "none of these" | +| handler arity sniffed with `inspect.signature` | gone: there are no handlers, just your code in each arm | +| label, reserved-name, duplicate checks at decoration | checked once in `build()`; `branches!` makes most of them impossible | +| option criterion is `str \| dict` | `Criterion::Plain` / `Criterion::Detailed` | +| Jev answers told apart by `isinstance` | `#[serde(tag = "type")] enum Answer { Choice, Noul, Other }` | + +Concretely: + +- **Exhaustiveness is the fallback check.** Leave out `NoMatch` or a variant's `Matched` arm and it + doesn't compile (`error[E0004]`). Add a variant to the enum and every `match` on it lights up until + you decide where it goes. The Python version could only find out at runtime, on the input that + hit the gap. +- **Illegal states are unrepresentable.** A verdict of `NoMatch` has no branch to read; a matched + `B` is always one of your variants, because labels coming back from the model are parsed into `B` + (an unknown label is `Error::UnknownOption`, not a string that flows onward). +- **Guards compose.** `when(...)` is the model-evaluated guard; Rust's own `if` guards still work on + top: `Verdict::Matched(Effect::ReadOnly) if auto_allow => allow()`. +- **A miss can be an error when you want one.** `m.into_result()?` turns anything but `Matched` into + `Unmatched`, carrying the full record, for code that would rather propagate than branch. +- **Enums constrain your side too.** The shell guard's decision type is `enum Permission { Allow, Ask }`. + There is no `Deny`, so "this hook never blocks outright" is a guarantee, not a comment. +- **Adding a branch is still one line.** Descriptions live next to the variant in `branches!`, and + the generated `branch()` is an exhaustive `match self`, so no variant can ship without one. + +## How it works + +Same design as the Python library: every variant's description becomes an option of one Choice +question (plus `none_of_these`), each `when` guard becomes a Noul question in the same request, +and Jev answers them all in one call. A call resolves to one `Verdict`: + +| Verdict | When | +| ------------------ | ------------------------------------------------------------- | +| `Matched(b)` | `b` ranks first, clears its `min_confidence`, and its guard holds | +| `LowConfidence(b)` | `b` ranks first but is below its `min_confidence` | +| `GuardFailed(b)` | `b` is confident but its `when` statement is false | +| `NoMatch` | the model picks "none of these" | + +`Match` carries the verdict alongside the confidence, the threshold it needed, the guard +probability, every option's probability, the model version and latency. `Matcher::builder` sets +the default `min_confidence` (0.7), `guard_threshold` (0.5), the "none of these" description, and +an `observer` called after every decision for logging. State is anything `Serialize`: a `&str`, +`serde_json::Value`, or your own struct. + +### Branch options + +```rust +Destructive => Branch::new("deletes or overwrites data in a way that may not be recoverable") + .min_confidence(0.5) // certainty this branch needs + .when("The command targets files outside the project") // model-evaluated guard + .examples(["rm -rf build", "git push --force"]) // to separate similar branches + .not_for("commands that only move files"), // what belongs to a neighbour +``` + +Labels sent to the model are the variant names (`Refund`, `OrderStatus`). You can implement the +`Branches` trait by hand instead of using the macro, e.g. to choose your own labels. + +## Examples + +- [`examples/support_router.rs`](examples/support_router.rs): support triage with thresholds scaled + to risk, a guard (refunds need an order number), and escalation that carries the ranked options. +- [`examples/shell_guard.rs`](examples/shell_guard.rs): the **Claude Code PreToolUse hook** that + judges Bash commands by what they'd do. It asks before destructive commands, never denies, and + falls back to Claude Code's normal permission flow on any error. As a compiled binary it starts + in milliseconds, which matters for a hook that runs before every command. + +```bash +export TYPESAFE_API_KEY=sk-... +cargo run --example support_router +echo '{"tool_name":"Bash","tool_input":{"command":"git clean -fdx"},"cwd":"/tmp"}' | cargo run --example shell_guard +``` + +## Testing without a network + +The matcher depends on the small `Decider` trait, and `JevDecider` is one implementation. Tests use +`ScriptedDecider`, which also records every call: + +```rust +use fuzzymatch::testing::{ScriptedDecider, choose}; + +let decider = ScriptedDecider::new(choose(Intent::Refund, 0.95).with_guard(Intent::Refund, 0.9)); +let support = Matcher::::new("What is the customer asking for?", decider)?; +assert_eq!(support.classify("Order #1 arrived broken")?.verdict, Verdict::Matched(Intent::Refund)); +assert_eq!(support.decider().calls().len(), 1); +``` + +Build with `default-features = false` to drop the Jev adapter (and its HTTP client) and bring your +own decider. + +```bash +cargo test +``` + +## Limits + +Everything in the Python version's [Limits](../README.md#limits) applies: pin the model before +tuning thresholds, Jev's own limits, and up to 254 branches. In addition: + +- The Jev adapter talks to the HTTP API directly (there's no official Rust SDK), without the Python + SDK's automatic retries. +- Blocking only for now (`ureq`); an async `Decider` is a natural next step. +- Branch enums must be fieldless: the model picks a variant, it doesn't fill in data. diff --git a/rust/examples/shell_guard.rs b/rust/examples/shell_guard.rs new file mode 100644 index 0000000..612dcac --- /dev/null +++ b/rust/examples/shell_guard.rs @@ -0,0 +1,186 @@ +//! A Claude Code PreToolUse hook that judges Bash commands by what they would do. +//! +//! Pattern rules like `Bash(rm *)` miss `find . -delete`, `git clean -fdx`, `> important.txt` and +//! many more ways to destroy data. This hook asks what the command would do instead, and scales the +//! certainty it needs to the cost of being wrong: +//! +//! * destructive, even at modest confidence -> "ask": you approve it yourself +//! * read-only, at very high confidence -> "allow" (only if FUZZYMATCH_AUTO_ALLOW=1) +//! * anything else, or any error -> no decision: Claude Code's normal permissions apply +//! +//! It never denies outright (there is no `Deny` in [`Permission`]) and fails open to the normal +//! permission flow, so a wrong judgment or a network error can't do more than the settings you +//! already have allow. +//! +//! Build it, then install in .claude/settings.json: +//! +//! cargo build --release --example shell_guard +//! +//! {"hooks": {"PreToolUse": [{"matcher": "Bash", "hooks": [{"type": "command", +//! "command": "/path/to/fuzzymatch/rust/target/release/examples/shell_guard"}]}]}} +//! +//! Try it without Claude Code: +//! +//! echo '{"tool_name":"Bash","tool_input":{"command":"git clean -fdx"},"cwd":"/tmp"}' \ +//! | cargo run --example shell_guard + +use std::io::Read; + +use fuzzymatch::{Branch, Decider, JevDecider, Match, Matcher, Verdict, branches}; +use serde::Serialize; +use serde_json::{Value, json}; + +branches! { + /// What running a command would do to the machine and its files. + enum Effect { + ReadOnly => Branch::new("only reads, lists, searches or prints; changes no files, settings or remote state") + .min_confidence(0.95) + .examples(["ls -la", "git status", "grep -rn TODO src", "cat README.md"]), + Reversible => Branch::new("changes files or state in a way that is easy to undo or regenerate") + .examples(["git add -A", "npm install", "mkdir build", "git commit -m 'wip'"]), + Destructive => Branch::new("deletes, overwrites or rewrites data or history in a way that may not be recoverable") + // Asking costs one click; missing a destructive command costs much more. + .min_confidence(0.5) + .examples(["rm -rf build", "git push --force", "git reset --hard", "find . -name '*.log' -delete"]), + } +} + +/// The decisions this hook can make. Deliberately no `Deny`: the worst a wrong call can do is ask. +#[derive(Clone, Copy, Debug, PartialEq, Serialize)] +#[serde(rename_all = "lowercase")] +enum Permission { + Allow, + Ask, +} + +fn shell(decider: D) -> Matcher { + Matcher::builder("What would running this shell command do to the machine and its files?") + .build(decider) + .expect("valid branches") +} + +/// `None` means no opinion: Claude Code's normal permission flow decides. +fn judge(m: &Match, auto_allow: bool) -> Option<(Permission, String)> { + match m.verdict { + Verdict::Matched(Effect::Destructive) => Some(( + Permission::Ask, + format!("fuzzymatch judged this destructive ({:.0}% confident)", m.confidence * 100.0), + )), + Verdict::Matched(Effect::ReadOnly) if auto_allow => { + Some((Permission::Allow, "judged read-only".into())) + } + Verdict::Matched(Effect::ReadOnly | Effect::Reversible) => None, + Verdict::LowConfidence(_) | Verdict::GuardFailed(_) | Verdict::NoMatch => None, + } +} + +/// The hook's stdout for one PreToolUse event, if it has anything to say. +fn hook(shell: &Matcher, event: &Value, auto_allow: bool) -> Option { + if event["tool_name"] != "Bash" { + return None; + } + // Only the command itself is judged. The agent's own description of the command is left out on + // purpose: text in the state can argue for an answer, and the agent writes that description. + let state = json!({ + "command": event["tool_input"]["command"].as_str().unwrap_or(""), + "working_directory": event["cwd"].as_str().unwrap_or(""), + }); + let m = shell + .classify(&state) + .inspect_err(|error| eprintln!("fuzzymatch shell_guard: {error}")) // fail open + .ok()?; + let (permission, reason) = judge(&m, auto_allow)?; + Some(json!({ + "hookSpecificOutput": { + "hookEventName": "PreToolUse", + "permissionDecision": permission, + "permissionDecisionReason": reason, + } + })) +} + +fn main() { + let mut input = String::new(); + let event: Value = match std::io::stdin().read_to_string(&mut input).map(|_| serde_json::from_str(&input)) + { + Ok(Ok(event)) => event, + Ok(Err(error)) => return eprintln!("fuzzymatch shell_guard: invalid hook input: {error}"), + Err(error) => return eprintln!("fuzzymatch shell_guard: {error}"), + }; + let decider = match JevDecider::from_env() { + Ok(decider) => decider, + Err(error) => return eprintln!("fuzzymatch shell_guard: {error}"), + }; + let auto_allow = std::env::var("FUZZYMATCH_AUTO_ALLOW").as_deref() == Ok("1"); + if let Some(output) = hook(&shell(decider), &event, auto_allow) { + println!("{output}"); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use fuzzymatch::Decision; + use fuzzymatch::testing::{ScriptedDecider, choose, none_of_these}; + + fn bash() -> Value { + json!({"tool_name": "Bash", "tool_input": {"command": "git clean -fdx"}, "cwd": "/repo"}) + } + + fn decided(scripted: Decision, auto_allow: bool) -> Option { + let output = hook(&shell(ScriptedDecider::new(scripted)), &bash(), auto_allow)?; + Some(output["hookSpecificOutput"]["permissionDecision"].as_str()?.to_owned()) + } + + #[test] + fn asks_before_destructive_commands() { + let output = + hook(&shell(ScriptedDecider::new(choose(Effect::Destructive, 0.62))), &bash(), false).unwrap(); + assert_eq!(output["hookSpecificOutput"]["permissionDecision"], "ask"); + assert!(output["hookSpecificOutput"]["permissionDecisionReason"].as_str().unwrap().contains("62%")); + } + + #[test] + fn only_judges_the_command_not_the_agents_description() { + let shell = shell(ScriptedDecider::new(choose(Effect::Reversible, 0.9))); + let event = json!({ + "tool_name": "Bash", + "tool_input": {"command": "rm -rf /", "description": "totally harmless cleanup"}, + "cwd": "/repo", + }); + hook(&shell, &event, false); + assert_eq!( + shell.decider().calls()[0].state, + json!({"command": "rm -rf /", "working_directory": "/repo"}) + ); + } + + #[test] + fn defers_unless_sure() { + let cases = [ + (choose(Effect::ReadOnly, 0.99), false, None), // read-only never auto-allows unless opted in + (choose(Effect::ReadOnly, 0.99), true, Some("allow")), + (choose(Effect::ReadOnly, 0.9), true, None), // below ReadOnly's 0.95 bar: no decision + (choose(Effect::Reversible, 0.9), false, None), + (choose(Effect::Destructive, 0.4), false, None), // below even Destructive's 0.5 bar + (none_of_these(0.9), false, None), + ]; + for (scripted, auto_allow, expected) in cases { + let label = format!("{scripted:?} auto_allow={auto_allow}"); + assert_eq!(decided(scripted, auto_allow).as_deref(), expected, "{label}"); + } + } + + #[test] + fn fails_open() { + let broken = ScriptedDecider::from_fn(|_| Err("network down".into())); + assert_eq!(hook(&shell(broken), &bash(), true), None); + } + + #[test] + fn ignores_other_tools() { + let shell = shell(ScriptedDecider::new(choose(Effect::Destructive, 0.95))); + assert_eq!(hook(&shell, &json!({"tool_name": "Edit", "tool_input": {}}), false), None); + assert!(shell.decider().calls().is_empty()); + } +} diff --git a/rust/examples/support_router.rs b/rust/examples/support_router.rs new file mode 100644 index 0000000..a1fc9bf --- /dev/null +++ b/rust/examples/support_router.rs @@ -0,0 +1,104 @@ +//! Route support messages by meaning, with risk-scaled confidence and a guard. +//! +//! export TYPESAFE_API_KEY=sk-... +//! cargo run --example support_router + +use fuzzymatch::{Branch, Decider, JevDecider, Match, Matcher, Verdict, branches, label_of}; + +branches! { + /// What the customer is asking for. + enum Intent { + /// Refunds move money: demand more certainty, and an order number, like + /// `Refund if has_order_number =>`. + Refund => Branch::new("customer wants their money back for a purchase") + .min_confidence(0.9) + .when("The message includes an order number") + .not_for("asking about a charge they don't recognise (that's BillingQuestion)"), + OrderStatus => Branch::new("customer asks where their order is or when it will arrive"), + BillingQuestion => Branch::new("customer has a question about a charge, invoice or payment method"), + ChurnRisk => Branch::new("customer says they will cancel or leave for a competitor").min_confidence(0.6), + } +} + +fn support(decider: D) -> Matcher { + Matcher::builder("What is the customer asking for?").build(decider).expect("valid branches") +} + +/// Add a variant to `Intent` and this stops compiling until you decide where it goes. +fn route(m: &Match) -> String { + match m.verdict { + Verdict::Matched(Intent::Refund) => "→ refund workflow".into(), + Verdict::Matched(Intent::OrderStatus) => "→ order lookup (no model needed)".into(), + Verdict::Matched(Intent::BillingQuestion) => "→ billing queue".into(), + Verdict::Matched(Intent::ChurnRisk) => "→ page the retention team".into(), + Verdict::LowConfidence(_) | Verdict::GuardFailed(_) | Verdict::NoMatch => escalate(m), + } +} + +/// Anything uncertain, off-topic, or missing a guard goes to a stronger model or a human, with the +/// ranked options as a head start. +fn escalate(m: &Match) -> String { + let top: Vec<_> = + m.ranked().iter().take(3).map(|&(pick, p)| format!("{} {:.0}%", label_of(pick), p * 100.0)).collect(); + format!("→ escalate ({}; {})", m.verdict, top.join(", ")) +} + +const MESSAGES: &[&str] = &[ + "Order #88213 arrived broken. I want a refund.", + "I want my money back.", // refund without an order number: the guard fails + "Where's my package? It's been two weeks.", + "What is this $14.99 charge on my card?", + "Honestly I'm about done with you people, your competitor is half the price.", + "Do you have a recipe for banana bread?", +]; + +fn main() -> Result<(), Box> { + let support = support(JevDecider::from_env()?); + for message in MESSAGES { + let m = support.classify(message)?; + println!("{message}"); + println!( + " [{}] top={} confidence={:.2} (needs {:.2}) {}ms {}", + m.verdict, + label_of(m.verdict.top()), + m.confidence, + m.threshold, + m.latency.as_millis(), + m.model.as_deref().unwrap_or("-"), + ); + println!(" {}", route(&m)); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use fuzzymatch::testing::{ScriptedDecider, choose}; + + fn reply(decision: fuzzymatch::Decision, message: &str) -> String { + route(&support(ScriptedDecider::new(decision)).classify(message).unwrap()) + } + + #[test] + fn refund_needs_an_order_number() { + let with_number = choose(Intent::Refund, 0.95).with_guard(Intent::Refund, 0.9); + let without_number = choose(Intent::Refund, 0.95).with_guard(Intent::Refund, 0.1); + + assert_eq!(reply(with_number, "Order #88213 arrived broken"), "→ refund workflow"); + assert!(reply(without_number, "I want my money back").starts_with("→ escalate (guard_failed")); + } + + #[test] + fn escalation_carries_the_ranked_options() { + let unsure = choose(Intent::Refund, 0.6).with_guard(Intent::Refund, 0.9).with_probabilities([ + (Some(Intent::Refund), 0.6), + (Some(Intent::BillingQuestion), 0.3), + (None, 0.1), + ]); + assert_eq!( + reply(unsure, "hmm"), + "→ escalate (low_confidence; Refund 60%, BillingQuestion 30%, none_of_these 10%)" + ); + } +} diff --git a/rust/rustfmt.toml b/rust/rustfmt.toml new file mode 100644 index 0000000..caafbf6 --- /dev/null +++ b/rust/rustfmt.toml @@ -0,0 +1,2 @@ +max_width = 110 +use_small_heuristics = "Max" diff --git a/rust/src/branch.rs b/rust/src/branch.rs new file mode 100644 index 0000000..3704b54 --- /dev/null +++ b/rust/src/branch.rs @@ -0,0 +1,201 @@ +//! Branches: the arms a matcher can pick, declared as variants of your own enum. + +use std::fmt::Debug; + +use serde::Serialize; + +/// Label of the implicit "none of these" option offered alongside every branch. +pub const NONE_LABEL: &str = "none_of_these"; + +/// Jev Choice questions take up to 255 options; one is reserved for [`NONE_LABEL`]. +pub const MAX_BRANCHES: usize = 254; + +/// A fieldless enum whose variants are the branches of a matcher. +/// +/// Implement it with the [`branches!`](crate::branches!) macro, which generates `ALL`, `label` and +/// `branch` from one declaration so they can't drift apart. A hand-written impl works too; the +/// `match self` in `branch` is exhaustive, so a new variant can't ship without a description. +pub trait Branches: Copy + Eq + Debug + 'static { + /// Every variant, in the order they are offered to the model. + const ALL: &'static [Self]; + + /// Option label sent to the model. Must be an identifier, unique within the enum. + fn label(self) -> &'static str; + + /// What belongs in this branch, and how sure the model must be to pick it. + fn branch(self) -> Branch; +} + +/// The label of a ranked option: a branch's label, or [`NONE_LABEL`] for `None`. +pub fn label_of(pick: Option) -> &'static str { + pick.map_or(NONE_LABEL, B::label) +} + +pub(crate) fn guard_key(label: &str) -> String { + format!("guard__{label}") +} + +/// Describes one branch: what belongs in it, and what it takes to run it. +/// +/// ``` +/// use fuzzymatch::Branch; +/// +/// Branch::new("deletes or overwrites data in a way that may not be recoverable") +/// .min_confidence(0.5) // certainty this branch needs (default: the matcher's 0.7) +/// .when("The command targets files outside the project") // guard, like `x if cond =>` +/// .examples(["rm -rf build", "git push --force"]) // to separate similar branches +/// .not_for("commands that only move files"); // what belongs to a neighbour +/// ``` +#[derive(Clone, Debug, PartialEq)] +pub struct Branch { + description: String, + min_confidence: Option, + when: Option, + examples: Vec, + not_for: Option, +} + +impl Branch { + /// A branch for states that fit `description`, in plain language. + pub fn new(description: impl Into) -> Self { + Self { + description: description.into(), + min_confidence: None, + when: None, + examples: Vec::new(), + not_for: None, + } + } + + /// Confidence needed to match this branch. Set it high for handlers that are hard to undo. + pub fn min_confidence(mut self, confidence: f64) -> Self { + self.min_confidence = Some(confidence); + self + } + + /// A statement that must also be true, like a guard on a match arm (`x if cond =>`). + /// Evaluated in the same request as the choice. + pub fn when(mut self, statement: impl Into) -> Self { + self.when = Some(statement.into()); + self + } + + /// Example inputs, to separate this branch from similar ones. + pub fn examples(mut self, examples: I) -> Self + where + I: IntoIterator, + S: Into, + { + self.examples = examples.into_iter().map(Into::into).collect(); + self + } + + /// What belongs to a neighbouring branch instead. + pub fn not_for(mut self, boundary: impl Into) -> Self { + self.not_for = Some(boundary.into()); + self + } + + pub fn description(&self) -> &str { + &self.description + } + + pub fn required_confidence(&self) -> Option { + self.min_confidence + } + + pub fn guard(&self) -> Option<&str> { + self.when.as_deref() + } + + /// The option description sent to the model. + /// + /// Plain text by default. With examples or a `not_for` boundary, a structured object, which + /// TypeSafe recommends when neighbouring options keep getting confused. + pub fn criterion(&self) -> Criterion { + if self.examples.is_empty() && self.not_for.is_none() { + return Criterion::Plain(self.description.clone()); + } + Criterion::Detailed { + description: self.description.clone(), + examples: self.examples.clone(), + not_for: self.not_for.clone(), + } + } +} + +/// How an option is described to the model: a sentence, or a description with examples and a +/// boundary. Serializes to a JSON string or object respectively. +#[derive(Clone, Debug, PartialEq, Serialize)] +#[serde(untagged)] +pub enum Criterion { + Plain(String), + Detailed { + description: String, + #[serde(skip_serializing_if = "Vec::is_empty")] + examples: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + not_for: Option, + }, +} + +impl From<&str> for Criterion { + fn from(description: &str) -> Self { + Criterion::Plain(description.to_owned()) + } +} + +/// Declare an enum whose variants are the branches of a matcher, each with its [`Branch`]. +/// +/// ``` +/// use fuzzymatch::{Branch, Branches, branches}; +/// +/// branches! { +/// /// What the customer is asking for. +/// pub enum Intent { +/// Refund => Branch::new("customer wants their money back").min_confidence(0.9), +/// OrderStatus => Branch::new("customer asks where their order is"), +/// } +/// } +/// +/// assert_eq!(Intent::ALL, &[Intent::Refund, Intent::OrderStatus]); +/// assert_eq!(Intent::OrderStatus.label(), "OrderStatus"); +/// ``` +/// +/// The enum derives `Clone, Copy, Debug, PartialEq, Eq, Hash`, so don't derive those again. Labels +/// are the variant names, so they are always valid, unique identifiers, and the macro needs at +/// least one variant, so an empty dispatch table doesn't compile. +#[macro_export] +macro_rules! branches { + ( + $(#[$meta:meta])* + $vis:vis enum $name:ident { + $( + $(#[$variant_meta:meta])* + $variant:ident => $branch:expr + ),+ $(,)? + } + ) => { + $(#[$meta])* + #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] + $vis enum $name { + $( $(#[$variant_meta])* $variant, )+ + } + + impl $crate::Branches for $name { + const ALL: &'static [Self] = &[$(Self::$variant),+]; + + fn label(self) -> &'static str { + match self { + $(Self::$variant => stringify!($variant),)+ + } + } + + fn branch(self) -> $crate::Branch { + match self { + $(Self::$variant => $branch,)+ + } + } + } + }; +} diff --git a/rust/src/decider.rs b/rust/src/decider.rs new file mode 100644 index 0000000..5bcd12d --- /dev/null +++ b/rust/src/decider.rs @@ -0,0 +1,105 @@ +//! The port between the matcher and whatever model makes the judgment. +//! +//! The matcher never talks to a model directly. It hands a [`Decider`] the state, the options (one +//! per branch, plus "none of these"), and any guard statements, and gets back a [`Decision`]. That +//! keeps the dispatch logic testable without a network and lets the model behind it be swapped. +//! +//! The port speaks in string labels, not your enum: that's the boundary where untyped model output +//! comes in, and the matcher is what turns it back into `B`. + +use std::collections::HashMap; +use std::sync::Arc; + +use serde_json::Value; + +use crate::branch::{Branches, Criterion, guard_key, label_of}; + +/// Any error a decider can report. +pub type BoxError = Box; + +/// One dispatch question, as the matcher asks it. +#[derive(Clone, Copy, Debug)] +pub struct Request<'a> { + /// Text or JSON data to judge. + pub state: &'a Value, + /// The dispatch question, e.g. "What is the customer asking for?". + pub instructions: &'a str, + /// Label and description of every option, in order, ending with "none of these". + pub options: &'a [(&'static str, Criterion)], + /// Guard name and a statement that is either true or false of the state. + pub guards: &'a [(String, String)], +} + +/// Anything that can pick one option for a state and check guard statements alongside it. +pub trait Decider { + /// Choose one of `request.options` and evaluate every guard in `request.guards`. + fn decide(&self, request: &Request<'_>) -> Result; +} + +impl Decider for &D { + fn decide(&self, request: &Request<'_>) -> Result { + (**self).decide(request) + } +} + +impl Decider for Box { + fn decide(&self, request: &Request<'_>) -> Result { + (**self).decide(request) + } +} + +impl Decider for Arc { + fn decide(&self, request: &Request<'_>) -> Result { + (**self).decide(request) + } +} + +/// A model's answer to one dispatch question, plus its guard checks. +#[derive(Clone, Debug, PartialEq)] +pub struct Decision { + /// Label of the most probable option. + pub choice: String, + /// How concentrated the distribution is (0-1), as reported by the model. + pub confidence: f64, + /// Probability of every option, keyed by label. + pub probabilities: Vec<(String, f64)>, + /// Probability (0-1) that each guard statement is true, keyed by guard name. + pub guards: HashMap, + /// The model version that answered, for audit logs. + pub model: Option, +} + +impl Decision { + /// A decision for `choice`, with probabilities defaulting to `{choice: confidence}`. + pub fn new(choice: impl Into, confidence: f64) -> Self { + let choice = choice.into(); + Self { + probabilities: vec![(choice.clone(), confidence)], + choice, + confidence, + guards: HashMap::new(), + model: None, + } + } + + /// Record the probability that `branch`'s guard holds. + pub fn with_guard(mut self, branch: B, probability: f64) -> Self { + self.guards.insert(guard_key(branch.label()), probability); + self + } + + /// Replace the probability distribution. `None` stands for "none of these". + pub fn with_probabilities( + mut self, + probabilities: impl IntoIterator, f64)>, + ) -> Self { + self.probabilities = + probabilities.into_iter().map(|(pick, p)| (label_of(pick).to_owned(), p)).collect(); + self + } + + pub fn with_model(mut self, model: impl Into) -> Self { + self.model = Some(model.into()); + self + } +} diff --git a/rust/src/jev.rs b/rust/src/jev.rs new file mode 100644 index 0000000..faf6e36 --- /dev/null +++ b/rust/src/jev.rs @@ -0,0 +1,319 @@ +//! The TypeSafe Jev adapter for the [`Decider`] port. +//! +//! One request carries the dispatch question (a Choice over every branch plus "none of these") and +//! every guard (one Noul each). Jev answers them all in parallel, so guards add input tokens but not +//! latency. There is no official Rust SDK, so this speaks the documented wire format directly. + +use std::collections::HashMap; +use std::time::Duration; + +use indexmap::IndexMap; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::branch::Criterion; +use crate::decider::{BoxError, Decider, Decision, Request}; + +/// Name of the dispatch question in the request. Guards can't use it. +pub const DISPATCH_KEY: &str = "branch"; + +pub const API_KEY_ENV: &str = "TYPESAFE_API_KEY"; +pub const BASE_URL_ENV: &str = "TYPESAFE_BASE_URL"; +pub const DEFAULT_MODEL_ENV: &str = "TYPESAFE_DEFAULT_MODEL"; +pub const DEFAULT_BASE_URL: &str = "https://api.typesafe.ai"; +pub const DEFAULT_MODEL: &str = "jev-latest"; +const SYSTEM_ONE_PATH: &str = "/v1/systemone"; +const DEFAULT_TIMEOUT: Duration = Duration::from_secs(10); + +/// What went wrong talking to Jev. +#[derive(Debug, thiserror::Error)] +pub enum JevError { + #[error("no API key was provided; pass one or set the {API_KEY_ENV} environment variable")] + MissingApiKey, + #[error("API key must contain only printable ASCII characters without whitespace")] + InvalidApiKey, + #[error("guard name {DISPATCH_KEY:?} is reserved")] + ReservedGuardName, + #[error("request to TypeSafe failed: {0}")] + Http(#[from] ureq::Error), + #[error("TypeSafe returned HTTP {status}: {body}")] + Status { status: u16, body: String }, + #[error("TypeSafe response was not valid: {0}")] + Decode(#[source] serde_json::Error), + #[error("TypeSafe response has no answer for {0:?}")] + MissingAnswer(String), + #[error("TypeSafe answered {name:?} with a {found} answer, expected {expected}")] + WrongAnswerType { name: String, expected: &'static str, found: &'static str }, +} + +/// Makes dispatch decisions with a TypeSafe System One model. +/// +/// Pin the model with [`model`](Self::model) if you tune thresholds, since a moving alias like +/// `jev-latest` can shift confidence under you. +#[derive(Clone)] +pub struct JevDecider { + agent: ureq::Agent, + api_key: String, + base_url: String, + model: String, +} + +impl JevDecider { + /// A decider configured from `TYPESAFE_API_KEY`, and optionally `TYPESAFE_BASE_URL` and + /// `TYPESAFE_DEFAULT_MODEL`. + pub fn from_env() -> Result { + Self::from_lookup(|name| std::env::var(name).ok()) + } + + fn from_lookup(lookup: impl Fn(&str) -> Option) -> Result { + let env = |name| lookup(name).map(|v| v.trim().to_owned()).filter(|v| !v.is_empty()); + let mut decider = Self::new(env(API_KEY_ENV).ok_or(JevError::MissingApiKey)?)?; + if let Some(url) = env(BASE_URL_ENV) { + decider = decider.base_url(url); + } + if let Some(model) = env(DEFAULT_MODEL_ENV) { + decider = decider.model(model); + } + Ok(decider) + } + + /// A decider with an explicit API key. + pub fn new(api_key: impl Into) -> Result { + let api_key = api_key.into().trim().to_owned(); + if api_key.is_empty() { + return Err(JevError::MissingApiKey); + } + if !api_key.chars().all(|c| c.is_ascii_graphic()) { + return Err(JevError::InvalidApiKey); + } + let config = ureq::Agent::config_builder() + .timeout_global(Some(DEFAULT_TIMEOUT)) + .http_status_as_error(false) + .build(); + Ok(Self { + agent: config.into(), + api_key, + base_url: DEFAULT_BASE_URL.to_owned(), + model: DEFAULT_MODEL.to_owned(), + }) + } + + /// Use a specific model, e.g. a pinned `jev-...` version. + pub fn model(mut self, model: impl Into) -> Self { + self.model = model.into(); + self + } + + /// The API root. Defaults to `https://api.typesafe.ai`. + pub fn base_url(mut self, url: impl Into) -> Self { + self.base_url = url.into().trim_end_matches('/').to_owned(); + self + } + + /// Use your own HTTP agent, e.g. for a proxy or a different timeout. + pub fn agent(mut self, agent: ureq::Agent) -> Self { + self.agent = agent; + self + } + + fn system_one(&self, request: &Request<'_>) -> Result { + if request.guards.iter().any(|(name, _)| name == DISPATCH_KEY) { + return Err(JevError::ReservedGuardName); + } + let body = request_body(request, &self.model); + + let mut response = self + .agent + .post(format!("{}{SYSTEM_ONE_PATH}", self.base_url)) + .header("Authorization", format!("Bearer {}", self.api_key)) + .header("Accept", "application/json") + .header("User-Agent", concat!("fuzzymatch-rust/", env!("CARGO_PKG_VERSION"))) + .send_json(&body) + .map_err(|error| match error { + ureq::Error::StatusCode(status) => JevError::Status { status, body: String::new() }, + other => JevError::Http(other), + })?; + let status = response.status().as_u16(); + let text = response.body_mut().read_to_string()?; + if !(200..300).contains(&status) { + return Err(JevError::Status { status, body: text.chars().take(200).collect() }); + } + parse_response(&text, request.guards) + } +} + +impl Decider for JevDecider { + fn decide(&self, request: &Request<'_>) -> Result { + Ok(self.system_one(request)?) + } +} + +impl std::fmt::Debug for JevDecider { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // The API key is left out on purpose. + f.debug_struct("JevDecider").field("base_url", &self.base_url).field("model", &self.model).finish() + } +} + +// -- wire format ------------------------------------------------------------------------------ + +#[derive(Serialize)] +struct Body<'a> { + state: &'a Value, + model: &'a str, + questions: IndexMap<&'a str, Question<'a>>, +} + +#[derive(Serialize)] +#[serde(tag = "type", rename_all = "lowercase")] +enum Question<'a> { + Choice { instructions: &'a str, criteria: IndexMap<&'a str, &'a Criterion> }, + Noul { instructions: &'a str }, +} + +#[derive(Deserialize)] +struct Response { + model: String, + #[serde(default)] + answers: HashMap, +} + +#[derive(Deserialize)] +#[serde(tag = "type", rename_all = "lowercase")] +enum Answer { + Choice { + choice: String, + confidence: f64, + probabilities: IndexMap, + }, + Noul { + noul: f64, + }, + /// Score answers, and any type added after this was written. + #[serde(other)] + Other, +} + +impl Answer { + fn kind(&self) -> &'static str { + match self { + Answer::Choice { .. } => "choice", + Answer::Noul { .. } => "noul", + Answer::Other => "unrecognised", + } + } +} + +fn request_body<'a>(request: &'a Request<'_>, model: &'a str) -> Body<'a> { + let criteria = request.options.iter().map(|(label, criterion)| (*label, criterion)).collect(); + let mut questions = IndexMap::new(); + questions.insert(DISPATCH_KEY, Question::Choice { instructions: request.instructions, criteria }); + for (name, statement) in request.guards { + questions.insert(name.as_str(), Question::Noul { instructions: statement }); + } + Body { state: request.state, model, questions } +} + +fn parse_response(text: &str, guards: &[(String, String)]) -> Result { + let mut response: Response = serde_json::from_str(text).map_err(JevError::Decode)?; + let mut take = + |name: &str| response.answers.remove(name).ok_or_else(|| JevError::MissingAnswer(name.to_owned())); + let wrong = |name: &str, expected, found: &Answer| JevError::WrongAnswerType { + name: name.to_owned(), + expected, + found: found.kind(), + }; + + let (choice, confidence, probabilities) = match take(DISPATCH_KEY)? { + Answer::Choice { choice, confidence, probabilities } => (choice, confidence, probabilities), + other => return Err(wrong(DISPATCH_KEY, "choice", &other)), + }; + let guards = guards + .iter() + .map(|(name, _)| match take(name)? { + Answer::Noul { noul } => Ok((name.clone(), noul)), + other => Err(wrong(name, "noul", &other)), + }) + .collect::>()?; + + Ok(Decision { + choice, + confidence, + probabilities: probabilities.into_iter().collect(), + guards, + model: Some(response.model), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn lookup(vars: &[(&str, &str)]) -> impl Fn(&str) -> Option { + let vars: HashMap = + vars.iter().map(|(k, v)| (k.to_string(), v.to_string())).collect(); + move |name| vars.get(name).cloned() + } + + #[test] + fn needs_an_api_key() { + assert!(matches!(JevDecider::from_lookup(lookup(&[])), Err(JevError::MissingApiKey))); + assert!(matches!( + JevDecider::from_lookup(lookup(&[(API_KEY_ENV, " ")])), + Err(JevError::MissingApiKey) + )); + assert!(matches!(JevDecider::new("sk bad"), Err(JevError::InvalidApiKey))); + } + + #[test] + fn reads_base_url_and_model_from_the_environment() { + let decider = JevDecider::from_lookup(lookup(&[ + (API_KEY_ENV, " sk-test "), + (BASE_URL_ENV, "http://localhost:9/"), + (DEFAULT_MODEL_ENV, "jev-1.13.0"), + ])) + .unwrap(); + assert_eq!(decider.api_key, "sk-test"); + assert_eq!(decider.base_url, "http://localhost:9"); + assert_eq!(decider.model, "jev-1.13.0"); + assert!(!format!("{decider:?}").contains("sk-test")); + } + + #[test] + fn defaults_to_the_moving_alias() { + let decider = JevDecider::from_lookup(lookup(&[(API_KEY_ENV, "sk-test")])).unwrap(); + assert_eq!(decider.model, DEFAULT_MODEL); + assert_eq!(decider.base_url, DEFAULT_BASE_URL); + } + + #[test] + fn ignores_answer_types_it_does_not_model() { + let text = r#"{"model": "jev", "answers": { + "branch": {"type": "choice", "choice": "a", "confidence": 0.9, "probabilities": {"a": 0.9}}, + "extra": {"type": "score", "score": 3, "legend": {}, "probabilities": {}} + }}"#; + assert_eq!(parse_response(text, &[]).unwrap().choice, "a"); + } + + #[test] + fn rejects_a_guard_answered_with_the_wrong_type() { + let text = r#"{"model": "jev", "answers": { + "branch": {"type": "choice", "choice": "a", "confidence": 0.9, "probabilities": {"a": 0.9}}, + "guard__a": {"type": "choice", "choice": "a", "confidence": 0.9, "probabilities": {}} + }}"#; + let guards = [("guard__a".to_owned(), "It is raining".to_owned())]; + let error = parse_response(text, &guards).unwrap_err(); + assert!(matches!(error, JevError::WrongAnswerType { expected: "noul", found: "choice", .. })); + } + + #[test] + fn reports_a_missing_guard_answer() { + let text = r#"{"model": "jev", "answers": { + "branch": {"type": "choice", "choice": "a", "confidence": 0.9, "probabilities": {"a": 0.9}} + }}"#; + let guards = [("guard__a".to_owned(), "It is raining".to_owned())]; + assert!( + matches!(parse_response(text, &guards), Err(JevError::MissingAnswer(name)) if name == "guard__a") + ); + } +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs new file mode 100644 index 0000000..0f5b750 --- /dev/null +++ b/rust/src/lib.rs @@ -0,0 +1,69 @@ +//! Pattern matching on meaning. +//! +//! Declare the branches as variants of your own enum, each with a plain-language description. +//! A [`Matcher`] asks a model which one a state belongs to and hands back a [`Verdict`], and you +//! dispatch on it with an ordinary `match`: +//! +//! ``` +//! use fuzzymatch::{Branch, Matcher, Verdict, branches}; +//! # use fuzzymatch::testing::{ScriptedDecider, choose}; +//! +//! branches! { +//! enum Intent { +//! Refund => Branch::new("customer wants their money back") +//! .min_confidence(0.9) +//! .when("The message includes an order number"), +//! OrderStatus => Branch::new("customer asks where their order is"), +//! } +//! } +//! +//! # fn main() -> Result<(), Box> { +//! # let decider = ScriptedDecider::new(choose(Intent::Refund, 0.95).with_guard(Intent::Refund, 0.9)); +//! let support = Matcher::::new("What is the customer asking for?", decider)?; +//! +//! let m = support.classify("Order #88213 arrived broken. I want a refund.")?; +//! let reply = match m.verdict { +//! Verdict::Matched(Intent::Refund) => "refund workflow", +//! Verdict::Matched(Intent::OrderStatus) => "order lookup", +//! Verdict::LowConfidence(Intent::Refund) => "ask a person to confirm the refund", +//! Verdict::GuardFailed(Intent::Refund) => "ask for the order number", +//! Verdict::LowConfidence(_) | Verdict::GuardFailed(_) | Verdict::NoMatch => "escalate", +//! }; +//! assert_eq!(reply, "refund workflow"); +//! # Ok(()) +//! # } +//! ``` +//! +//! The `match` is checked for exhaustiveness like any other, so forgetting a branch, or forgetting +//! to handle the ways a call can *fail* to match, doesn't compile: +//! +//! ```compile_fail,E0004 +//! # use fuzzymatch::{Branch, Verdict, branches}; +//! # branches! { enum Intent { Refund => Branch::new("a"), OrderStatus => Branch::new("b") } } +//! fn route(verdict: Verdict) -> &'static str { +//! match verdict { +//! Verdict::Matched(Intent::Refund) => "refund workflow", +//! Verdict::Matched(Intent::OrderStatus) => "order lookup", +//! // error[E0004]: `Verdict::LowConfidence(_)`, `Verdict::GuardFailed(_)` and +//! // `Verdict::NoMatch` not covered +//! } +//! } +//! ``` +//! +//! All branch descriptions become the options of one Choice question and every guard a yes/no +//! question in the same request, so adding a branch costs a few input tokens rather than another +//! round trip. The model's calibrated confidence decides whether the top branch matches, and each +//! branch can demand its own confidence, so risky branches need more certainty than harmless ones. + +mod branch; +mod decider; +#[cfg(feature = "jev")] +pub mod jev; +mod matcher; +pub mod testing; + +pub use branch::{Branch, Branches, Criterion, MAX_BRANCHES, NONE_LABEL, label_of}; +pub use decider::{BoxError, Decider, Decision, Request}; +#[cfg(feature = "jev")] +pub use jev::JevDecider; +pub use matcher::{ConfigError, Error, Match, Matcher, MatcherBuilder, Unmatched, Verdict}; diff --git a/rust/src/matcher.rs b/rust/src/matcher.rs new file mode 100644 index 0000000..f9aef75 --- /dev/null +++ b/rust/src/matcher.rs @@ -0,0 +1,373 @@ +//! Classify a state into one of your enum's variants, gated by calibrated confidence. + +use std::fmt; +use std::time::{Duration, Instant}; + +use serde::Serialize; +use serde_json::Value; + +use crate::branch::{Branch, Branches, Criterion, MAX_BRANCHES, NONE_LABEL, guard_key, label_of}; +use crate::decider::{BoxError, Decider, Decision, Request}; + +/// Where a call landed. The data each variant carries is exactly what that outcome knows: there is +/// no branch to speak of when the model picked "none of these". +/// +/// Match on it like any other enum. The compiler makes sure every branch and every way of *not* +/// matching is handled, so a miss can't be silently dropped. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[must_use] +pub enum Verdict { + /// `B` won with enough confidence, and its guard (if any) held. + Matched(B), + /// `B` ranked first, but the model was less sure than `B` requires. + LowConfidence(B), + /// `B` won confidently, but its `when` statement did not hold. + GuardFailed(B), + /// The model judged that none of the branches describe the state. + NoMatch, +} + +impl Verdict { + /// The branch, only if it matched. + pub fn matched(&self) -> Option { + match *self { + Verdict::Matched(branch) => Some(branch), + _ => None, + } + } + + /// The branch the model ranked first, whether or not it matched. + pub fn top(&self) -> Option { + match *self { + Verdict::Matched(branch) | Verdict::LowConfidence(branch) | Verdict::GuardFailed(branch) => { + Some(branch) + } + Verdict::NoMatch => None, + } + } + + pub fn is_matched(&self) -> bool { + matches!(self, Verdict::Matched(_)) + } + + /// The outcome's name, for logs: `matched`, `low_confidence`, `guard_failed` or `no_match`. + pub fn as_str(&self) -> &'static str { + match self { + Verdict::Matched(_) => "matched", + Verdict::LowConfidence(_) => "low_confidence", + Verdict::GuardFailed(_) => "guard_failed", + Verdict::NoMatch => "no_match", + } + } +} + +impl fmt::Display for Verdict { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +/// The full record of one dispatch decision. +#[derive(Clone, Debug, PartialEq)] +#[must_use] +pub struct Match { + pub verdict: Verdict, + pub confidence: f64, + /// The confidence the top branch required. + pub threshold: f64, + /// Probability that the top branch's guard holds, if it has one. + pub guard: Option, + /// Probability of every option. `None` is "none of these". + pub probabilities: Vec<(Option, f64)>, + pub model: Option, + pub latency: Duration, +} + +impl Match { + /// Options ordered from most to least probable. Useful when escalating an uncertain call. + pub fn ranked(&self) -> Vec<(Option, f64)> { + let mut ranked = self.probabilities.clone(); + ranked.sort_by(|a, b| b.1.total_cmp(&a.1)); + ranked + } + + /// The matched branch, or [`Unmatched`] carrying this record, so a miss can propagate with `?`. + pub fn into_result(self) -> Result> { + match self.verdict { + Verdict::Matched(branch) => Ok(branch), + _ => Err(Unmatched(self)), + } + } +} + +/// Nothing matched. Carries the full [`Match`] so the caller can see why. +#[derive(Clone, Debug, PartialEq, thiserror::Error)] +#[error( + "no branch matched ({}; top={}, confidence={:.2}, threshold={:.2})", + .0.verdict, label_of(.0.verdict.top()), .0.confidence, .0.threshold +)] +pub struct Unmatched(pub Match); + +/// Why a call could not be decided at all (as opposed to deciding "no match"). +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("state could not be serialized to JSON: {0}")] + State(#[source] serde_json::Error), + #[error("decider failed: {0}")] + Decider(#[source] BoxError), + #[error("decider returned an option that was never offered: {0:?}")] + UnknownOption(String), + #[error("decider did not evaluate the guard for {0:?}")] + MissingGuard(&'static str), +} + +/// A matcher that can't be built. Checked once, in [`MatcherBuilder::build`]. +#[derive(Clone, Debug, PartialEq, thiserror::Error)] +pub enum ConfigError { + #[error("a matcher needs at least one branch")] + NoBranches, + #[error("a matcher supports at most {MAX_BRANCHES} branches, got {0}")] + TooManyBranches(usize), + #[error("label {0:?} must be an identifier")] + InvalidLabel(&'static str), + #[error("label {NONE_LABEL:?} is reserved")] + ReservedLabel, + #[error("a branch labelled {0:?} is already registered")] + DuplicateLabel(&'static str), + #[error("the description for {0:?} must not be empty")] + EmptyDescription(&'static str), + #[error("{name} must be between 0 and 1, got {value}")] + Probability { name: String, value: f64 }, +} + +type Observer = Box) + Send + Sync>; + +struct Arm { + branch: B, + label: &'static str, + min_confidence: f64, + guard_key: Option, +} + +/// A dispatch table keyed by natural-language descriptions, classifying into `B`. +/// +/// Every branch description becomes an option of one Choice question and every guard a yes/no +/// question in the same request, so adding a branch costs a few input tokens rather than another +/// round trip. +pub struct Matcher { + instructions: String, + decider: D, + guard_threshold: f64, + observer: Option>, + arms: Vec>, + options: Vec<(&'static str, Criterion)>, + guards: Vec<(String, String)>, + default_confidence: f64, +} + +impl Matcher { + /// Start a matcher for `instructions`, the question the branches answer, e.g. + /// "What is the user trying to do?". + pub fn builder(instructions: impl Into) -> MatcherBuilder { + MatcherBuilder { + instructions: instructions.into(), + min_confidence: 0.7, + guard_threshold: 0.5, + none_description: "None of the other options describe this.".to_owned(), + observer: None, + } + } +} + +impl Matcher { + /// A matcher with default settings. See [`Matcher::builder`] for the rest. + pub fn new(instructions: impl Into, decider: D) -> Result { + Matcher::builder(instructions).build(decider) + } + + /// Decide which branch `state` belongs to. `state` can be a string or anything serializable + /// to JSON (`{"subject": ..., "body": ...}`). + pub fn classify(&self, state: &S) -> Result, Error> { + let state = serde_json::to_value(state).map_err(Error::State)?; + let request = Request { + state: &state, + instructions: &self.instructions, + options: &self.options, + guards: &self.guards, + }; + + let started = Instant::now(); + let decision = self.decider.decide(&request).map_err(Error::Decider)?; + let latency = started.elapsed(); + + let found = self.resolve(decision, latency)?; + if let Some(observer) = &self.observer { + observer(&state, &found); + } + Ok(found) + } + + pub fn instructions(&self) -> &str { + &self.instructions + } + + pub fn decider(&self) -> &D { + &self.decider + } + + /// Parse a label from the decider back into an arm. `None` is "none of these". + fn parse(&self, label: &str) -> Result>, Error> { + if label == NONE_LABEL { + return Ok(None); + } + match self.arms.iter().find(|arm| arm.label == label) { + Some(arm) => Ok(Some(arm)), + None => Err(Error::UnknownOption(label.to_owned())), + } + } + + fn resolve(&self, decision: Decision, latency: Duration) -> Result, Error> { + let probabilities = decision + .probabilities + .iter() + .map(|(label, p)| Ok((self.parse(label)?.map(|arm| arm.branch), *p))) + .collect::>()?; + + let (verdict, threshold, guard) = match self.parse(&decision.choice)? { + None => (Verdict::NoMatch, self.default_confidence, None), + Some(arm) => { + let guard = match &arm.guard_key { + Some(key) => Some(*decision.guards.get(key).ok_or(Error::MissingGuard(arm.label))?), + None => None, + }; + let verdict = match guard { + _ if decision.confidence < arm.min_confidence => Verdict::LowConfidence(arm.branch), + Some(p) if p < self.guard_threshold => Verdict::GuardFailed(arm.branch), + _ => Verdict::Matched(arm.branch), + }; + (verdict, arm.min_confidence, guard) + } + }; + + Ok(Match { + verdict, + confidence: decision.confidence, + threshold, + guard, + probabilities, + model: decision.model, + latency, + }) + } +} + +impl fmt::Debug for Matcher { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let labels: Vec<_> = self.arms.iter().map(|arm| arm.label).collect(); + f.debug_struct("Matcher") + .field("instructions", &self.instructions) + .field("branches", &labels) + .finish() + } +} + +/// Settings for a [`Matcher`]. Everything is validated once, in [`build`](Self::build). +pub struct MatcherBuilder { + instructions: String, + min_confidence: f64, + guard_threshold: f64, + none_description: String, + observer: Option>, +} + +impl MatcherBuilder { + /// Confidence a branch needs to match unless it sets its own. Default 0.7. + pub fn min_confidence(mut self, confidence: f64) -> Self { + self.min_confidence = confidence; + self + } + + /// Probability a `when` statement needs to count as true. Default 0.5. + pub fn guard_threshold(mut self, threshold: f64) -> Self { + self.guard_threshold = threshold; + self + } + + /// Description of the implicit "none of these" option. + pub fn none_description(mut self, description: impl Into) -> Self { + self.none_description = description.into(); + self + } + + /// Called with `(state, match)` after every decision, for logging and audit. + pub fn observer(mut self, observer: impl Fn(&Value, &Match) + Send + Sync + 'static) -> Self { + self.observer = Some(Box::new(observer)); + self + } + + /// Check every branch of `B` and the settings, and build the matcher. + pub fn build(self, decider: D) -> Result, ConfigError> { + check_probability("min_confidence".to_owned(), self.min_confidence)?; + check_probability("guard_threshold".to_owned(), self.guard_threshold)?; + match B::ALL.len() { + 0 => return Err(ConfigError::NoBranches), + n if n > MAX_BRANCHES => return Err(ConfigError::TooManyBranches(n)), + _ => {} + } + + let mut arms: Vec> = Vec::with_capacity(B::ALL.len()); + let mut options = Vec::with_capacity(B::ALL.len() + 1); + let mut guards = Vec::new(); + for &branch in B::ALL { + let label = branch.label(); + let spec: Branch = branch.branch(); + if !is_identifier(label) { + return Err(ConfigError::InvalidLabel(label)); + } + if label == NONE_LABEL { + return Err(ConfigError::ReservedLabel); + } + if arms.iter().any(|arm| arm.label == label) { + return Err(ConfigError::DuplicateLabel(label)); + } + if spec.description().trim().is_empty() { + return Err(ConfigError::EmptyDescription(label)); + } + if let Some(p) = spec.required_confidence() { + check_probability(format!("min_confidence for {label}"), p)?; + } + if let Some(statement) = spec.guard() { + guards.push((guard_key(label), statement.to_owned())); + } + options.push((label, spec.criterion())); + arms.push(Arm { + branch, + label, + min_confidence: spec.required_confidence().unwrap_or(self.min_confidence), + guard_key: spec.guard().map(|_| guard_key(label)), + }); + } + options.push((NONE_LABEL, Criterion::Plain(self.none_description))); + + Ok(Matcher { + instructions: self.instructions, + decider, + guard_threshold: self.guard_threshold, + observer: self.observer, + arms, + options, + guards, + default_confidence: self.min_confidence, + }) + } +} + +fn check_probability(name: String, value: f64) -> Result<(), ConfigError> { + if (0.0..=1.0).contains(&value) { Ok(()) } else { Err(ConfigError::Probability { name, value }) } +} + +fn is_identifier(label: &str) -> bool { + let mut chars = label.chars(); + matches!(chars.next(), Some(c) if c.is_ascii_alphabetic() || c == '_') + && chars.all(|c| c.is_ascii_alphanumeric() || c == '_') +} diff --git a/rust/src/testing.rs b/rust/src/testing.rs new file mode 100644 index 0000000..107679b --- /dev/null +++ b/rust/src/testing.rs @@ -0,0 +1,78 @@ +//! Test doubles, so dispatch logic can be tested without a network or an API key. + +use std::sync::Mutex; + +use serde_json::Value; + +use crate::branch::{Branches, Criterion, NONE_LABEL}; +use crate::decider::{BoxError, Decider, Decision, Request}; + +/// A decision for `branch` at `confidence`, from a model called "scripted". +pub fn choose(branch: B, confidence: f64) -> Decision { + Decision::new(branch.label(), confidence).with_model("scripted") +} + +/// A decision for "none of these" at `confidence`. +pub fn none_of_these(confidence: f64) -> Decision { + Decision::new(NONE_LABEL, confidence).with_model("scripted") +} + +/// What the matcher asked the decider, recorded for assertions. +#[derive(Clone, Debug, PartialEq)] +pub struct Call { + pub state: Value, + pub instructions: String, + pub options: Vec<(String, Criterion)>, + pub guards: Vec<(String, String)>, +} + +type Script = Box Result + Send + Sync>; + +/// Returns decisions you script, and records every call. +/// +/// ``` +/// # use fuzzymatch::{Branch, branches}; +/// # branches! { enum Intent { Refund => Branch::new("wants money back") } } +/// use fuzzymatch::testing::{ScriptedDecider, choose, none_of_these}; +/// +/// let decider = ScriptedDecider::from_fn(|state| { +/// Ok(match state.as_str() { +/// Some(text) if text.contains("charged") => choose(Intent::Refund, 0.95), +/// _ => none_of_these(0.9), +/// }) +/// }); +/// ``` +pub struct ScriptedDecider { + script: Script, + calls: Mutex>, +} + +impl ScriptedDecider { + /// Return `decision` every time. + pub fn new(decision: Decision) -> Self { + Self::from_fn(move |_| Ok(decision.clone())) + } + + /// Compute each decision (or an error) from the state. + pub fn from_fn(script: impl Fn(&Value) -> Result + Send + Sync + 'static) -> Self { + Self { script: Box::new(script), calls: Mutex::new(Vec::new()) } + } + + /// Every call so far, oldest first. + pub fn calls(&self) -> Vec { + self.calls.lock().unwrap_or_else(|poisoned| poisoned.into_inner()).clone() + } +} + +impl Decider for ScriptedDecider { + fn decide(&self, request: &Request<'_>) -> Result { + let call = Call { + state: request.state.clone(), + instructions: request.instructions.to_owned(), + options: request.options.iter().map(|(label, c)| ((*label).to_owned(), c.clone())).collect(), + guards: request.guards.to_vec(), + }; + self.calls.lock().unwrap_or_else(|poisoned| poisoned.into_inner()).push(call); + (self.script)(request.state) + } +} diff --git a/rust/tests/jev.rs b/rust/tests/jev.rs new file mode 100644 index 0000000..20ee637 --- /dev/null +++ b/rust/tests/jev.rs @@ -0,0 +1,147 @@ +//! Exercise JevDecider over real HTTP against a local stand-in for the TypeSafe API. +//! +//! This checks that the request body has the wire shape, that auth is sent, and that a response in +//! the documented shape parses back into a Match. +#![cfg(feature = "jev")] + +use std::io::{BufRead, BufReader, Read, Write}; +use std::net::TcpListener; +use std::thread; + +use fuzzymatch::jev::{DISPATCH_KEY, JevError}; +use fuzzymatch::{Branch, Error, JevDecider, Matcher, NONE_LABEL, Verdict, branches}; +use serde_json::{Value, json}; + +struct Captured { + headers: Vec, + body: Value, +} + +/// Serve one request with `status` and `response`, and hand back what was sent. +fn serve_once(status: u16, response: Value) -> (String, thread::JoinHandle) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}", listener.local_addr().unwrap()); + let handle = thread::spawn(move || { + let (stream, _) = listener.accept().unwrap(); + let mut reader = BufReader::new(stream); + let mut headers = Vec::new(); + loop { + let mut line = String::new(); + reader.read_line(&mut line).unwrap(); + if line.trim().is_empty() { + break; + } + headers.push(line.trim().to_owned()); + } + let length: usize = headers + .iter() + .find_map(|h| { + h.to_ascii_lowercase().strip_prefix("content-length:").map(|v| v.trim().parse().unwrap()) + }) + .unwrap(); + let mut body = vec![0; length]; + reader.read_exact(&mut body).unwrap(); + + let payload = response.to_string(); + write!( + reader.get_mut(), + "HTTP/1.1 {status} X\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{payload}", + payload.len() + ) + .unwrap(); + Captured { headers, body: serde_json::from_slice(&body).unwrap() } + }); + (url, handle) +} + +fn decider(url: &str) -> JevDecider { + let agent = ureq::Agent::config_builder().proxy(None).http_status_as_error(false).build().into(); + JevDecider::new("sk-test").unwrap().base_url(url).agent(agent) +} + +branches! { + enum Intent { + Refund => Branch::new("wants money back").when("The message includes an order number"), + OrderStatus => Branch::new("where is my order"), + } +} + +#[test] +fn request_carries_one_choice_and_a_noul_per_guard() { + let (url, server) = serve_once( + 200, + json!({ + "model": "jev-1.13.0", + "usage": {"input_tokens": 120, "output_tokens": 0}, + "answers": { + DISPATCH_KEY: { + "type": "choice", + "choice": "Refund", + "confidence": 0.91, + "probabilities": {"Refund": 0.93, "OrderStatus": 0.04, NONE_LABEL: 0.03}, + }, + "guard__Refund": {"type": "noul", "noul": 0.88}, + }, + }), + ); + let support = Matcher::builder("What is the customer asking for?") + .none_description("none") + .build(decider(&url).model("jev-1.13.0")) + .unwrap(); + + let m = support.classify("I was charged twice for order #4411").unwrap(); + + let Captured { headers, body } = server.join().unwrap(); + assert!(headers.iter().any(|h| h.eq_ignore_ascii_case("authorization: Bearer sk-test"))); + assert!(headers[0].starts_with("POST /v1/systemone ")); + assert_eq!( + body, + json!({ + "state": "I was charged twice for order #4411", + "model": "jev-1.13.0", + "questions": { + DISPATCH_KEY: { + "type": "choice", + "instructions": "What is the customer asking for?", + "criteria": {"Refund": "wants money back", "OrderStatus": "where is my order", NONE_LABEL: "none"}, + }, + "guard__Refund": {"type": "noul", "instructions": "The message includes an order number"}, + }, + }) + ); + // Question order is part of the wire shape: the dispatch question first, then guards. + let keys: Vec<_> = body["questions"].as_object().unwrap().keys().cloned().collect(); + assert_eq!(keys, [DISPATCH_KEY, "guard__Refund"]); + + assert_eq!(m.verdict, Verdict::Matched(Intent::Refund)); + assert_eq!((m.confidence, m.guard), (0.91, Some(0.88))); + assert_eq!(m.ranked()[2], (None, 0.03)); + assert_eq!(m.model.as_deref(), Some("jev-1.13.0")); +} + +#[test] +fn sends_the_moving_alias_when_no_model_is_pinned() { + let (url, server) = serve_once( + 200, + json!({"model": "jev-1.13.0", "answers": { + DISPATCH_KEY: {"type": "choice", "choice": "OrderStatus", "confidence": 0.97, "probabilities": {}}, + "guard__Refund": {"type": "noul", "noul": 0.1}, + }}), + ); + let m = Matcher::new("?", decider(&url)).unwrap().classify("where's my stuff").unwrap(); + assert_eq!(m.verdict, Verdict::Matched(Intent::OrderStatus)); + assert_eq!(server.join().unwrap().body["model"], "jev-latest"); +} + +#[test] +fn http_errors_surface_with_the_server_message() { + let (url, server) = serve_once(401, json!({"error": "invalid api key"})); + let error = Matcher::::new("?", decider(&url)).unwrap().classify("x").unwrap_err(); + server.join().unwrap(); + + let Error::Decider(source) = error else { panic!("expected a decider error, got {error:?}") }; + match source.downcast_ref::() { + Some(JevError::Status { status: 401, body }) => assert!(body.contains("invalid api key")), + other => panic!("expected an HTTP 401, got {other:?}"), + } +} diff --git a/rust/tests/matcher.rs b/rust/tests/matcher.rs new file mode 100644 index 0000000..184b38f --- /dev/null +++ b/rust/tests/matcher.rs @@ -0,0 +1,286 @@ +use std::sync::{Arc, Mutex}; + +use fuzzymatch::testing::{ScriptedDecider, choose, none_of_these}; +use fuzzymatch::{ + Branch, Branches, ConfigError, Criterion, Decision, Error, Matcher, NONE_LABEL, Verdict, branches, +}; +use serde_json::json; + +branches! { + enum Intent { + Refund => Branch::new("customer wants their money back").min_confidence(0.9), + OrderStatus => Branch::new("customer asks where their order is"), + } +} + +fn support(decision: Decision) -> Matcher { + Matcher::new("What is the customer asking for?", ScriptedDecider::new(decision)).unwrap() +} + +#[test] +fn matches_the_winning_branch() { + let m = support(choose(Intent::Refund, 0.95)).classify("I was charged twice").unwrap(); + assert_eq!(m.verdict, Verdict::Matched(Intent::Refund)); + assert_eq!(m.clone().into_result(), Ok(Intent::Refund)); + assert_eq!(m.confidence, 0.95); + assert_eq!(m.model.as_deref(), Some("scripted")); +} + +#[test] +fn sends_every_branch_plus_none_as_options() { + let support = support(choose(Intent::Refund, 0.95)); + let _ = support.classify("anything").unwrap(); + + let [call] = support.decider().calls().try_into().unwrap(); + assert_eq!(call.instructions, "What is the customer asking for?"); + assert_eq!( + call.options, + [ + ("Refund".to_owned(), Criterion::from("customer wants their money back")), + ("OrderStatus".to_owned(), Criterion::from("customer asks where their order is")), + (NONE_LABEL.to_owned(), Criterion::from("None of the other options describe this.")), + ] + ); + assert!(call.guards.is_empty()); +} + +#[test] +fn examples_and_not_for_become_structured_criteria() { + branches! { + enum Plan { + Cancel => Branch::new("customer wants to cancel").examples(["kill my plan"]).not_for("pausing the subscription"), + } + } + let m = Matcher::::new("?", ScriptedDecider::new(choose(Plan::Cancel, 0.9))).unwrap(); + let _ = m.classify("x").unwrap(); + let options = &m.decider().calls()[0].options; + assert_eq!( + serde_json::to_value(&options[0].1).unwrap(), + json!({"description": "customer wants to cancel", "examples": ["kill my plan"], "not_for": "pausing the subscription"}) + ); +} + +#[test] +fn none_of_these_is_no_match() { + let m = support(none_of_these(0.97)).classify("what's your favourite colour?").unwrap(); + assert_eq!(m.verdict, Verdict::NoMatch); + assert_eq!(m.verdict.top(), None); +} + +#[test] +fn per_branch_thresholds_are_risk_scaled() { + // 0.8 confidence clears the default (0.7) for OrderStatus but not Refund's 0.9. + let status = support(choose(Intent::OrderStatus, 0.8)).classify("x").unwrap(); + let refund = support(choose(Intent::Refund, 0.8)).classify("x").unwrap(); + + assert_eq!((status.verdict, status.threshold), (Verdict::Matched(Intent::OrderStatus), 0.7)); + assert_eq!((refund.verdict, refund.threshold), (Verdict::LowConfidence(Intent::Refund), 0.9)); + assert_eq!(refund.verdict.top(), Some(Intent::Refund)); // the top pick is still reported, for escalation + assert_eq!(refund.verdict.matched(), None); +} + +#[test] +fn builder_sets_the_default_threshold() { + let m = Matcher::builder("?") + .min_confidence(0.85) + .build(ScriptedDecider::new(choose(Intent::OrderStatus, 0.8))) + .unwrap() + .classify("x") + .unwrap(); + assert_eq!((m.verdict, m.threshold), (Verdict::LowConfidence(Intent::OrderStatus), 0.85)); +} + +#[test] +fn into_result_carries_the_miss() { + let unmatched = support(choose(Intent::Refund, 0.5)).classify("hmm").unwrap().into_result().unwrap_err(); + assert_eq!(unmatched.0.verdict, Verdict::LowConfidence(Intent::Refund)); + assert_eq!( + unmatched.to_string(), + "no branch matched (low_confidence; top=Refund, confidence=0.50, threshold=0.90)" + ); +} + +branches! { + enum Guarded { + Refund => Branch::new("customer wants their money back").when("The message includes an order number"), + Vip => Branch::new("a VIP customer").when("The customer says they are a premium member"), + } +} + +#[test] +fn guards_are_asked_in_the_same_call_for_every_branch() { + let decision = + choose(Guarded::Refund, 0.95).with_guard(Guarded::Refund, 0.9).with_guard(Guarded::Vip, 0.1); + let m = Matcher::new("?", ScriptedDecider::new(decision)).unwrap(); + + assert_eq!(m.classify("refund order #123").unwrap().verdict, Verdict::Matched(Guarded::Refund)); + let [call] = m.decider().calls().try_into().unwrap(); + assert_eq!( + call.guards, + [ + ("guard__Refund".to_owned(), "The message includes an order number".to_owned()), + ("guard__Vip".to_owned(), "The customer says they are a premium member".to_owned()), + ] + ); +} + +#[test] +fn failed_guard_does_not_match() { + let decision = + choose(Guarded::Refund, 0.95).with_guard(Guarded::Refund, 0.2).with_guard(Guarded::Vip, 0.9); + let m = + Matcher::new("?", ScriptedDecider::new(decision)).unwrap().classify("give me my money back").unwrap(); + assert_eq!((m.verdict, m.guard), (Verdict::GuardFailed(Guarded::Refund), Some(0.2))); +} + +#[test] +fn low_confidence_wins_over_a_failed_guard() { + let decision = + choose(Guarded::Refund, 0.3).with_guard(Guarded::Refund, 0.2).with_guard(Guarded::Vip, 0.9); + let m = Matcher::new("?", ScriptedDecider::new(decision)).unwrap().classify("x").unwrap(); + assert_eq!(m.verdict, Verdict::LowConfidence(Guarded::Refund)); +} + +#[test] +fn missing_guard_is_an_error() { + let m = Matcher::::new("?", ScriptedDecider::new(choose(Guarded::Refund, 0.95))).unwrap(); + assert!(matches!(m.classify("x"), Err(Error::MissingGuard("Refund")))); +} + +#[test] +fn observer_sees_every_decision() { + let seen = Arc::new(Mutex::new(Vec::new())); + let log = Arc::clone(&seen); + let support = Matcher::builder("?") + .observer(move |state, m| log.lock().unwrap().push((state.clone(), m.verdict))) + .build(ScriptedDecider::new(choose(Intent::Refund, 0.95))) + .unwrap(); + let _ = support.classify("charged twice").unwrap(); + assert_eq!(*seen.lock().unwrap(), [(json!("charged twice"), Verdict::Matched(Intent::Refund))]); +} + +#[test] +fn ranked_orders_by_probability() { + let decision = choose(Intent::Refund, 0.4).with_probabilities([ + (None, 0.2), + (Some(Intent::Refund), 0.5), + (Some(Intent::OrderStatus), 0.3), + ]); + let m = support(decision).classify("x").unwrap(); + let order: Vec<_> = m.ranked().into_iter().map(|(pick, _)| pick).collect(); + assert_eq!(order, [Some(Intent::Refund), Some(Intent::OrderStatus), None]); +} + +#[test] +fn state_can_be_structured() { + let support = support(choose(Intent::Refund, 0.95)); + let state = json!({"subject": "Duplicate charge", "body": "Charged twice"}); + let _ = support.classify(&state).unwrap(); + assert_eq!(support.decider().calls()[0].state, state); +} + +#[test] +fn state_can_be_any_serializable_type() { + #[derive(serde::Serialize)] + struct Ticket<'a> { + subject: &'a str, + } + let support = support(choose(Intent::Refund, 0.95)); + let _ = support.classify(&Ticket { subject: "Duplicate charge" }).unwrap(); + assert_eq!(support.decider().calls()[0].state, json!({"subject": "Duplicate charge"})); +} + +#[test] +fn rejects_options_that_were_never_offered() { + let m = support(Decision::new("made_up", 0.9)); + assert!(matches!(m.classify("x"), Err(Error::UnknownOption(label)) if label == "made_up")); +} + +#[test] +fn rejects_probabilities_for_options_that_were_never_offered() { + let mut decision = choose(Intent::Refund, 0.95); + decision.probabilities.push(("made_up".to_owned(), 0.01)); + assert!(matches!(support(decision).classify("x"), Err(Error::UnknownOption(_)))); +} + +#[test] +fn decider_errors_propagate() { + let m = Matcher::::new("?", ScriptedDecider::from_fn(|_| Err("network down".into()))).unwrap(); + assert!(matches!(m.classify("x"), Err(Error::Decider(e)) if e.to_string() == "network down")); +} + +// -- configuration --------------------------------------------------------------------------- +// `branches!` can't produce most of these (labels are variant names, and it needs at least one +// variant), so they use hand-written impls. + +macro_rules! hand_written { + ($name:ident, [$($label:literal => $description:expr),*]) => { + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + struct $name(usize); + + const LABELS: &[&str] = &[$($label),*]; + + impl Branches for $name { + const ALL: &'static [Self] = &{ + let mut all = [$name(0); LABELS.len()]; + let mut i = 0; + while i < all.len() { + all[i] = $name(i); + i += 1; + } + all + }; + fn label(self) -> &'static str { + LABELS[self.0] + } + fn branch(self) -> Branch { + let descriptions: Vec = vec![$($description),*]; + descriptions[self.0].clone() + } + } + }; +} + +fn build() -> Result, ConfigError> { + Matcher::new("?", ScriptedDecider::new(none_of_these(0.9))) +} + +#[test] +fn rejects_the_reserved_label() { + hand_written!(Reserved, ["none_of_these" => Branch::new("x")]); + assert_eq!(build::().unwrap_err(), ConfigError::ReservedLabel); +} + +#[test] +fn rejects_labels_that_are_not_identifiers() { + hand_written!(Spaced, ["not valid" => Branch::new("x")]); + assert_eq!(build::().unwrap_err(), ConfigError::InvalidLabel("not valid")); +} + +#[test] +fn rejects_duplicate_labels() { + hand_written!(Twice, ["same" => Branch::new("first"), "same" => Branch::new("second")]); + assert_eq!(build::().unwrap_err(), ConfigError::DuplicateLabel("same")); +} + +#[test] +fn rejects_empty_descriptions() { + hand_written!(Blank, ["blank" => Branch::new(" ")]); + assert_eq!(build::().unwrap_err(), ConfigError::EmptyDescription("blank")); +} + +#[test] +fn needs_at_least_one_branch() { + hand_written!(Empty, []); + assert_eq!(build::().unwrap_err(), ConfigError::NoBranches); +} + +#[test] +fn rejects_confidence_outside_zero_to_one() { + hand_written!(Sure, ["sure" => Branch::new("x").min_confidence(1.5)]); + assert!(matches!(build::(), Err(ConfigError::Probability { value: 1.5, .. }))); + + let decider = ScriptedDecider::new(none_of_these(0.9)); + let error = Matcher::::builder("?").guard_threshold(f64::NAN).build(decider).unwrap_err(); + assert!(matches!(error, ConfigError::Probability { ref name, .. } if name == "guard_threshold")); +} From b711ccb338a0eb067f1e2063ba86d5d93c74c316 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 15:18:42 +0000 Subject: [PATCH 2/2] Reject decider confidence and guard values outside 0 to 1 A NaN confidence or guard probability compares false against every threshold, so `resolve` fell through to `Verdict::Matched` and a malformed answer could run a high-risk branch. Validate both before comparing and return `Error::InvalidProbability` naming the value. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XvSGXaFnctMyew8CowkNRS --- rust/src/matcher.rs | 20 ++++++++++++++++++-- rust/tests/matcher.rs | 25 +++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/rust/src/matcher.rs b/rust/src/matcher.rs index f9aef75..70025b4 100644 --- a/rust/src/matcher.rs +++ b/rust/src/matcher.rs @@ -119,6 +119,8 @@ pub enum Error { UnknownOption(String), #[error("decider did not evaluate the guard for {0:?}")] MissingGuard(&'static str), + #[error("decider returned {name} = {value}, which is not a probability between 0 and 1")] + InvalidProbability { name: String, value: f64 }, } /// A matcher that can't be built. Checked once, in [`MatcherBuilder::build`]. @@ -233,15 +235,21 @@ impl Matcher { .map(|(label, p)| Ok((self.parse(label)?.map(|arm| arm.branch), *p))) .collect::>()?; + // NaN compares false against every threshold, so an unchecked NaN would fall through to + // `Matched`: a malformed answer must never be able to run a branch. + let confidence = checked_probability("confidence", decision.confidence)?; let (verdict, threshold, guard) = match self.parse(&decision.choice)? { None => (Verdict::NoMatch, self.default_confidence, None), Some(arm) => { let guard = match &arm.guard_key { - Some(key) => Some(*decision.guards.get(key).ok_or(Error::MissingGuard(arm.label))?), + Some(key) => { + let p = *decision.guards.get(key).ok_or(Error::MissingGuard(arm.label))?; + Some(checked_probability(key, p)?) + } None => None, }; let verdict = match guard { - _ if decision.confidence < arm.min_confidence => Verdict::LowConfidence(arm.branch), + _ if confidence < arm.min_confidence => Verdict::LowConfidence(arm.branch), Some(p) if p < self.guard_threshold => Verdict::GuardFailed(arm.branch), _ => Verdict::Matched(arm.branch), }; @@ -362,6 +370,14 @@ impl MatcherBuilder { } } +fn checked_probability(name: &str, value: f64) -> Result { + if (0.0..=1.0).contains(&value) { + Ok(value) + } else { + Err(Error::InvalidProbability { name: name.to_owned(), value }) + } +} + fn check_probability(name: String, value: f64) -> Result<(), ConfigError> { if (0.0..=1.0).contains(&value) { Ok(()) } else { Err(ConfigError::Probability { name, value }) } } diff --git a/rust/tests/matcher.rs b/rust/tests/matcher.rs index 184b38f..cbe55a2 100644 --- a/rust/tests/matcher.rs +++ b/rust/tests/matcher.rs @@ -147,6 +147,31 @@ fn missing_guard_is_an_error() { assert!(matches!(m.classify("x"), Err(Error::MissingGuard("Refund")))); } +#[test] +fn rejects_confidence_that_is_not_a_probability() { + // NaN compares false against every threshold, so it must not slip through as a match. + for confidence in [f64::NAN, 1.5, -0.1] { + let result = support(choose(Intent::Refund, confidence)).classify("x"); + assert!( + matches!(result, Err(Error::InvalidProbability { ref name, .. }) if name == "confidence"), + "{confidence}: {result:?}" + ); + } +} + +#[test] +fn rejects_guard_probabilities_that_are_not_probabilities() { + for p in [f64::NAN, 1.5, -0.1] { + let decision = + choose(Guarded::Refund, 0.95).with_guard(Guarded::Refund, p).with_guard(Guarded::Vip, 0.9); + let result = Matcher::::new("?", ScriptedDecider::new(decision)).unwrap().classify("x"); + assert!( + matches!(result, Err(Error::InvalidProbability { ref name, .. }) if name == "guard__Refund"), + "{p}: {result:?}" + ); + } +} + #[test] fn observer_sees_every_decision() { let seen = Arc::new(Mutex::new(Vec::new()));