diff --git a/.gitattributes b/.gitattributes index 9fd71a11cb5..b3103a1d799 100644 --- a/.gitattributes +++ b/.gitattributes @@ -3,7 +3,7 @@ # Excludes locale files that have been automatically trimmed during prepr # from the diff viewer + diff calculation on GitHub. **/locales/*/index.json linguist-generated=true -!**/locales/en-US/index.json linguist-generated +**/locales/en-US/index.json !linguist-generated # SQLx calculates a checksum of migration scripts at build time to compare # it with the checksum of the applied migration for the same version at diff --git a/.github/workflows/daedalus-docker.yml b/.github/workflows/minecraft-meta-build.yml similarity index 63% rename from .github/workflows/daedalus-docker.yml rename to .github/workflows/minecraft-meta-build.yml index dfb12d160f7..06eeaa92ca0 100644 --- a/.github/workflows/daedalus-docker.yml +++ b/.github/workflows/minecraft-meta-build.yml @@ -1,4 +1,4 @@ -name: daedalus-docker-build +name: Minecraft Meta Build on: push: @@ -6,17 +6,17 @@ on: - 'main' - 'prod' paths: - - .github/workflows/daedalus-docker.yml - - 'apps/daedalus_client/**' - - 'packages/daedalus/**' + - .github/workflows/minecraft-meta-build.yml + - 'apps/modrinth-minecraft-meta/**' + - 'packages/modrinth-log/**' - Cargo.toml - Cargo.lock pull_request: types: [opened, synchronize] paths: - - .github/workflows/daedalus-docker.yml - - 'apps/daedalus_client/**' - - 'packages/daedalus/**' + - .github/workflows/minecraft-meta-build.yml + - 'apps/modrinth-minecraft-meta/**' + - 'packages/modrinth-log/**' - Cargo.toml - Cargo.lock merge_group: @@ -31,7 +31,7 @@ env: jobs: build: - name: Build daedalus_client + name: Build modrinth-minecraft-meta runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && 'namespace-profile-medium-amd64' || 'ubuntu-latest' }} steps: - name: Check out code @@ -51,38 +51,28 @@ jobs: - name: Setup mold uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 / Mold 2.41.0 - - name: Build daedalus_client - run: cargo build --release --package daedalus_client + - name: Build modrinth-minecraft-meta + run: cargo build --locked --release --package modrinth-minecraft-meta - name: Stage Docker context run: | - mkdir -p apps/daedalus_client/docker-stage - cp target/release/daedalus_client apps/daedalus_client/docker-stage/daedalus_client + mkdir -p apps/modrinth-minecraft-meta/docker-stage + cp target/release/modrinth-minecraft-meta apps/modrinth-minecraft-meta/docker-stage/modrinth-minecraft-meta - name: Upload Docker context if: env.INTERNAL == 'true' uses: namespace-actions/upload-artifact@f6ccaacc655aec41b93af180d1d7eef21af862d2 # v1.0.3 with: - name: daedalus-docker-context + name: minecraft-meta-docker-context retention-days: 1 - path: apps/daedalus_client/docker-stage + path: apps/modrinth-minecraft-meta/docker-stage docker-build: needs: [build] if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} uses: SparkUniverse/workflows/.github/workflows/docker-build.yaml@main with: - image-name: daedalus - dockerfile-path: apps/daedalus_client/Dockerfile - artifacts-name: daedalus-docker-context + image-name: minecraft-meta + dockerfile-path: apps/modrinth-minecraft-meta/Dockerfile + artifacts-name: minecraft-meta-docker-context - deploy: - needs: [docker-build] - if: ${{ github.ref == 'refs/heads/prod' && github.event_name == 'push' }} - uses: SparkUniverse/workflows/.github/workflows/argo-update.yaml@main - secrets: - ARGOCD_DEPLOY_KEY: ${{ secrets.ARGOCD_DEPLOY_KEY }} - with: - application-set: launcher-meta - branch: main - environment-name: production diff --git a/Cargo.lock b/Cargo.lock index 5f4b5ef3732..271a7a5782e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -316,6 +316,16 @@ dependencies = [ "generic-array", ] +[[package]] +name = "aegis" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41585f248ed9406567ae6d61ca2a55ed3dc7348e65203a01229881253acb95e3" +dependencies = [ + "cc", + "softaes", +] + [[package]] name = "aes" version = "0.8.4" @@ -404,6 +414,12 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "allocator-api2" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c880a97d28a3681c0267bd29cff89621202715b065127cd445fa0f0fe0aa2880" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -424,9 +440,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.21" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" dependencies = [ "anstyle", "anstyle-parse", @@ -439,15 +455,15 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" [[package]] name = "anstyle-parse" -version = "0.2.7" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" dependencies = [ "utf8parse", ] @@ -472,6 +488,22 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "antithesis_sdk" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08410fcac93669a476c006cd6c4512ac1e2b30fd117231a5d55d8a2c76599b82" +dependencies = [ + "libc", + "libloading 0.8.8", + "linkme", + "once_cell", + "rand 0.8.5", + "rustc_version_runtime", + "serde", + "serde_json", +] + [[package]] name = "antlr4rust" version = "0.5.2" @@ -491,9 +523,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.100" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "arbitrary" @@ -550,11 +582,37 @@ dependencies = [ "serde_bytes", "serde_cbor", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.21", "utoipa", "uuid 1.23.3", ] +[[package]] +name = "aristo" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcdeefa800110050103e2459a2f8dbdbd560ad046e30f1f87e24ce19c2cb8bde" +dependencies = [ + "aristo-macros", +] + +[[package]] +name = "aristo-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d64a66d21a80182b35b1741997a6d2456911f54b7eb1918aa4e2382fc205268d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "array-init" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d62b7694a562cdf5a74227903507c56ab2cc8bdd1f781ed5cb4cf9c9f810bfc" + [[package]] name = "arrayvec" version = "0.7.6" @@ -613,7 +671,7 @@ dependencies = [ "proc-macro2", "quote", "syn 2.0.106", - "synstructure", + "synstructure 0.13.2", ] [[package]] @@ -627,6 +685,12 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "assoc" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfdc70193dadb9d7287fa4b633f15f90c876915b31f6af17da307fc59c9859a8" + [[package]] name = "astral-tokio-tar" version = "0.5.6" @@ -838,7 +902,7 @@ dependencies = [ "chrono", "futures-util", "hex", - "hmac", + "hmac 0.12.1", "http-types", "hyper 0.14.32", "hyper-rustls 0.24.2", @@ -846,7 +910,7 @@ dependencies = [ "serde_json", "serde_path_to_error", "serde_qs 0.10.1", - "sha2", + "sha2 0.10.9", "smart-default", "smol_str", "thiserror 1.0.69", @@ -887,7 +951,7 @@ dependencies = [ "tokio", "tokio-rustls 0.26.4", "tungstenite 0.27.0", - "webpki-roots 1.0.3", + "webpki-roots", ] [[package]] @@ -898,7 +962,7 @@ checksum = "37672978ae0febce7516ae0a85b53e6185159a9a28787391eb63fc44ec36037d" dependencies = [ "async-fs", "futures-lite 2.6.1", - "thiserror 2.0.17", + "thiserror 2.0.21", ] [[package]] @@ -912,7 +976,7 @@ dependencies = [ "crc32fast", "futures-lite 2.6.1", "pin-project", - "thiserror 2.0.17", + "thiserror 2.0.21", "tokio", "tokio-util", ] @@ -949,6 +1013,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "atomic" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89cbf775b137e9b968e67227ef7f775587cde3fd31b0d8599dbd0f598a48340" +dependencies = [ + "bytemuck", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -968,7 +1041,7 @@ dependencies = [ "serde", "serde_json", "url", - "webpki-roots 1.0.3", + "webpki-roots", ] [[package]] @@ -1035,7 +1108,7 @@ dependencies = [ "quick-xml 0.38.3", "rust-ini", "serde", - "thiserror 2.0.17", + "thiserror 2.0.21", "time", "url", ] @@ -1056,7 +1129,7 @@ version = "0.32.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2b715a6010afb9e457ca2b7c9d2b9c344baa8baed7b38dc476034c171b32575" dependencies = [ - "bindgen", + "bindgen 0.72.1", "cc", "cmake", "dunce", @@ -1070,7 +1143,7 @@ version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5532f65342f789f9c1b7078ea9c9cd9293cd62dcc284fa99adc4a1c9ba43469c" dependencies = [ - "thiserror 2.0.17", + "thiserror 2.0.21", ] [[package]] @@ -1123,14 +1196,14 @@ dependencies = [ "bytes", "fastrand 2.3.0", "hex", - "hmac", + "hmac 0.12.1", "http 0.2.12", "http 1.3.1", "http-body 1.0.1", - "lru", + "lru 0.16.4", "percent-encoding", "regex-lite", - "sha2", + "sha2 0.10.9", "tracing", "url", ] @@ -1149,11 +1222,11 @@ dependencies = [ "bytes", "form_urlencoded", "hex", - "hmac", + "hmac 0.12.1", "http 0.2.12", "http 1.3.1", "percent-encoding", - "sha2", + "sha2 0.10.9", "time", "tracing", ] @@ -1183,10 +1256,10 @@ dependencies = [ "http 1.3.1", "http-body 1.0.1", "http-body-util", - "md-5", + "md-5 0.10.6", "pin-project-lite", "sha1 0.10.6", - "sha2", + "sha2 0.10.9", "tracing", ] @@ -1426,6 +1499,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" +[[package]] +name = "base16ct" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd307490d624467aa6f74b0eabb77633d1f758a7b25f12bceb0b22e08d9726f6" + [[package]] name = "base32" version = "0.5.1" @@ -1483,6 +1562,42 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4372b9543397a4b86050cc5e7ee36953edf4bac9518e8a774c2da694977fb6e4" +[[package]] +name = "bigdecimal" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fa3f3d8cbf4dffcfe4991de61d012bef509a409ecbe9dd41049bfe32b4d4653" +dependencies = [ + "autocfg", + "libm", + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "bindgen" +version = "0.69.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" +dependencies = [ + "bitflags 2.9.4", + "cexpr", + "clang-sys", + "itertools 0.12.1", + "lazy_static", + "lazycell", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash 1.1.0", + "shlex 1.3.0", + "syn 2.0.106", + "which 4.4.2", +] + [[package]] name = "bindgen" version = "0.72.1" @@ -1521,6 +1636,16 @@ dependencies = [ "bit-vec 0.8.0", ] +[[package]] +name = "bit-set" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56d87354e4229f54a44f7bf2435906a4656dba36026ab6eaca629a2c436a691c" +dependencies = [ + "bit-vec 0.10.1", + "serde", +] + [[package]] name = "bit-vec" version = "0.6.3" @@ -1533,6 +1658,16 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" +[[package]] +name = "bit-vec" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5727b15fa97d4f4fee0a3b7c3d550ed0269f54329207b86388de918604e31269" +dependencies = [ + "borsh", + "serde", +] + [[package]] name = "bit_field" version = "0.10.3" @@ -1679,7 +1814,7 @@ dependencies = [ "serde_json", "serde_repr", "serde_urlencoded", - "thiserror 2.0.17", + "thiserror 2.0.21", "tokio", "tokio-stream", "tokio-util", @@ -1744,27 +1879,43 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "borrow-or-share" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c" + [[package]] name = "borsh" -version = "1.5.7" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad8646f98db542e39fc66e68a20b2144f6a732636df7c2354e74645faaa433ce" +checksum = "553c5d846a6ba5150c65e3b1b8ec073bcf1abc20f9b7220de384a4443ea4e20a" dependencies = [ "borsh-derive", + "bytes", "cfg_aliases", ] [[package]] name = "borsh-derive" -version = "1.5.7" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdd1d3c0c2f5833f22386f252fe8ed005c7f59fdcddeef025c01b4c3b9fd9ac3" +checksum = "12cdfe656708a01f89b451a7d36466e6fe6c414de0aa18fc54f864f6f9ca9f56" dependencies = [ "once_cell", "proc-macro-crate 3.4.0", "proc-macro2", "quote", - "syn 2.0.106", + "syn 3.0.3", +] + +[[package]] +name = "branches" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7436b21fd6195415058eb41c9090e156b556bc7e0377bb57c5c026a421521525" +dependencies = [ + "rustc_version", ] [[package]] @@ -1809,6 +1960,12 @@ version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" +[[package]] +name = "by_address" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64fa3c856b712db6612c019f14756e64e4bcea13337a6b33b696333a9eaa2d06" + [[package]] name = "bytecheck" version = "0.6.12" @@ -1836,6 +1993,20 @@ name = "bytemuck" version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fbdf580320f38b612e485521afda1ee26d10cc9884efaaa750d383e13e3c5f4" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a1f896587b6f2c069c73d2f0913e2d590c3990285cd2f0b6aa02b786b4c679c" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] [[package]] name = "byteorder" @@ -1851,9 +2022,9 @@ checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" [[package]] name = "bytes" -version = "1.10.1" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" dependencies = [ "serde", ] @@ -1940,7 +2111,7 @@ dependencies = [ "semver", "serde", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.21", ] [[package]] @@ -1965,9 +2136,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.4" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273" +checksum = "f360145194ee8e21db5ee7f3fcd4fe52210864c75c985dae33218202c8bbe040" dependencies = [ "find-msvc-tools", "jobserver", @@ -2060,6 +2231,12 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "cfg_block" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18758054972164c3264f7c8386f5fc6da6114cb46b619fd365d4e3b2dc3ae487" + [[package]] name = "chacha20" version = "0.10.0" @@ -2166,9 +2343,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.48" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2134bb3ea021b78629caa971416385309e0131b351b25e01dc16fb54e1b5fae" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" dependencies = [ "clap_builder", "clap_derive", @@ -2176,9 +2353,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.48" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2ba64afa3c0a6df7fa517765e31314e983f51dda798ffba27b988194fb65dc9" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" dependencies = [ "anstream", "anstyle", @@ -2188,21 +2365,21 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.47" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbfd7eae0b0f1a6e63d4b13c9c478de77c2eb546fba158ad50b4203dc24b9f9c" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.106", + "syn 3.0.3", ] [[package]] name = "clap_lex" -version = "0.7.5" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" [[package]] name = "clickhouse" @@ -2225,7 +2402,7 @@ dependencies = [ "sealed", "serde", "static_assertions", - "thiserror 2.0.17", + "thiserror 2.0.21", "time", "tokio", "url", @@ -2251,7 +2428,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "235f72141cfbe1d2d930d8156a34814c8a3d60491febb9af64cc52a203444764" dependencies = [ "bytes", - "thiserror 2.0.17", + "thiserror 2.0.21", ] [[package]] @@ -2263,13 +2440,19 @@ dependencies = [ "cc", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "cobs" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" dependencies = [ - "thiserror 2.0.17", + "thiserror 2.0.21", ] [[package]] @@ -2283,7 +2466,7 @@ dependencies = [ "eyre", "indenter", "once_cell", - "owo-colors", + "owo-colors 4.2.3", "tracing-error", ] @@ -2294,7 +2477,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8b88ea9df13354b55bc7234ebcce36e6ef896aca2e42a15de9e10edce01b427" dependencies = [ "once_cell", - "owo-colors", + "owo-colors 4.2.3", "tracing-core", "tracing-error", ] @@ -2595,6 +2778,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "cpubits" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -2646,6 +2835,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "338089f42c427b86394a5ee60ff321da23a5c89c9d89514c829687b26359fcff" +[[package]] +name = "crc32c" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +dependencies = [ + "rustc_version", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -2722,6 +2920,20 @@ dependencies = [ "zeroize", ] +[[package]] +name = "crypto-bigint" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a52aa3fcda4e6302a9f48734f234d35d4721b96f8fe07d073f07ce9df4f0271" +dependencies = [ + "cpubits", + "ctutils", + "num-traits", + "rand_core 0.10.1", + "serdect", + "zeroize", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -2742,6 +2954,16 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "crypto-primes" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3633a51a39c69ebbaa4feaa694bd83d241e4093901c84a0963b19d9bb3f0cf8f" +dependencies = [ + "crypto-bigint 0.7.5", + "rand_core 0.10.1", +] + [[package]] name = "cssparser" version = "0.29.6" @@ -2828,6 +3050,15 @@ dependencies = [ "cipher", ] +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + [[package]] name = "daedalus" version = "0.2.3" @@ -2835,7 +3066,7 @@ dependencies = [ "chrono", "serde", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.21", ] [[package]] @@ -2858,7 +3089,7 @@ dependencies = [ "serde", "serde_json", "sha1_smol", - "thiserror 2.0.17", + "thiserror 2.0.21", "tokio", "tracing", "tracing-error", @@ -3012,18 +3243,28 @@ name = "deadpool" version = "0.12.3" source = "git+https://github.com/modrinth/deadpool?rev=c7209b086829572f368f1194aba54bdea8dc540b#c7209b086829572f368f1194aba54bdea8dc540b" dependencies = [ - "deadpool-runtime", + "deadpool-runtime 0.1.5", "num_cpus", "tokio", "tracing", ] +[[package]] +name = "deadpool" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e98a7e119cd347f4201e1159b19831029e203e2d8b790547708e8157b4acf1e" +dependencies = [ + "deadpool-runtime 0.3.1", + "tokio", +] + [[package]] name = "deadpool-redis" version = "0.23.0" source = "git+https://github.com/modrinth/deadpool?rev=c7209b086829572f368f1194aba54bdea8dc540b#c7209b086829572f368f1194aba54bdea8dc540b" dependencies = [ - "deadpool", + "deadpool 0.12.3", "redis", "tracing", ] @@ -3036,6 +3277,15 @@ dependencies = [ "tokio", ] +[[package]] +name = "deadpool-runtime" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2657f61fb1dd8bf37a8d51093cc7cee4e77125b22f7753f49b289f831bec2bae" +dependencies = [ + "tokio", +] + [[package]] name = "debugid" version = "0.8.0" @@ -3052,6 +3302,37 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26bf8fc351c5ed29b5c2f0cbbac1b209b74f60ecd62e675a998df72c49af5204" +[[package]] +name = "defmt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror 2.0.21", +] + [[package]] name = "der" version = "0.7.10" @@ -3059,7 +3340,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ "const-oid 0.9.6", - "pem-rfc7468", + "der_derive", + "flagset", + "pem-rfc7468 0.7.0", + "zeroize", +] + +[[package]] +name = "der" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a878c850e9e421b20262e9b41f9c860e4785fa07541c266b62ff9d1ef998a80a" +dependencies = [ + "const-oid 0.10.2", + "pem-rfc7468 1.0.0", "zeroize", ] @@ -3077,13 +3371,23 @@ dependencies = [ "rusticata-macros", ] +[[package]] +name = "der_derive" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + [[package]] name = "deranged" -version = "0.5.4" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a41953f86f8a05768a6cda24def994fd2f424b04ec5c719cf89989779f199071" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" dependencies = [ - "powerfmt", "serde_core", ] @@ -3165,6 +3469,18 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "dialoguer" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25f104b501bf2364e78d0d3974cbc774f738f5865306ed128e1e0d7499c0ad96" +dependencies = [ + "console", + "shell-words", + "tempfile", + "zeroize", +] + [[package]] name = "digest" version = "0.10.7" @@ -3186,6 +3502,7 @@ dependencies = [ "block-buffer 0.12.1", "const-oid 0.10.2", "crypto-common 0.2.2", + "ctutils", ] [[package]] @@ -3235,7 +3552,7 @@ dependencies = [ "serde", "serde_json", "subsecond", - "thiserror 2.0.17", + "thiserror 2.0.21", "tracing", "tungstenite 0.28.0", ] @@ -3329,7 +3646,7 @@ dependencies = [ "serde_derive", "serde_json", "serde_repr", - "thiserror 2.0.17", + "thiserror 2.0.21", "uuid 0.8.2", ] @@ -3498,12 +3815,12 @@ version = "0.16.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" dependencies = [ - "der", + "der 0.7.10", "digest 0.10.7", "elliptic-curve", "rfc6979", - "signature", - "spki", + "signature 2.2.0", + "spki 0.7.3", ] [[package]] @@ -3521,15 +3838,15 @@ version = "0.13.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" dependencies = [ - "base16ct", - "crypto-bigint", + "base16ct 0.2.0", + "crypto-bigint 0.5.5", "digest 0.10.7", "ff", "generic-array", "group", - "hkdf", - "pem-rfc7468", - "pkcs8", + "hkdf 0.12.4", + "pem-rfc7468 0.7.0", + "pkcs8 0.10.2", "rand_core 0.6.4", "sec1", "subtle", @@ -3754,6 +4071,16 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "etcetera" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de48cc4d1c1d97a20fd819def54b890cadde72ed3ad0c614822a0a433361be96" +dependencies = [ + "cfg-if", + "windows-sys 0.61.2", +] + [[package]] name = "event-listener" version = "2.5.3" @@ -3806,6 +4133,18 @@ dependencies = [ "once_cell", ] +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + [[package]] name = "fancy-regex" version = "0.13.0" @@ -3817,6 +4156,17 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "fastbloom" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7f34442dbe69c60fe8eaf58a8cafff81a1f278816d8ab4db255b3bef4ac3c4" +dependencies = [ + "getrandom 0.3.3", + "libm", + "siphasher 1.0.1", +] + [[package]] name = "fastrand" version = "1.9.0" @@ -3884,6 +4234,19 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "figment" +version = "0.10.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cb01cd46b0cf372153850f4c6c272d9cbea2da513e07538405148f95bd789f3" +dependencies = [ + "atomic", + "pear", + "serde", + "uncased", + "version_check", +] + [[package]] name = "filetime" version = "0.2.26" @@ -3898,9 +4261,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.11" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" [[package]] name = "findshlibs" @@ -3920,6 +4283,12 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9844ddc3a6e533d62bba727eb6c28b5d360921d5175e9ff0f1e621a5c590a4d5" +[[package]] +name = "flagset" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7ac824320a75a52197e8f2d787f6a38b6718bb6897a35142d749af3c0e8f4fe" + [[package]] name = "flate2" version = "1.1.4" @@ -3931,6 +4300,17 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "fluent-uri" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc74ac4d8359ae70623506d512209619e5cf8f347124910440dbc221714b328e" +dependencies = [ + "borrow-or-share", + "ref-cast", + "serde", +] + [[package]] name = "flume" version = "0.11.1" @@ -3942,6 +4322,17 @@ dependencies = [ "spin 0.9.8", ] +[[package]] +name = "flume" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e139bc46ca777eb5efaf62df0ab8cc5fd400866427e56c68b22e414e53bd3be" +dependencies = [ + "futures-core", + "futures-sink", + "spin 0.9.8", +] + [[package]] name = "fnv" version = "1.0.7" @@ -4098,9 +4489,9 @@ checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" [[package]] name = "futures-executor" -version = "0.3.31" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" dependencies = [ "futures-core", "futures-task", @@ -4306,6 +4697,21 @@ dependencies = [ "x11", ] +[[package]] +name = "genawaiter" +version = "0.99.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c86bd0361bcbde39b13475e6e36cb24c329964aa2611be285289d1e4b751c1a0" +dependencies = [ + "genawaiter-macro", +] + +[[package]] +name = "genawaiter-macro" +version = "0.99.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b32dfe1fdfc0bbde1f22a5da25355514b5e450c33a6af6770884c8750aedfbc" + [[package]] name = "genco" version = "0.19.0" @@ -4338,6 +4744,21 @@ dependencies = [ "tracing", ] +[[package]] +name = "generator" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54ade96dc9003043bce7c035c85a9df5a858bfb2039c5a2e6fdf00f324f6c551" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows-link 0.2.1", + "windows-result 0.4.1", +] + [[package]] name = "generic-array" version = "0.14.9" @@ -4690,7 +5111,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ "ahash 0.8.12", - "allocator-api2", + "allocator-api2 0.2.21", ] [[package]] @@ -4699,7 +5120,7 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ - "allocator-api2", + "allocator-api2 0.2.21", "equivalent", "foldhash 0.1.5", ] @@ -4710,7 +5131,7 @@ version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" dependencies = [ - "allocator-api2", + "allocator-api2 0.2.21", "equivalent", "foldhash 0.2.0", ] @@ -4720,6 +5141,11 @@ name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2 0.2.21", + "equivalent", + "foldhash 0.2.0", +] [[package]] name = "hashlink" @@ -4730,6 +5156,15 @@ dependencies = [ "hashbrown 0.15.5", ] +[[package]] +name = "hashlink" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824e001ac4f3012dd16a264bec811403a67ca9deb6c102fc5049b32c4574b35f" +dependencies = [ + "hashbrown 0.16.0", +] + [[package]] name = "heck" version = "0.3.3" @@ -4814,7 +5249,7 @@ dependencies = [ "once_cell", "rand 0.9.2", "ring", - "thiserror 2.0.17", + "thiserror 2.0.21", "tinyvec", "tokio", "tracing", @@ -4858,7 +5293,7 @@ dependencies = [ "rand 0.9.2", "resolv-conf", "smallvec", - "thiserror 2.0.17", + "thiserror 2.0.21", "tokio", "tracing", ] @@ -4869,7 +5304,16 @@ version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" dependencies = [ - "hmac", + "hmac 0.12.1", +] + +[[package]] +name = "hkdf" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aaa26c720c68b866f2c96ef5c1264b3e6f473fe5d4ce61cd44bbe913e553018" +dependencies = [ + "hmac 0.13.0", ] [[package]] @@ -4881,6 +5325,15 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", +] + [[package]] name = "home" version = "0.5.11" @@ -5120,7 +5573,7 @@ dependencies = [ "tokio", "tokio-rustls 0.26.4", "tower-service", - "webpki-roots 1.0.3", + "webpki-roots", ] [[package]] @@ -5227,85 +5680,154 @@ dependencies = [ "png 0.17.16", ] +[[package]] +name = "icu_collator" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08984ed58ac439ebf3e13d2cf26b0c46a60afcd21721c1d14087c0b240344dda" +dependencies = [ + "icu_collator_data", + "icu_collections", + "icu_locale_core", + "icu_locale_fallback", + "icu_normalizer", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "zerovec", +] + +[[package]] +name = "icu_collator_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d7e54efdddeb1208c08dd5d32b53a879ac00d2d3d051b2255fd26821d16368" + [[package]] name = "icu_collections" -version = "2.0.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" dependencies = [ "displaydoc", "potential_utf", + "utf8_iter", "yoke", "zerofrom", "zerovec", ] +[[package]] +name = "icu_locale" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "785f595c61ef57169a467eeed7b4b6936a66f6cacbb116a96ee86da983251bf4" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_locale_data", + "icu_locale_fallback", + "icu_provider", + "tinystr", + "zerovec", +] + [[package]] name = "icu_locale_core" -version = "2.0.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" dependencies = [ "displaydoc", "litemap", + "serde", "tinystr", "writeable", "zerovec", ] +[[package]] +name = "icu_locale_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d37d91460e362a5cf58907cd7ce871411775ee6294e49a5cc8e6cec16dfa75ea" + +[[package]] +name = "icu_locale_fallback" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "251af8e57c9400e3eb58242fe5b8b1152b2a64fdf4cf632f923c38ccee6f2fa9" +dependencies = [ + "icu_locale_core", + "icu_locale_fallback_data", + "icu_provider", + "potential_utf", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locale_fallback_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "decf2a22ec8fa68f1a0c1129a3f8583f8f8bc24e8b9ccbe98ead99f62a4dc3a8" + [[package]] name = "icu_normalizer" -version = "2.0.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" dependencies = [ - "displaydoc", "icu_collections", "icu_normalizer_data", "icu_properties", "icu_provider", "smallvec", + "utf16_iter", + "utf8_iter", + "write16", "zerovec", ] [[package]] name = "icu_normalizer_data" -version = "2.0.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" [[package]] name = "icu_properties" -version = "2.0.1" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" dependencies = [ "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", "icu_provider", - "potential_utf", "zerotrie", "zerovec", ] [[package]] name = "icu_properties_data" -version = "2.0.1" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" [[package]] name = "icu_provider" -version = "2.0.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" dependencies = [ "displaydoc", "icu_locale_core", + "serde", "stable_deref_trait", - "tinystr", "writeable", "yoke", "zerofrom", @@ -5417,6 +5939,12 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "964de6e86d545b246d84badc0fef527924ace5134f30641c203ef52ba83f58d5" +[[package]] +name = "index_vec" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44faf5bb8861a9c72e20d3fb0fdbd59233e43056e2b80475ab0aacdc2e781355" + [[package]] name = "indexmap" version = "1.9.3" @@ -5484,7 +6012,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e96d2465363ed2d81857759fc864cf6bb7997f79327aec028d65bd7989393685" dependencies = [ "ahash 0.8.12", - "clap 4.5.48", + "clap 4.6.7", "crossbeam-channel", "crossbeam-utils", "dashmap", @@ -5499,6 +6027,12 @@ dependencies = [ "str_stack", ] +[[package]] +name = "inlinable_string" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb" + [[package]] name = "inotify" version = "0.11.0" @@ -5548,6 +6082,35 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "intrusive-collections" +version = "0.9.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "189d0897e4cbe8c75efedf3502c18c887b05046e59d28404d4d8e46cbc4d1e86" +dependencies = [ + "memoffset", +] + +[[package]] +name = "inventory" +version = "0.3.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" +dependencies = [ + "rustversion", +] + +[[package]] +name = "io-uring" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3bd0ecfbb87805f538bb7b32e5239ca0763890c623e349860ecba69469f2bb" +dependencies = [ + "bitflags 2.9.4", + "cfg-if", + "libc", +] + [[package]] name = "ipconfig" version = "0.3.2" @@ -5701,72 +6264,175 @@ dependencies = [ ] [[package]] -name = "jni" -version = "0.21.1" +name = "jiff" +version = "0.2.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +checksum = "0ab1baf72f08796de0260609515130699b890ac25f30e610ad894bc5856cafdb" dependencies = [ - "cesu8", - "cfg-if", - "combine", - "jni-sys", + "defmt", + "jiff-core", + "jiff-static", + "jiff-tzdb-platform", "log", - "thiserror 1.0.69", - "walkdir", - "windows-sys 0.45.0", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "windows-link 0.2.1", ] [[package]] -name = "jni-sys" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" - -[[package]] -name = "jobserver" -version = "0.1.34" +name = "jiff-core" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +checksum = "5e52fe76043ccecc9005d2305ebaadf7d7fc0cc89ca6baa10a94d6bc68c7128c" dependencies = [ - "getrandom 0.3.3", - "libc", + "defmt", + "log", ] [[package]] -name = "js-sys" -version = "0.3.91" +name = "jiff-static" +version = "0.2.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c" +checksum = "378268a1116ad67ae6228701118ac9f491d78fda38a40a1f1a9e1348de6f7212" dependencies = [ - "once_cell", - "wasm-bindgen", + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.106", ] [[package]] -name = "json-patch" -version = "4.1.0" +name = "jiff-tzdb" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f300e415e2134745ef75f04562dd0145405c2f7fd92065db029ac4b16b57fe90" -dependencies = [ - "jsonptr", - "serde", - "serde_json", - "thiserror 1.0.69", -] +checksum = "142bd39932ad231f10513df9ab62661fead8719872150b7ad02a2df79f4e141e" [[package]] -name = "json5" -version = "1.3.1" +name = "jiff-tzdb-platform" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "733a844dbd6fef128e98cb4487b887cb55454d92cd9994b1bafe004fabbe670c" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" dependencies = [ - "serde", - "ucd-trie", + "jiff-tzdb", ] [[package]] -name = "jsonptr" -version = "0.7.1" +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys 0.3.0", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys 0.4.1", + "log", + "simd_cesu8", + "thiserror 2.0.21", + "walkdir", + "windows-link 0.2.1", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.106", +] + +[[package]] +name = "jni-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.106", +] + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.3", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "json-patch" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f300e415e2134745ef75f04562dd0145405c2f7fd92065db029ac4b16b57fe90" +dependencies = [ + "jsonptr", + "serde", + "serde_json", + "thiserror 1.0.69", +] + +[[package]] +name = "json5" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "733a844dbd6fef128e98cb4487b887cb55454d92cd9994b1bafe004fabbe670c" +dependencies = [ + "serde", + "ucd-trie", +] + +[[package]] +name = "jsonptr" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a5a3cc660ba5d72bce0b3bb295bf20847ccbb40fd423f3f05b61273672e561fe" dependencies = [ @@ -5844,7 +6510,7 @@ dependencies = [ "censor", "chrono", "chumsky", - "clap 4.5.48", + "clap 4.6.7", "clickhouse", "color-eyre", "color-thief", @@ -5861,7 +6527,7 @@ dependencies = [ "futures-util", "heck 0.5.0", "hex", - "hmac", + "hmac 0.12.1", "hyper-rustls 0.27.7", "hyper-util", "iana-time-zone", @@ -5903,13 +6569,13 @@ dependencies = [ "serde_json", "serde_with", "sha1 0.10.6", - "sha2", + "sha2 0.10.9", "smallvec", "spdx", - "sqlx", + "sqlx 0.9.0", "sqlx-tracing", - "strum", - "thiserror 2.0.17", + "strum 0.27.2", + "thiserror 2.0.21", "tikv-jemalloc-ctl", "tikv-jemallocator", "tokio", @@ -5949,6 +6615,12 @@ dependencies = [ "spin 0.9.8", ] +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "leb128fmt" version = "0.1.0" @@ -6073,13 +6745,14 @@ checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" [[package]] name = "libredox" -version = "0.1.10" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "416f7e718bdb06000964960ffa43b4335ad4012ae8b99060261aa4a8088d5ccb" +checksum = "61ff90caf6077a803a240f62fdbe88645a890bbca49ef8174c3cb0404362171d" dependencies = [ "bitflags 2.9.4", "libc", - "redox_syscall", + "plain", + "redox_syscall 0.9.4", ] [[package]] @@ -6216,8 +6889,8 @@ dependencies = [ "regex", "serde", "serde-wasm-bindgen", - "strum", - "strum_macros", + "strum 0.27.2", + "strum_macros 0.27.2", "wasm-bindgen", ] @@ -6911,6 +7584,26 @@ dependencies = [ "memchr", ] +[[package]] +name = "linkme" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" +dependencies = [ + "linkme-impl", +] + +[[package]] +name = "linkme-impl" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "linux-raw-sys" version = "0.4.15" @@ -6973,6 +7666,19 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3bd0dd2cd90571056fdb71f6275fada10131182f84899f4b2a916e565d81d86" +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + [[package]] name = "loop9" version = "0.1.5" @@ -6991,6 +7697,15 @@ dependencies = [ "hashbrown 0.16.0", ] +[[package]] +name = "lru" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef9ac18847474e638e3702b76c65d4eb93428471a74778ef0f1be711717f89b5" +dependencies = [ + "hashbrown 0.17.1", +] + [[package]] name = "lru-cache" version = "0.1.2" @@ -7104,7 +7819,7 @@ dependencies = [ "log", "memchr", "serde", - "thiserror 2.0.17", + "thiserror 2.0.21", ] [[package]] @@ -7138,6 +7853,16 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", +] + [[package]] name = "md5" version = "0.8.0" @@ -7177,6 +7902,59 @@ dependencies = [ "autocfg", ] +[[package]] +name = "meta-generator" +version = "0.1.0" +dependencies = [ + "anyhow", + "async_zip", + "clap 4.6.7", + "derive_more 2.1.1", + "directories", + "dotenvy", + "figment", + "futures", + "hex", + "indexmap 2.14.2", + "jiff", + "modrinth-minecraft-meta", + "reqwest 0.12.24", + "rust-s3", + "secrecy", + "serde", + "serde_json", + "toasty", + "toasty-cli", + "tokio", + "tracing", + "tracing-anyhow", + "tracing-subscriber", + "url", + "uuid 1.23.3", +] + +[[package]] +name = "miette" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f98efec8807c63c752b5bd61f862c165c115b0a35685bdcfd9238c7aeb592b7" +dependencies = [ + "cfg-if", + "miette-derive", + "unicode-width 0.1.14", +] + +[[package]] +name = "miette-derive" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db5b29714e950dbb20d5e6f74f9dcec4edbcc1067bb7f8ed198c097b8c1a818b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + [[package]] name = "mime" version = "0.3.17" @@ -7243,7 +8021,7 @@ dependencies = [ "async-trait", "chrono", "serde", - "thiserror 2.0.17", + "thiserror 2.0.21", "tokio", "utoipa", ] @@ -7265,7 +8043,7 @@ name = "modrinth-maxmind" version = "0.0.0" dependencies = [ "bytes", - "clap 4.5.48", + "clap 4.6.7", "directories", "eyre", "flate2", @@ -7278,6 +8056,25 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "modrinth-minecraft-meta" +version = "0.1.0" +dependencies = [ + "anyhow", + "derive_more 2.1.1", + "hex", + "jiff", + "regex", + "serde", + "serde_json", + "serde_path_to_error", + "sha1 0.10.6", + "sha2 0.10.9", + "toasty", + "url", + "uuid 1.23.3", +] + [[package]] name = "modrinth-util" version = "0.0.0" @@ -7338,7 +8135,7 @@ dependencies = [ "once_cell", "png 0.18.0", "serde", - "thiserror 2.0.17", + "thiserror 2.0.21", "windows-sys 0.61.2", ] @@ -7357,7 +8154,7 @@ dependencies = [ "serde", "serde_json", "serde_with", - "strum", + "strum 0.27.2", "utoipa", "uuid 1.23.3", ] @@ -7400,10 +8197,10 @@ dependencies = [ "objc2-core-graphics", "objc2-foundation 0.3.2", "raw-window-handle", - "thiserror 2.0.17", + "thiserror 2.0.21", "versions", "wfd", - "which", + "which 7.0.3", "winapi", ] @@ -7431,7 +8228,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3f42e7bbe13d351b6bead8286a43aac9534b82bd3cc43e47037f012ebfd62d4" dependencies = [ "bitflags 2.9.4", - "jni-sys", + "jni-sys 0.3.0", "log", "ndk-sys", "num_enum", @@ -7451,7 +8248,7 @@ version = "0.6.0+11769913" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee6cda3051665f1fb8d9e08fc35c96d5a244fb1be711a03b71118828afc9a873" dependencies = [ - "jni-sys", + "jni-sys 0.3.0", ] [[package]] @@ -7624,9 +8421,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.1.0" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] name = "num-derive" @@ -7949,6 +8746,15 @@ dependencies = [ "objc2-foundation 0.3.2", ] +[[package]] +name = "objc2-system-configuration" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7216bd11cbda54ccabcab84d523dc93b858ec75ecfb3a7d89513fa22464da396" +dependencies = [ + "objc2-core-foundation", +] + [[package]] name = "objc2-ui-kit" version = "0.3.2" @@ -8099,7 +8905,7 @@ dependencies = [ "futures-sink", "js-sys", "pin-project-lite", - "thiserror 2.0.17", + "thiserror 2.0.21", "tracing", ] @@ -8143,7 +8949,7 @@ dependencies = [ "opentelemetry_sdk", "prost 0.13.5", "reqwest 0.12.24", - "thiserror 2.0.17", + "thiserror 2.0.21", "tokio", "tonic 0.13.1", "tracing", @@ -8202,7 +9008,7 @@ dependencies = [ "percent-encoding", "rand 0.9.2", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.21", "tokio", "tokio-stream", ] @@ -8256,7 +9062,7 @@ dependencies = [ "objc2-osa-kit", "serde", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.21", ] [[package]] @@ -8265,6 +9071,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" +[[package]] +name = "owo-colors" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" + [[package]] name = "owo-colors" version = "4.2.3" @@ -8280,7 +9092,16 @@ dependencies = [ "ecdsa", "elliptic-curve", "primeorder", - "sha2", + "sha2 0.10.9", +] + +[[package]] +name = "pack1" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3b7bb0ecf2e447b1f20ee94ee79ef6eed1e9d4b3c36ce1903b9dea3bf205523" +dependencies = [ + "bytemuck", ] [[package]] @@ -8332,7 +9153,7 @@ checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.5.18", "smallvec", "windows-link 0.2.1", ] @@ -8413,6 +9234,29 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" +[[package]] +name = "pear" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdeeaa00ce488657faba8ebf44ab9361f9365a97bd39ffb8a60663f57ff4b467" +dependencies = [ + "inlinable_string", + "pear_codegen", + "yansi", +] + +[[package]] +name = "pear_codegen" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bab5b985dc082b345f812b7df84e1bef27e7207b39e448439ba8bd69c93f147" +dependencies = [ + "proc-macro2", + "proc-macro2-diagnostics", + "quote", + "syn 2.0.106", +] + [[package]] name = "pem-rfc7468" version = "0.7.0" @@ -8422,6 +9266,15 @@ dependencies = [ "base64ct", ] +[[package]] +name = "pem-rfc7468" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6305423e0e7738146434843d1694d621cce767262b2a86910beab705e4493d9" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -8664,9 +9517,19 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" dependencies = [ - "der", - "pkcs8", - "spki", + "der 0.7.10", + "pkcs8 0.10.2", + "spki 0.7.3", +] + +[[package]] +name = "pkcs1" +version = "0.8.0-rc.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "986d2e952779af96ea048f160fd9194e1751b4faea78bcf3ceb456efe008088e" +dependencies = [ + "der 0.8.2", + "spki 0.8.0", ] [[package]] @@ -8675,8 +9538,18 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der", - "spki", + "der 0.7.10", + "spki 0.7.3", +] + +[[package]] +name = "pkcs8" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7" +dependencies = [ + "der 0.8.2", + "spki 0.8.0", ] [[package]] @@ -8686,7 +9559,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" [[package]] -name = "plist" +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + +[[package]] +name = "plist" version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "740ebea15c5d1428f910cd1a5f52cebf8d25006245ed8ade92702f4943d91e07" @@ -8698,6 +9577,16 @@ dependencies = [ "time", ] +[[package]] +name = "pluralizer" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b3eba432a00a1f6c16f39147847a870e94e2e9b992759b503e330efec778cbe" +dependencies = [ + "once_cell", + "regex", +] + [[package]] name = "png" version = "0.17.16" @@ -8756,6 +9645,15 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" +[[package]] +name = "portable-atomic-util" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715" +dependencies = [ + "portable-atomic", +] + [[package]] name = "postcard" version = "1.1.3" @@ -8807,12 +9705,45 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "postgres-protocol" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08808e3c483c46e999108051c78334f473d5adb59d78bb80a1268c7e6aa6c514" +dependencies = [ + "base64 0.22.1", + "byteorder", + "bytes", + "fallible-iterator 0.2.0", + "hmac 0.13.0", + "md-5 0.11.0", + "memchr", + "rand 0.10.1", + "sha2 0.11.0", + "stringprep", +] + +[[package]] +name = "postgres-types" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "851ca9db4932932d69f3ea811b1abe63087a0f740a47692619dd40d4899b68be" +dependencies = [ + "array-init", + "bytes", + "fallible-iterator 0.2.0", + "jiff", + "postgres-protocol", + "uuid 1.23.3", +] + [[package]] name = "potential_utf" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84df19adbe5b5a0782edcab45899906947ab039ccf4573713735ee7de1e6b08a" dependencies = [ + "serde", "zerovec", ] @@ -8968,13 +9899,26 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.101" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] +[[package]] +name = "proc-macro2-diagnostics" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", + "version_check", + "yansi", +] + [[package]] name = "procfs" version = "0.17.0" @@ -9030,7 +9974,7 @@ dependencies = [ "parking_lot", "procfs", "protobuf", - "thiserror 2.0.17", + "thiserror 2.0.21", ] [[package]] @@ -9271,7 +10215,7 @@ dependencies = [ "rustc-hash 2.1.1", "rustls 0.23.32", "socket2 0.6.5", - "thiserror 2.0.17", + "thiserror 2.0.21", "tokio", "tracing", "web-time", @@ -9293,7 +10237,7 @@ dependencies = [ "rustls 0.23.32", "rustls-pki-types", "slab", - "thiserror 2.0.17", + "thiserror 2.0.21", "tinyvec", "tracing", "web-time", @@ -9315,9 +10259,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.41" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce25767e7b499d1b604768e7cde645d14cc8584231ea6b295e9c9eb22c02e1d1" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] @@ -9368,7 +10312,7 @@ dependencies = [ "rand_chacha 0.2.2", "rand_core 0.5.1", "rand_hc", - "rand_pcg", + "rand_pcg 0.2.1", ] [[package]] @@ -9484,6 +10428,24 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rand_pcg" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59cad018caf63deb318e5a4586d99a24424a364f40f1e5778c29aca23f4fc73e" +dependencies = [ + "rand_core 0.6.4", +] + +[[package]] +name = "rapidhash" +version = "4.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5da7e78a036ce858e8d55b7e7dc8ba3a88b78350fd2155d3591bbd966b58589e" +dependencies = [ + "rustversion", +] + [[package]] name = "rav1e" version = "0.7.1" @@ -9639,6 +10601,15 @@ dependencies = [ "bitflags 2.9.4", ] +[[package]] +name = "redox_syscall" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "737970939a87c6fa31e7acad13307bccbb017a073b695b6089a2c484f929e20e" +dependencies = [ + "bitflags 2.9.4", +] + [[package]] name = "redox_users" version = "0.4.6" @@ -9658,7 +10629,7 @@ checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" dependencies = [ "getrandom 0.2.16", "libredox", - "thiserror 2.0.17", + "thiserror 2.0.21", ] [[package]] @@ -9796,7 +10767,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams 0.4.2", "web-sys", - "webpki-roots 1.0.3", + "webpki-roots", ] [[package]] @@ -9822,7 +10793,7 @@ dependencies = [ "quinn", "rustls 0.23.32", "rustls-pki-types", - "rustls-platform-verifier", + "rustls-platform-verifier 0.6.2", "serde", "serde_json", "sync_wrapper", @@ -9857,7 +10828,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" dependencies = [ - "hmac", + "hmac 0.12.1", "subtle", ] @@ -9938,6 +10909,16 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "roaring" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dedc5658c6ecb3bdb5ef5f3295bb9253f42dcf3fd1402c03f6b1f7659c3c4a9" +dependencies = [ + "bytemuck", + "byteorder", +] + [[package]] name = "rsa" version = "0.9.8" @@ -9949,15 +10930,33 @@ dependencies = [ "num-bigint-dig", "num-integer", "num-traits", - "pkcs1", - "pkcs8", + "pkcs1 0.7.5", + "pkcs8 0.10.2", "rand_core 0.6.4", - "signature", - "spki", + "signature 2.2.0", + "spki 0.7.3", "subtle", "zeroize", ] +[[package]] +name = "rsa" +version = "0.10.0-rc.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b2aa4ba0d89f73d1e332df05be0eeab8840351c36ca5654341dfdb57bb3caf" +dependencies = [ + "const-oid 0.10.2", + "crypto-bigint 0.7.5", + "crypto-primes", + "digest 0.11.3", + "pkcs1 0.8.0-rc.4", + "pkcs8 0.11.0", + "rand_core 0.10.1", + "signature 3.0.0", + "spki 0.8.0", + "zeroize", +] + [[package]] name = "rust-embed" version = "8.11.0" @@ -9988,7 +10987,7 @@ version = "8.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5bcdef0be6fe7f6fa333b1073c949729274b05f123a0ad7efcb8efd878e5c3b1" dependencies = [ - "sha2", + "sha2 0.10.9", "walkdir", ] @@ -10016,7 +11015,7 @@ dependencies = [ "cfg-if", "futures-util", "hex", - "hmac", + "hmac 0.12.1", "http 1.3.1", "log", "maybe-async", @@ -10028,9 +11027,9 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "sha2", + "sha2 0.10.9", "sysinfo", - "thiserror 2.0.17", + "thiserror 2.0.21", "time", "tokio", "tokio-stream", @@ -10103,6 +11102,16 @@ dependencies = [ "semver", ] +[[package]] +name = "rustc_version_runtime" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dd18cd2bae1820af0b6ad5e54f4a51d0f3fcc53b05f845675074efcc7af071d" +dependencies = [ + "rustc_version", + "semver", +] + [[package]] name = "rusticata-macros" version = "4.1.0" @@ -10226,12 +11235,33 @@ checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" dependencies = [ "core-foundation 0.10.1", "core-foundation-sys", - "jni", + "jni 0.21.1", + "log", + "once_cell", + "rustls 0.23.32", + "rustls-native-certs 0.8.1", + "rustls-platform-verifier-android 0.1.1", + "rustls-webpki 0.103.7", + "security-framework 3.5.1", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1167586491e2b18b8bfbb293e8180ec17c201c4f076d7cb3070ca964e7598f98" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni 0.22.4", "log", "once_cell", "rustls 0.23.32", "rustls-native-certs 0.8.1", - "rustls-platform-verifier-android", + "rustls-platform-verifier-android 0.2.0", "rustls-webpki 0.103.7", "security-framework 3.5.1", "security-framework-sys", @@ -10245,6 +11275,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" +[[package]] +name = "rustls-platform-verifier-android" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f" + [[package]] name = "rustls-webpki" version = "0.101.7" @@ -10474,10 +11510,10 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ - "base16ct", - "der", + "base16ct 0.2.0", + "der 0.7.10", "generic-array", - "pkcs8", + "pkcs8 0.10.2", "subtle", "zeroize", ] @@ -10488,6 +11524,7 @@ version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e891af845473308773346dc847b2c23ee78fe442e0472ac50e22a18a93d3ae5a" dependencies = [ + "serde", "zeroize", ] @@ -10675,7 +11712,7 @@ dependencies = [ "rand 0.9.2", "serde", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.21", "time", "url", "uuid 1.23.3", @@ -10956,6 +11993,16 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "serdect" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66cf8fedced2fcf12406bcb34223dffb92eaf34908ede12fed414c82b7f00b3e" +dependencies = [ + "base16ct 1.0.0", + "serde", +] + [[package]] name = "serial_test" version = "3.3.1" @@ -11062,6 +12109,17 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -11071,6 +12129,12 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shell-words" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6fe69c597f9c37bfeeeeeb33da3530379845f10be461a66d16d03eca2ded77" + [[package]] name = "shlex" version = "1.3.0" @@ -11083,6 +12147,26 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "shuttle" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab17edba38d63047f46780cf7360acf7467fec2c048928689a5c1dd1c2b4e31" +dependencies = [ + "assoc", + "bitvec", + "cfg-if", + "generator", + "hex", + "owo-colors 3.5.0", + "rand 0.8.5", + "rand_core 0.6.4", + "rand_pcg 0.3.1", + "scoped-tls", + "smallvec", + "tracing", +] + [[package]] name = "signal-hook-registry" version = "1.4.6" @@ -11102,12 +12186,32 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "signature" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d567dcbaf0049cb8ac2608a76cd95ff9e4412e1899d389ee400918ca7537f5" +dependencies = [ + "digest 0.11.3", + "rand_core 0.10.1", +] + [[package]] name = "simd-adler32" version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + [[package]] name = "simd_helpers" version = "0.1.0" @@ -11123,6 +12227,15 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" +[[package]] +name = "simsimd" +version = "6.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4fb3bc3cdce07a7d7d4caa4c54f8aa967f6be41690482b54b24100a2253fa70" +dependencies = [ + "cc", +] + [[package]] name = "siphasher" version = "0.3.11" @@ -11200,6 +12313,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "softaes" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4fdfe13e1c42e45f497416afb8dc4e783bf0e25b7be9077c1c77c5fb14cb998" + [[package]] name = "softbuffer" version = "0.4.6" @@ -11216,7 +12335,7 @@ dependencies = [ "objc2-foundation 0.2.2", "objc2-quartz-core 0.2.2", "raw-window-handle", - "redox_syscall", + "redox_syscall 0.5.18", "wasm-bindgen", "web-sys", "windows-sys 0.59.0", @@ -11291,7 +12410,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", - "der", + "der 0.7.10", +] + +[[package]] +name = "spki" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d9efca8738c78ee9484207732f728b1ef517bbb1833d6fc0879ca898a522f6f" +dependencies = [ + "base64ct", + "der 0.8.2", ] [[package]] @@ -11300,11 +12429,24 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fefb893899429669dcdd979aff487bd78f4064e5e7907e4269081e0ef7d97dc" dependencies = [ - "sqlx-core", - "sqlx-macros", - "sqlx-mysql", - "sqlx-postgres", - "sqlx-sqlite", + "sqlx-core 0.8.6", + "sqlx-macros 0.8.6", + "sqlx-mysql 0.8.6", + "sqlx-postgres 0.8.6", + "sqlx-sqlite 0.8.6", +] + +[[package]] +name = "sqlx" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "378620ccc25c62c89d8be1c819e76a88d59bdcc3304733330788948e619bfd71" +dependencies = [ + "sqlx-core 0.9.0", + "sqlx-macros 0.9.0", + "sqlx-mysql 0.9.0", + "sqlx-postgres 0.9.0", + "sqlx-sqlite 0.9.0", ] [[package]] @@ -11315,7 +12457,6 @@ checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" dependencies = [ "base64 0.22.1", "bytes", - "chrono", "crc", "crossbeam-queue", "either", @@ -11325,25 +12466,61 @@ dependencies = [ "futures-io", "futures-util", "hashbrown 0.15.5", - "hashlink", + "hashlink 0.10.0", "indexmap 2.14.2", "log", "memchr", "once_cell", "percent-encoding", + "serde", + "serde_json", + "sha2 0.10.9", + "smallvec", + "thiserror 2.0.21", + "tokio", + "tokio-stream", + "tracing", + "url", +] + +[[package]] +name = "sqlx-core" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b44e85bf579a8eeb4ceaa77a3a523baf2bf0e9bac7e40f405d537b5d2d5ccb" +dependencies = [ + "base64 0.22.1", + "bytes", + "cfg-if", + "chrono", + "crc", + "crossbeam-queue", + "either", + "event-listener 5.4.1", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashbrown 0.16.0", + "hashlink 0.11.1", + "indexmap 2.14.2", + "log", + "memchr", + "percent-encoding", "rust_decimal", "rustls 0.23.32", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "smallvec", - "thiserror 2.0.17", + "thiserror 2.0.21", + "time", "tokio", "tokio-stream", "tracing", "url", "uuid 1.23.3", - "webpki-roots 0.26.11", + "webpki-roots", ] [[package]] @@ -11354,8 +12531,21 @@ checksum = "a2d452988ccaacfbf5e0bdbc348fb91d7c8af5bee192173ac3636b5fb6e6715d" dependencies = [ "proc-macro2", "quote", - "sqlx-core", - "sqlx-macros-core", + "sqlx-core 0.8.6", + "sqlx-macros-core 0.8.6", + "syn 2.0.106", +] + +[[package]] +name = "sqlx-macros" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd2b84f2bc39a5705ef27ec785a11c934a41bbd4a24941e257927cddc26b60bf" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core 0.9.0", + "sqlx-macros-core 0.9.0", "syn 2.0.106", ] @@ -11374,59 +12564,114 @@ dependencies = [ "quote", "serde", "serde_json", - "sha2", - "sqlx-core", - "sqlx-mysql", - "sqlx-postgres", - "sqlx-sqlite", + "sha2 0.10.9", + "sqlx-core 0.8.6", + "sqlx-mysql 0.8.6", + "sqlx-postgres 0.8.6", + "sqlx-sqlite 0.8.6", "syn 2.0.106", "tokio", "url", ] [[package]] -name = "sqlx-mysql" -version = "0.8.6" +name = "sqlx-macros-core" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" +checksum = "fb8d96de5fdc85a5c4ec813432b523ec637e80ba98f046555f75f7908ddac7c3" dependencies = [ - "atoi", - "base64 0.22.1", - "bitflags 2.9.4", - "byteorder", - "bytes", - "chrono", - "crc", - "digest 0.10.7", + "cfg-if", "dotenvy", "either", - "futures-channel", + "heck 0.5.0", + "hex", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2 0.10.9", + "sqlx-core 0.9.0", + "sqlx-mysql 0.9.0", + "sqlx-postgres 0.9.0", + "sqlx-sqlite 0.9.0", + "syn 2.0.106", + "thiserror 2.0.21", + "tokio", + "url", +] + +[[package]] +name = "sqlx-mysql" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" +dependencies = [ + "atoi", + "base64 0.22.1", + "bitflags 2.9.4", + "byteorder", + "bytes", + "crc", + "digest 0.10.7", + "dotenvy", + "either", + "futures-channel", "futures-core", "futures-io", "futures-util", "generic-array", "hex", - "hkdf", - "hmac", + "hkdf 0.12.4", + "hmac 0.12.1", "itoa", "log", - "md-5", + "md-5 0.10.6", "memchr", "once_cell", "percent-encoding", "rand 0.8.5", - "rsa", - "rust_decimal", + "rsa 0.9.8", "serde", "sha1 0.10.6", - "sha2", + "sha2 0.10.9", "smallvec", - "sqlx-core", + "sqlx-core 0.8.6", "stringprep", - "thiserror 2.0.17", + "thiserror 2.0.21", + "tracing", + "whoami 1.6.1", +] + +[[package]] +name = "sqlx-mysql" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90b8020fe17c5f2c245bfa2505d7ef59c5604839527c740266ad2214acebea27" +dependencies = [ + "bitflags 2.9.4", + "byteorder", + "bytes", + "chrono", + "crc", + "digest 0.11.3", + "dotenvy", + "either", + "futures-core", + "futures-util", + "generic-array", + "log", + "percent-encoding", + "rand 0.10.1", + "rsa 0.10.0-rc.18", + "rust_decimal", + "serde", + "sha1 0.11.0", + "sha2 0.11.0", + "sqlx-core 0.9.0", + "thiserror 2.0.21", + "time", "tracing", "uuid 1.23.3", - "whoami", ] [[package]] @@ -11439,7 +12684,6 @@ dependencies = [ "base64 0.22.1", "bitflags 2.9.4", "byteorder", - "chrono", "crc", "dotenvy", "etcetera 0.8.0", @@ -11447,26 +12691,62 @@ dependencies = [ "futures-core", "futures-util", "hex", - "hkdf", - "hmac", + "hkdf 0.12.4", + "hmac 0.12.1", "home", "itoa", "log", - "md-5", + "md-5 0.10.6", "memchr", "once_cell", "rand 0.8.5", + "serde", + "serde_json", + "sha2 0.10.9", + "smallvec", + "sqlx-core 0.8.6", + "stringprep", + "thiserror 2.0.21", + "tracing", + "whoami 1.6.1", +] + +[[package]] +name = "sqlx-postgres" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87a2bdd6e83f6b3ea525ca9fee568030508b58355a43d0b2c1674d5f79dcd65e" +dependencies = [ + "atoi", + "base64 0.22.1", + "bitflags 2.9.4", + "byteorder", + "chrono", + "crc", + "dotenvy", + "etcetera 0.11.0", + "futures-channel", + "futures-core", + "futures-util", + "hex", + "hkdf 0.13.0", + "hmac 0.13.0", + "itoa", + "log", + "md-5 0.11.0", + "memchr", + "rand 0.10.1", "rust_decimal", "serde", "serde_json", - "sha2", + "sha2 0.11.0", "smallvec", - "sqlx-core", + "sqlx-core 0.9.0", "stringprep", - "thiserror 2.0.17", + "thiserror 2.0.21", "tracing", "uuid 1.23.3", - "whoami", + "whoami 2.1.3", ] [[package]] @@ -11476,8 +12756,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2d12fe70b2c1b4401038055f90f151b78208de1f9f89a7dbfd41587a10c3eea" dependencies = [ "atoi", - "chrono", - "flume", + "flume 0.11.1", "futures-channel", "futures-core", "futures-executor", @@ -11488,8 +12767,33 @@ dependencies = [ "percent-encoding", "serde", "serde_urlencoded", - "sqlx-core", - "thiserror 2.0.17", + "sqlx-core 0.8.6", + "thiserror 2.0.21", + "tracing", + "url", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488e99c397a62007e4229aec669a179816339afc6d2620ca6fa420dbee2e982c" +dependencies = [ + "atoi", + "chrono", + "flume 0.12.0", + "form_urlencoded", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "sqlx-core 0.9.0", + "thiserror 2.0.21", "tracing", "url", "uuid 1.23.3", @@ -11503,7 +12807,8 @@ dependencies = [ "futures", "opentelemetry", "opentelemetry-testing", - "sqlx", + "sqlx 0.8.6", + "sqlx 0.9.0", "testcontainers", "tokio", "tracing", @@ -11665,13 +12970,35 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros 0.26.4", +] + [[package]] name = "strum" version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" dependencies = [ - "strum_macros", + "strum_macros 0.27.2", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.106", ] [[package]] @@ -11699,7 +13026,7 @@ dependencies = [ "memmap2", "serde", "subsecond-types", - "thiserror 2.0.17", + "thiserror 2.0.21", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", @@ -11731,6 +13058,12 @@ dependencies = [ "serde_json", ] +[[package]] +name = "symlink" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" + [[package]] name = "syn" version = "1.0.109" @@ -11784,6 +13117,17 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "synstructure" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "sys-locale" version = "0.3.2" @@ -11865,7 +13209,7 @@ dependencies = [ "gdkwayland-sys", "gdkx11-sys", "gtk", - "jni", + "jni 0.21.1", "libc", "log", "ndk", @@ -11939,7 +13283,7 @@ dependencies = [ "heck 0.5.0", "http 1.3.1", "http-range", - "jni", + "jni 0.21.1", "libc", "log", "mime", @@ -11963,7 +13307,7 @@ dependencies = [ "tauri-runtime", "tauri-runtime-wry", "tauri-utils", - "thiserror 2.0.17", + "thiserror 2.0.21", "tokio", "tray-icon", "url", @@ -12011,10 +13355,10 @@ dependencies = [ "semver", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "syn 2.0.106", "tauri-utils", - "thiserror 2.0.17", + "thiserror 2.0.21", "time", "url", "uuid 1.23.3", @@ -12065,7 +13409,7 @@ dependencies = [ "tauri", "tauri-plugin", "tauri-utils", - "thiserror 2.0.17", + "thiserror 2.0.21", "tracing", "url", "windows-registry", @@ -12086,7 +13430,7 @@ dependencies = [ "tauri", "tauri-plugin", "tauri-plugin-fs", - "thiserror 2.0.17", + "thiserror 2.0.21", "url", ] @@ -12107,7 +13451,7 @@ dependencies = [ "tauri", "tauri-plugin", "tauri-utils", - "thiserror 2.0.17", + "thiserror 2.0.21", "toml 0.9.8", "url", ] @@ -12130,7 +13474,7 @@ dependencies = [ "tauri", "tauri-plugin", "tauri-plugin-fs", - "thiserror 2.0.17", + "thiserror 2.0.21", "tokio", "url", "urlpattern 0.3.0", @@ -12152,7 +13496,7 @@ dependencies = [ "serde_json", "tauri", "tauri-plugin", - "thiserror 2.0.17", + "thiserror 2.0.21", "url", "windows", "zbus", @@ -12173,7 +13517,7 @@ dependencies = [ "sys-locale", "tauri", "tauri-plugin", - "thiserror 2.0.17", + "thiserror 2.0.21", ] [[package]] @@ -12185,7 +13529,7 @@ dependencies = [ "serde", "serde_json", "tauri", - "thiserror 2.0.17", + "thiserror 2.0.21", "tracing", "windows-sys 0.60.2", "zbus", @@ -12215,7 +13559,7 @@ dependencies = [ "tauri", "tauri-plugin", "tempfile", - "thiserror 2.0.17", + "thiserror 2.0.21", "time", "tokio", "url", @@ -12235,7 +13579,7 @@ dependencies = [ "serde_json", "tauri", "tauri-plugin", - "thiserror 2.0.17", + "thiserror 2.0.21", ] [[package]] @@ -12247,7 +13591,7 @@ dependencies = [ "dpi", "gtk", "http 1.3.1", - "jni", + "jni 0.21.1", "objc2 0.6.4", "objc2-ui-kit", "objc2-web-kit", @@ -12255,7 +13599,7 @@ dependencies = [ "serde", "serde_json", "tauri-utils", - "thiserror 2.0.17", + "thiserror 2.0.21", "url", "webkit2gtk", "webview2-com", @@ -12269,7 +13613,7 @@ source = "git+https://github.com/tauri-apps/tauri?rev=7cc68e74ff6981f5c50a52a67d dependencies = [ "gtk", "http 1.3.1", - "jni", + "jni 0.21.1", "log", "objc2 0.6.4", "objc2-app-kit", @@ -12318,7 +13662,7 @@ dependencies = [ "serde_json", "serde_with", "swift-rs", - "thiserror 2.0.17", + "thiserror 2.0.21", "toml 1.1.6+spec-1.1.0", "url", "urlpattern 0.6.0", @@ -12410,7 +13754,7 @@ dependencies = [ "serde", "serde_json", "serde_with", - "thiserror 2.0.17", + "thiserror 2.0.21", "tokio", "tokio-stream", "tokio-util", @@ -12494,13 +13838,13 @@ dependencies = [ "serde_json", "serde_with", "sha1_smol", - "sha2", + "sha2 0.10.9", "shlex 1.3.0", - "sqlx", + "sqlx 0.9.0", "sysinfo", "tauri", "tempfile", - "thiserror 2.0.17", + "thiserror 2.0.21", "tokio", "tokio-util", "toml 0.9.8", @@ -12511,7 +13855,7 @@ dependencies = [ "url", "urlencoding", "uuid 1.23.3", - "whoami", + "whoami 1.6.1", "windows", "windows-core 0.61.2", "winreg 0.55.0", @@ -12533,7 +13877,7 @@ dependencies = [ "dashmap", "either", "enumset", - "hkdf", + "hkdf 0.12.4", "hyper 1.7.0", "hyper-util", "image", @@ -12546,7 +13890,7 @@ dependencies = [ "serde", "serde_json", "serde_with", - "sha2", + "sha2 0.10.9", "tauri", "tauri-build", "tauri-plugin-deep-link", @@ -12559,7 +13903,7 @@ dependencies = [ "tauri-plugin-updater", "tauri-plugin-window-state", "theseus", - "thiserror 2.0.17", + "thiserror 2.0.21", "tokio", "tokio-util", "tracing", @@ -12591,11 +13935,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.17" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" dependencies = [ - "thiserror-impl 2.0.17", + "thiserror-impl 2.0.21", ] [[package]] @@ -12611,13 +13955,13 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.17" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 3.0.3", ] [[package]] @@ -12676,30 +14020,29 @@ dependencies = [ [[package]] name = "time" -version = "0.3.44" +version = "0.3.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" dependencies = [ "deranged", - "itoa", "num-conv", "powerfmt", - "serde", + "serde_core", "time-core", "time-macros", ] [[package]] name = "time-core" -version = "0.1.6" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40868e7c1d2f0b8d73e4a8c7f0ff63af4f6d19be117e90bd73eb1d62cf831c6b" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.24" +version = "0.2.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30cfb0125f12d9c277f35663a0a33f8c30190f4e4574868a330595412d34ebf3" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" dependencies = [ "num-conv", "time-core", @@ -12716,11 +14059,12 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.1" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" dependencies = [ "displaydoc", + "serde_core", "zerovec", ] @@ -12739,6 +14083,184 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "tls_codec" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de2e01245e2bb89d6f05801c564fa27624dbd7b1846859876c7dad82e90bf6b" +dependencies = [ + "tls_codec_derive", + "zeroize", +] + +[[package]] +name = "tls_codec_derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d2e76690929402faae40aebdda620a2c0e25dd6d3b9afe48867dfd95991f4bd" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "toasty" +version = "0.11.0" +dependencies = [ + "async-trait", + "bit-set 0.11.1", + "by_address", + "deadpool 0.13.1", + "hashbrown 0.17.1", + "index_vec", + "indexmap 2.14.2", + "inventory", + "jiff", + "serde", + "serde_core", + "serde_json", + "toasty-core", + "toasty-driver-mysql", + "toasty-driver-postgresql", + "toasty-driver-turso", + "toasty-macros", + "tokio", + "tokio-stream", + "toml 1.1.6+spec-1.1.0", + "toml_edit 0.25.15+spec-1.1.0", + "tracing", + "uuid 1.23.3", +] + +[[package]] +name = "toasty-cli" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "756d16e2809ceaad4e2e78c0f65f7241160951648574435b331ec8ec50735cbd" +dependencies = [ + "anyhow", + "clap 4.6.7", + "console", + "dialoguer", + "hashbrown 0.17.1", + "jiff", + "rand 0.10.1", + "serde", + "toasty", + "toasty-core", + "toml 1.1.6+spec-1.1.0", +] + +[[package]] +name = "toasty-core" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ce722358481c0ad55b7b5edaaf9d0669ee5de6e9c737a73ca4c27621003732" +dependencies = [ + "async-trait", + "bit-set 0.11.1", + "fluent-uri", + "form_urlencoded", + "hashbrown 0.17.1", + "heck 0.5.0", + "indexmap 2.14.2", + "jiff", + "percent-encoding", + "serde", + "tokio-stream", + "toml 1.1.6+spec-1.1.0", + "tracing", + "uuid 1.23.3", +] + +[[package]] +name = "toasty-driver-mysql" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfa5622479dff74a1db2178c8475e014f32ab434ffb06c8cb69142b0abebd876" +dependencies = [ + "async-trait", + "jiff", + "sqlx-core 0.9.0", + "sqlx-mysql 0.9.0", + "time", + "toasty-core", + "toasty-sql", + "tokio", + "tracing", + "uuid 1.23.3", +] + +[[package]] +name = "toasty-driver-postgresql" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6cdb083715b23faddc3b667151cfda8b02feac02e0066acd933d8f8028fce4c" +dependencies = [ + "async-trait", + "fallible-iterator 0.2.0", + "hashbrown 0.17.1", + "jiff", + "lru 0.18.5", + "postgres-protocol", + "postgres-types", + "rustls 0.23.32", + "rustls-pemfile 2.2.0", + "rustls-platform-verifier 0.7.1", + "rustls-webpki 0.103.7", + "toasty-core", + "toasty-sql", + "tokio", + "tokio-postgres", + "tokio-postgres-rustls", + "tracing", + "uuid 1.23.3", +] + +[[package]] +name = "toasty-driver-turso" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f3d4680ddf5e6ec2ba584ec91763420b572d0d818e10ca105a85d312d6f10a" +dependencies = [ + "async-trait", + "serde", + "toasty-core", + "toasty-sql", + "tokio", + "tracing", + "turso", + "uuid 1.23.3", +] + +[[package]] +name = "toasty-macros" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f5b5a87266b1cccdd184244733b26fdac9ed3997446c5aca88c2c71f5b03f61" +dependencies = [ + "heck 0.5.0", + "indexmap 2.14.2", + "pluralizer", + "proc-macro2", + "quote", + "syn 3.0.3", + "toasty-core", +] + +[[package]] +name = "toasty-sql" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29b50cee2799bc54257ba202a9497f07b59df7e446b6df27fbf801cffe4b1b8a" +dependencies = [ + "jiff", + "serde", + "serde_json", + "toasty-core", +] + [[package]] name = "tokio" version = "1.53.0" @@ -12769,13 +14291,53 @@ dependencies = [ ] [[package]] -name = "tokio-native-tls" -version = "0.3.1" +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-postgres" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a528f7d280f6d5b9cd149635c8705b0dd049754bc67d81d31fa25169a93809d3" +dependencies = [ + "async-trait", + "byteorder", + "bytes", + "fallible-iterator 0.2.0", + "futures-channel", + "futures-util", + "log", + "parking_lot", + "percent-encoding", + "phf 0.13.1", + "pin-project-lite", + "postgres-protocol", + "postgres-types", + "rand 0.10.1", + "socket2 0.6.5", + "tokio", + "tokio-util", + "whoami 2.1.3", +] + +[[package]] +name = "tokio-postgres-rustls" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +checksum = "4c2ad44aa0ae96db89c4742212ed41645b2f597311ff6e1945542a4d9fadc2fb" dependencies = [ - "native-tls", + "rustls 0.23.32", + "sha2 0.11.0", "tokio", + "tokio-postgres", + "tokio-rustls 0.26.4", + "x509-cert", ] [[package]] @@ -12800,9 +14362,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.17" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" dependencies = [ "futures-core", "pin-project-lite", @@ -12930,6 +14492,21 @@ dependencies = [ "winnow 0.7.13", ] +[[package]] +name = "toml_edit" +version = "0.25.15+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1340ea94a5856333492c9064b02c778b191dd2c853778d9609debdcdfea3a614" +dependencies = [ + "indexmap 2.14.2", + "serde_core", + "serde_spanned 1.1.1", + "toml_datetime 1.1.1+spec-1.1.0", + "toml_parser", + "toml_writer", + "winnow 1.0.4", +] + [[package]] name = "toml_parser" version = "1.1.3+spec-1.1.0" @@ -13019,10 +14596,10 @@ checksum = "f124352108f58ef88299e909f6e9470f1cdc8d2a1397963901b4a6366206bf72" dependencies = [ "base32", "constant_time_eq", - "hmac", + "hmac 0.12.1", "rand 0.9.2", "sha1 0.10.6", - "sha2", + "sha2 0.10.9", ] [[package]] @@ -13104,6 +14681,30 @@ dependencies = [ "uuid 1.23.3", ] +[[package]] +name = "tracing-anyhow" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6420b6ddb954e26d308339aec0e633f2aa5756c86b8632db49d913d7b3d380d2" +dependencies = [ + "anyhow", + "tracing", + "tracing-error", +] + +[[package]] +name = "tracing-appender" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c" +dependencies = [ + "crossbeam-channel", + "symlink", + "thiserror 2.0.21", + "time", + "tracing-subscriber", +] + [[package]] name = "tracing-attributes" version = "0.1.31" @@ -13134,7 +14735,7 @@ dependencies = [ "chrono", "serde", "serde_json", - "thiserror 2.0.17", + "thiserror 2.0.21", "tracing", "tracing-core", "tracing-log", @@ -13217,7 +14818,7 @@ dependencies = [ "once_cell", "png 0.18.0", "serde", - "thiserror 2.0.17", + "thiserror 2.0.21", "windows-sys 0.61.2", ] @@ -13231,7 +14832,7 @@ dependencies = [ "sequential_gen", "serde", "spin 0.10.0", - "thiserror 2.0.17", + "thiserror 2.0.21", ] [[package]] @@ -13247,7 +14848,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "756050066659291d47a554a9f558125db17428b073c5ffce1daf5dcb0f7231d8" dependencies = [ "chrono", - "thiserror 2.0.17", + "thiserror 2.0.21", "ts-rs-macros", "uuid 1.23.3", ] @@ -13279,7 +14880,7 @@ dependencies = [ "rustls 0.23.32", "rustls-pki-types", "sha1 0.10.6", - "thiserror 2.0.17", + "thiserror 2.0.21", "utf-8", ] @@ -13296,10 +14897,210 @@ dependencies = [ "log", "rand 0.9.2", "sha1 0.10.6", - "thiserror 2.0.17", + "thiserror 2.0.21", "utf-8", ] +[[package]] +name = "turso" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9491d7a80312c5abe66a4409e4dce02065503a235453c94b9e4133877e39ffc" +dependencies = [ + "thiserror 2.0.21", + "tracing", + "tracing-subscriber", + "turso_core", + "turso_sdk_kit", + "turso_sync_sdk_kit", +] + +[[package]] +name = "turso_core" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a833cc3bf8d4e6c101c504fa470f8ab4270c2202ff2591b61b2e373b4f20d9b" +dependencies = [ + "aegis", + "aes", + "aes-gcm", + "allocator-api2 0.4.0", + "antithesis_sdk", + "arc-swap", + "aristo", + "bigdecimal", + "bitflags 2.9.4", + "branches", + "bumpalo", + "bytemuck", + "cfg_aliases", + "cfg_block", + "chrono", + "crc32c", + "crossbeam-epoch", + "crossbeam-utils", + "either", + "fallible-iterator 0.3.0", + "fastbloom", + "hex", + "icu_collator", + "icu_locale", + "intrusive-collections", + "io-uring", + "libc", + "libloading 0.8.8", + "libm", + "loom", + "miette", + "num-bigint", + "num-traits", + "pack1", + "parking_lot", + "pastey 0.2.3", + "polling", + "rand 0.9.2", + "rapidhash", + "regex", + "regex-syntax", + "roaring", + "rustc-hash 2.1.1", + "rustix 1.1.2", + "ryu", + "serde_json", + "shuttle", + "simsimd", + "smallvec", + "strum 0.26.3", + "strum_macros 0.26.4", + "tempfile", + "thiserror 2.0.21", + "tracing", + "tracing-subscriber", + "turso_ext", + "turso_macros", + "turso_parser", + "twox-hash", + "uncased", + "uuid 1.23.3", + "windows-sys 0.61.2", +] + +[[package]] +name = "turso_ext" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7452fe676450b6840e9eb80e512d14293265221154eba66aba3845dfc0d659f" +dependencies = [ + "chrono", + "getrandom 0.4.2", + "turso_macros", +] + +[[package]] +name = "turso_macros" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2825eaa6b7b893693636947f988e252c1196393fa54d4b85637c70d616d92fa" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "turso_parser" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0e58d029f02e442df4be0b5036ce520b49c553505d9d52ef5a98fc4056e95b7" +dependencies = [ + "bitflags 2.9.4", + "memchr", + "miette", + "strum 0.26.3", + "strum_macros 0.26.4", + "thiserror 2.0.21", + "turso_macros", +] + +[[package]] +name = "turso_sdk_kit" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18c1dc1c0304348c39b97bc6b27cdcb1d7292454ebd0de0f30b5ee3a4c61f9bb" +dependencies = [ + "bindgen 0.69.5", + "env_logger", + "parking_lot", + "tracing", + "tracing-appender", + "tracing-subscriber", + "turso_core", + "turso_ext", + "turso_sdk_kit_macros", +] + +[[package]] +name = "turso_sdk_kit_macros" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4708b5fe678b8730c85964e3d378f75e18c23c4409971360b2209f4e53e4956" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "turso_sync_engine" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce8f03e430240d590d209ac874db52773b7382c627cd604a685affde77622b09" +dependencies = [ + "base64 0.22.1", + "bytes", + "crc32c", + "genawaiter", + "http 1.3.1", + "libc", + "prost 0.14.3", + "roaring", + "serde", + "serde_json", + "thiserror 2.0.21", + "tracing", + "turso_core", + "turso_parser", + "uuid 1.23.3", +] + +[[package]] +name = "turso_sync_sdk_kit" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d48cb47d056c3ec567745761fa6f832358ca30ef9eb0435fdcfb77c558e70089" +dependencies = [ + "bindgen 0.69.5", + "env_logger", + "genawaiter", + "parking_lot", + "tracing", + "tracing-appender", + "tracing-subscriber", + "turso_core", + "turso_sdk_kit", + "turso_sdk_kit_macros", + "turso_sync_engine", +] + +[[package]] +name = "twox-hash" +version = "2.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5283634e518fe9e82c7b20520bb4bc209009fd16c82077c802f8111ecbb0117a" +dependencies = [ + "rand 0.10.1", +] + [[package]] name = "typed-arena" version = "2.0.2" @@ -13360,6 +15161,15 @@ dependencies = [ "libc", ] +[[package]] +name = "uncased" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1b88fcfe09e89d3866a5c11019378088af2d24c3fbd4f0543f96b479ec90697" +dependencies = [ + "version_check", +] + [[package]] name = "unic-char-property" version = "0.9.0" @@ -13494,7 +15304,7 @@ dependencies = [ "rustls-pki-types", "ureq-proto", "utf-8", - "webpki-roots 1.0.3", + "webpki-roots", ] [[package]] @@ -13557,6 +15367,12 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -13614,6 +15430,7 @@ dependencies = [ "js-sys", "rand 0.10.1", "serde_core", + "sha1_smol", "wasm-bindgen", ] @@ -13828,6 +15645,15 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" +[[package]] +name = "wasite" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fe902b4a6b8028a753d5424909b764ccf79b7a209eac9bf97e59cda9f71a42" +dependencies = [ + "wasi 0.14.7+wasi-0.2.4", +] + [[package]] name = "wasm-bindgen" version = "0.2.114" @@ -14169,15 +15995,6 @@ dependencies = [ "rustls-pki-types", ] -[[package]] -name = "webpki-roots" -version = "0.26.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" -dependencies = [ - "webpki-roots 1.0.3", -] - [[package]] name = "webpki-roots" version = "1.0.3" @@ -14218,7 +16035,7 @@ version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "36695906a1b53a3bf5c4289621efedac12b73eeb0b89e7e1a89b517302d5d75c" dependencies = [ - "thiserror 2.0.17", + "thiserror 2.0.21", "windows", "windows-core 0.61.2", ] @@ -14239,6 +16056,18 @@ dependencies = [ "winapi", ] +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix 0.38.44", +] + [[package]] name = "which" version = "7.0.3" @@ -14258,7 +16087,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" dependencies = [ "libredox", - "wasite", + "wasite 0.1.0", + "web-sys", +] + +[[package]] +name = "whoami" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "626c4bac6755d76ffc12cb01b2eac751db1996b9e0041de9aa02c8c211ddc82c" +dependencies = [ + "libc", + "libredox", + "objc2-system-configuration", + "wasite 1.0.2", "web-sys", ] @@ -14802,6 +16644,9 @@ name = "winnow" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +dependencies = [ + "memchr", +] [[package]] name = "winreg" @@ -14933,11 +16778,17 @@ dependencies = [ "regex", ] +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + [[package]] name = "writeable" -version = "0.6.1" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" [[package]] name = "wry" @@ -14957,7 +16808,7 @@ dependencies = [ "gtk", "http 1.3.1", "javascriptcore-rs", - "jni", + "jni 0.21.1", "libc", "ndk", "objc2 0.6.4", @@ -14969,10 +16820,10 @@ dependencies = [ "once_cell", "percent-encoding", "raw-window-handle", - "sha2", + "sha2 0.10.9", "soup3", "tao-macros", - "thiserror 2.0.17", + "thiserror 2.0.21", "url", "webkit2gtk", "webkit2gtk-sys", @@ -15013,6 +16864,18 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "x509-cert" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1301e935010a701ae5f8655edc0ad17c44bad3ac5ce8c39185f75453b720ae94" +dependencies = [ + "const-oid 0.9.6", + "der 0.7.10", + "spki 0.7.3", + "tls_codec", +] + [[package]] name = "x509-parser" version = "0.16.0" @@ -15067,7 +16930,7 @@ dependencies = [ "prometheus", "redis", "serde", - "thiserror 2.0.17", + "thiserror 2.0.21", "tokio", "tracing", ] @@ -15078,6 +16941,12 @@ version = "0.8.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "985eec839aaf2a1270af8f4ebcf63cf9401cfd90f0902f97c28d9f104ffbde72" +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + [[package]] name = "yaserde" version = "0.12.0" @@ -15107,11 +16976,10 @@ dependencies = [ [[package]] name = "yoke" -version = "0.8.0" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" dependencies = [ - "serde", "stable_deref_trait", "yoke-derive", "zerofrom", @@ -15119,14 +16987,14 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.0" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" +checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", - "synstructure", + "syn 3.0.3", + "synstructure 0.14.0", ] [[package]] @@ -15228,7 +17096,7 @@ dependencies = [ "proc-macro2", "quote", "syn 2.0.106", - "synstructure", + "synstructure 0.13.2", ] [[package]] @@ -15236,24 +17104,40 @@ name = "zeroize" version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] [[package]] name = "zerotrie" -version = "0.2.2" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" dependencies = [ "displaydoc", "yoke", "zerofrom", + "zerovec", ] [[package]] name = "zerovec" -version = "0.11.4" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7aa2bd55086f1ab526693ecbe444205da57e25f4489879da80635a46d90e73b" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" dependencies = [ + "serde", "yoke", "zerofrom", "zerovec-derive", @@ -15261,13 +17145,13 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.1" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 3.0.3", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 4ae0d16b5cd..b6d711b3c8f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ members = [ "apps/app-playground", "apps/daedalus_client", "apps/labrinth", + "apps/meta-generator", "packages/app-lib", "packages/ariadne", "packages/component-derive", @@ -12,16 +13,18 @@ members = [ "packages/modrinth-content-management", "packages/modrinth-log", "packages/modrinth-maxmind", + "packages/modrinth-minecraft-meta", "packages/modrinth-util", "packages/neverbounce", "packages/path-util", "packages/serde-binhum", "packages/xredis", ] +exclude = ["packages/toasty"] [workspace.package] edition = "2024" -rust-version = "1.90.0" +rust-version = "1.95.0" repository = "https://github.com/modrinth/code" [workspace.dependencies] @@ -34,6 +37,7 @@ actix-web = "4.11.0" actix-web-prom = "0.10.0" actix-ws = "0.3.0" aes-gcm = "0.10.3" +anyhow = "1.0.104" arc-swap = "1.7.1" argon2 = { version = "0.5.3", features = ["std"] } ariadne = { path = "packages/ariadne" } @@ -88,6 +92,7 @@ either = "1.15.0" encoding_rs = "0.8.35" enumset = "1.1.10" eyre = "0.6.12" +figment = { version = "0.10.19" } flate2 = "1.1.4" fs4 = { version = "0.13.1", default-features = false } futures = "0.3.31" @@ -114,6 +119,7 @@ indexmap = "2.11.4" indicatif = "0.18.0" itertools = "0.14.0" jemalloc_pprof = "0.8.1" +jiff = { version = "0.2.37" } json-patch = { version = "4.1.0", default-features = false } json5 = "1.3.1" lettre = { version = "0.11.19", default-features = false, features = [ @@ -137,10 +143,11 @@ lz4_flex = { version = "0.11.5", default-features = false, features = [ ] } maxminddb = "0.26.0" md5 = "0.8.0" -modrinth-content-management = { path = "packages/modrinth-content-management" } -modrinth-log = { path = "packages/modrinth-log" } -modrinth-util = { path = "packages/modrinth-util" } -muralpay = { path = "packages/muralpay" } +modrinth-content-management = { path = "packages/modrinth-content-management/" } +modrinth-log = { path = "packages/modrinth-log/" } +modrinth-minecraft-meta = { path = "packages/modrinth-minecraft-meta/" } +modrinth-util = { path = "packages/modrinth-util/" } +muralpay = { path = "packages/muralpay/" } murmur2 = "0.1.0" native-dialog = "0.9.2" neverbounce = { path = "packages/neverbounce" } @@ -197,6 +204,7 @@ serde_bytes = "0.11.19" serde_cbor = "0.11.2" serde_ini = "0.2.0" serde_json = "1.0.145" +serde_path_to_error = { version = "0.1.20" } serde_with = "3.15.0" sha1 = "0.10.6" sha1_smol = { version = "1.0.1", features = ["std"] } @@ -204,7 +212,7 @@ sha2 = "0.10.9" shlex = "1.3.0" smallvec = "1.15.1" spdx = "0.12.0" -sqlx = { version = "0.8.6", default-features = false } +sqlx = { version = "0.9.0", default-features = false } sqlx-tracing = { path = "packages/sqlx-tracing" } strum = "0.27.2" syn = { version = "2.0" } @@ -229,6 +237,8 @@ theseus = { path = "packages/app-lib" } thiserror = "2.0.17" tikv-jemalloc-ctl = "0.6.0" tikv-jemallocator = "0.6.0" +toasty = { version = "0.11.0" } +toasty-cli = { version = "0.11.0" } tokio = "1.47.1" tokio-stream = "0.1.17" tokio-util = "0.7.16" @@ -236,6 +246,7 @@ toml = "0.9.8" totp-rs = "5.7.0" tracing = "0.1.41" tracing-actix-web = { version = "0.7.19", default-features = false } +tracing-anyhow = { version = "0.1.0" } tracing-ecs = "0.5.0" tracing-error = "0.2.1" tracing-subscriber = "0.3.20" @@ -319,6 +330,8 @@ tauri-macros = { git = "https://github.com/tauri-apps/tauri", rev = "7cc68e74ff6 tauri-runtime = { git = "https://github.com/tauri-apps/tauri", rev = "7cc68e74ff6981f5c50a52a67d56c5eb2d227188" } tauri-runtime-wry = { git = "https://github.com/tauri-apps/tauri", rev = "7cc68e74ff6981f5c50a52a67d56c5eb2d227188" } tauri-utils = { git = "https://github.com/tauri-apps/tauri", rev = "7cc68e74ff6981f5c50a52a67d56c5eb2d227188" } +# Toasty's unused SQLite driver conflicts with SQLx's native SQLite bindings. +toasty = { path = "packages/toasty" } [profile.dev.package.sqlx-macros] opt-level = 3 diff --git a/apps/meta-generator/.env.local b/apps/meta-generator/.env.local new file mode 100644 index 00000000000..a8594988b81 --- /dev/null +++ b/apps/meta-generator/.env.local @@ -0,0 +1,17 @@ +# MR_FASTLY__BASE_URL=https://api.fastly.com/ +# MR_FASTLY__KEY= +MR_DATABASE_URL=postgresql://launchermeta:launchermeta@localhost:5433/launchermeta +MR_CONCURRENCY__DOWNLOAD=8 +MR_CONCURRENCY__EXTRACT_FILES=8 +AWS_ACCESS_KEY_ID=minecraftmeta +AWS_SECRET_ACCESS_KEY=minecraftmeta-local +MR_PUBLIC_S3__BUCKET=minecraft-meta-public +MR_PUBLIC_S3__REGION=us-east-1 +MR_PUBLIC_S3__ENDPOINT=http://127.0.0.1:9000/ +MR_PUBLIC_S3__BASE_URL=http://127.0.0.1:9000/minecraft-meta-public/ +MR_PUBLIC_S3__PATH_STYLE=true +MR_PRIVATE_S3__BUCKET=minecraft-meta-private +MR_PRIVATE_S3__REGION=us-east-1 +MR_PRIVATE_S3__ENDPOINT=http://127.0.0.1:9000/ +MR_PRIVATE_S3__BASE_URL=http://127.0.0.1:9000/minecraft-meta-private/ +MR_PRIVATE_S3__PATH_STYLE=true diff --git a/apps/meta-generator/Cargo.toml b/apps/meta-generator/Cargo.toml new file mode 100644 index 00000000000..d3af511f76f --- /dev/null +++ b/apps/meta-generator/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "meta-generator" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +default-run = "meta-generator" + +[dependencies] +anyhow = { workspace = true } +async_zip = { workspace = true, features = ["deflate"] } +clap = { workspace = true, features = ["derive"] } +derive_more = { workspace = true, features = ["debug", "deref", "display"] } +directories = { workspace = true } +dotenvy = { workspace = true } +figment = { workspace = true, features = ["env"] } +futures = { workspace = true } +hex = { workspace = true } +indexmap = { workspace = true } +jiff = { workspace = true, features = ["serde"] } +modrinth-minecraft-meta = { workspace = true } +reqwest = { workspace = true, features = ["default"] } +rust-s3 = { workspace = true } +secrecy = { workspace = true, features = ["serde"] } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +toasty = { workspace = true, features = ["jiff", "postgresql", "serde"] } +toasty-cli = { workspace = true } +tokio = { workspace = true, features = ["full"] } +tracing = { workspace = true } +tracing-anyhow = { workspace = true } +tracing-subscriber = { workspace = true, features = ["chrono", "env-filter"] } +url = { workspace = true, features = ["serde"] } +uuid = { workspace = true, features = ["v7"] } diff --git a/apps/meta-generator/Dockerfile b/apps/meta-generator/Dockerfile new file mode 100644 index 00000000000..c76207f310a --- /dev/null +++ b/apps/meta-generator/Dockerfile @@ -0,0 +1,17 @@ +# syntax=docker/dockerfile:1 + +FROM debian:trixie-slim + +LABEL org.opencontainers.image.source=https://github.com/modrinth/code +LABEL org.opencontainers.image.title=modrinth-minecraft-meta +LABEL org.opencontainers.image.description="Modrinth Minecraft launcher metadata" +LABEL org.opencontainers.image.licenses=AGPL-3.0-only + +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates dumb-init curl \ + && rm -rf /var/lib/apt/lists/* + +COPY --chmod=0755 modrinth-minecraft-meta /modrinth-minecraft-meta/modrinth-minecraft-meta + +WORKDIR /modrinth-minecraft-meta +ENTRYPOINT ["dumb-init", "--", "/modrinth-minecraft-meta/modrinth-minecraft-meta"] diff --git a/apps/meta-generator/README.md b/apps/meta-generator/README.md new file mode 100644 index 00000000000..00ccff5cd90 --- /dev/null +++ b/apps/meta-generator/README.md @@ -0,0 +1,62 @@ +Game version and launcher version related metadata manifests. + +The generator reuses models, upstream schemas, and shared utilities from the `modrinth-minecraft-meta` package. Import/export orchestration, configuration, storage, and migrations remain in `meta-generator`. + +# Architecture + +The general flow of the metadata service is: +- Import + - Download metadata and manifests from upstream sources (Mojang, Fabric, NeoForge, etc.), and save them into storage as raw JSON blobs. + - Take the JSON blobs and extract structured information out of them, useful for our purposes, and save that in our database. + - Do other extraction, like reading JSON manifests embedded in installer JAR files, and save that in our database. + - Download required Maven artifacts, or extract them from installer JARs, into content-addressed storage. Record their hashes alongside the structured metadata. + - If an upstream's downloads fail, keep its previous structured metadata and continue with other upstreams. Commit new structured metadata only after its required artifacts are stored. Report failures at the end of the run. +- Export + - Read all entries in our database, and turn them into manifests used by the app. + - Upload these manifests, and stored Maven artifacts, into the local filesystem or S3. Export never downloads from upstream sources. + - Publish referenced artifacts and version manifests before publishing the overall catalog. + - Historical catalogs can contribute versions absent from newer catalogs. Raw import history is retained, while selected structured metadata can be replaced when upstream information changes. + +This means that: +- Every time we import, we save the source manifests that we download - from this, we can derive any information that we want without needing to query the upstreams again. +- We have a historical record of all upstream manifests - if an upstream goes down, we're not affected and can keep using old information until it's back up. +- Imports and exports can be rerun. This does not guarantee that overlapping runs are safe; run them sequentially. +- Much of our data is effectively immutable, simplifying things a lot - no need to deal with missing versions if we never delete versions. +- Given the immutability, we can track the provenance of each entity (game version, loader version, etc.) +- We still have the option to hide information by not including it in the final manifest upload - e.g. if we no longer want to show a version, we can mark it as `unlisted` in our database, and the final manifest creation won't include it, but it'll still exist in the DB. + +## Running + +From the repository root: + +```sh +cargo run -p meta-generator -- import +cargo run -p meta-generator -- export +``` + +Without upstream flags, all upstreams are selected. Put flags before the command to select specific upstreams: + +```sh +cargo run -p meta-generator -- --fabric --quilt import +cargo run -p meta-generator -- --fabric --quilt export +``` + +Forge and NeoForge currently import installer metadata and embedded artifacts, and export the embedded artifacts. Their launcher manifests are not implemented yet. + +## Forge and NeoForge + +Forge and NeoForge (henceforth Forgelike) store much of their important info in the installer JAR file, rather than as a JSON file served at some HTTP route. + +## Fabric and Quilt + +Fabric's and Quilt's manifests contain (among other things) two fields: +- `game`, a list of Minecraft game versions, like `26.4-snapshot-2` +- `loader`, a list of Fabric loader versions, like `0.19.5` + +For each pair of game and loader version, Fabric/Quilt serve a file at the path `/v2/versions/loader/{minecraft_version}/{loader_version}/profile/json`. However, it would be impractical to download and mirror every single file here, especially considering that for any given `loader_version`, the `profile/json` file doesn't change between `minecraft_version`s - usually (foreshadowing). Therefore, instead of downloading every `(minecraft_version, loader_version)` pair, we instead download every `(1.21, loader_version)` pair - that is, `minecraft_version` is always 1.21[^1], but `loader_version` varies. We fix up this downloadad manifest and replace `1.21` with `${modrinth.gameVersion}`; then when a client downloads our manifests, they replace `${modrinth.gameVersion}` back to whatever game version they want to play. + +However, this doesn't always work. Specifically with Quilt, and Minecraft version 26 and later, there was a change to how the `hashed` field is treated, so we need special behavior here. This is where version groups come in. + +TODO explain version groups + +[^1]: Don't ask me why we picked 1.21 specifically, I don't know. diff --git a/apps/meta-generator/Toasty.toml b/apps/meta-generator/Toasty.toml new file mode 100644 index 00000000000..f8d63d65f37 --- /dev/null +++ b/apps/meta-generator/Toasty.toml @@ -0,0 +1,5 @@ +[migration] +path = "toasty" +prefix_style = "Timestamp" +checksums = true +statement_breakpoints = true diff --git a/apps/meta-generator/src/bin/toasty.rs b/apps/meta-generator/src/bin/toasty.rs new file mode 100644 index 00000000000..112ae09c035 --- /dev/null +++ b/apps/meta-generator/src/bin/toasty.rs @@ -0,0 +1,20 @@ +use anyhow::{Context, Result}; +use toasty_cli::ToastyCli; +use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; + +#[tokio::main] +async fn main() -> Result<()> { + dotenvy::dotenv().ok(); + tracing_subscriber::registry() + .with(tracing_anyhow::ErrorLayer::default()) + .init(); + + let toasty_config = + toasty_cli::Config::load().context("loading toasty config")?; + let app_config = meta_generator::create_config() + .context("loading app config")?; + let db = meta_generator::connect_to_db(&app_config).await?; + let cli = ToastyCli::with_config(db, toasty_config); + cli.parse_and_run().await?; + Ok(()) +} diff --git a/apps/meta-generator/src/config.rs b/apps/meta-generator/src/config.rs new file mode 100644 index 00000000000..e73b93480c9 --- /dev/null +++ b/apps/meta-generator/src/config.rs @@ -0,0 +1,39 @@ +use std::num::NonZero; + +use secrecy::SecretString; +use serde::Deserialize; +use url::Url; + +#[derive(Debug, Clone, Deserialize)] +pub struct Config { + pub database_url: Url, + pub concurrency: Concurrency, + #[serde(default)] + pub public_s3: Option, + #[serde(default)] + pub private_s3: Option, + #[serde(default)] + pub fastly: Option, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct S3 { + pub bucket: String, + pub region: String, + pub endpoint: Url, + pub base_url: Url, + #[serde(default)] + pub path_style: bool, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct Concurrency { + pub download: NonZero, + pub extract_files: NonZero, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct Fastly { + pub base_url: Url, + pub key: SecretString, +} diff --git a/apps/meta-generator/src/export/fabric.rs b/apps/meta-generator/src/export/fabric.rs new file mode 100644 index 00000000000..606b2b075d7 --- /dev/null +++ b/apps/meta-generator/src/export/fabric.rs @@ -0,0 +1,9 @@ +use anyhow::Result; + +use crate::{AppState, export::fabriclike, model::FabriclikeLoader}; + +pub const FORMAT_VERSION: u32 = 0; + +pub async fn export(app: &AppState) -> Result<()> { + fabriclike::export(app, FabriclikeLoader::Fabric, FORMAT_VERSION).await +} diff --git a/apps/meta-generator/src/export/fabriclike.rs b/apps/meta-generator/src/export/fabriclike.rs new file mode 100644 index 00000000000..1d58f2f72cb --- /dev/null +++ b/apps/meta-generator/src/export/fabriclike.rs @@ -0,0 +1,448 @@ +use std::{ + collections::HashMap, + sync::atomic::{AtomicUsize, Ordering}, +}; + +use anyhow::{Context, Result, ensure}; +use futures::{TryStreamExt, stream}; +use indexmap::IndexMap; +use serde::Serialize; +use tracing::{info, info_span}; +use tracing_anyhow::FutureContext; +use url::Url; + +use crate::model::GAME_VERSION_PLACEHOLDER; +use crate::{ + AppState, + model::{ + FabriclikeCatalog, FabriclikeGameVersion, FabriclikeLoader, + FabriclikeProfile, FabriclikeProfileGroup, FabriclikeProfileMetadata, + FabriclikeStoredArtifact, + }, + store::ContentType, +}; + +/// Publishes extracted catalog state and profiles, with store-specific URLs. +pub async fn export( + app: &AppState, + loader: FabriclikeLoader, + format_version: u32, +) -> Result<()> { + let (catalog, rows) = fetch_catalog(app, loader).await?; + ensure!( + !rows.is_empty(), + "no extracted {loader:?} profiles; run extraction before exporting" + ); + let games = catalog + .games + .0 + .into_iter() + .map(|game| (game.id.clone(), game)) + .collect(); + let groups = catalog.groups.0; + let artifacts = catalog.artifacts.0; + let public_maven = app.public_blobs.url_for("maven/"); + let mut profiles = Vec::with_capacity(rows.len()); + for row in rows { + let group = if row.version_group.is_empty() { + None + } else { + Some(row.version_group) + }; + ensure!( + groups.iter().any(|candidate| candidate.id == group), + "extracted profile {} has unknown group {:?}", + row.loader_version, + group + ); + let path = profile_path( + loader, + format_version, + group.as_deref(), + &row.loader_version, + ); + let profile = prepare_profile(row.metadata.0, &public_maven); + profiles.push(ProfileExport { + loader: LoaderVersion { + id: row.loader_version, + stable: row.stable, + url: app.public_blobs.url_for(&path), + }, + group, + path, + json: serde_json::to_vec(&profile) + .context("serializing loader profile")?, + }); + } + let profile_files = profiles + .iter() + .map(|profile| (profile.path.clone(), profile.json.clone())) + .collect::>(); + let manifest = make_manifest(games, &groups, profiles)?; + let json = + serde_json::to_vec(&manifest).context("serializing loader manifest")?; + // publish libraries before publishing profiles which reference them + let num_total = artifacts.len(); + info!(num_artifacts = num_total, "publishing loader dependencies"); + let num_done = AtomicUsize::new(0); + stream::iter(artifacts.into_iter().map(Ok::<_, anyhow::Error>)) + .try_for_each_concurrent( + app.concurrency.download.get(), + |artifact| async { + publish_artifact(app, artifact).await?; + let num_done = num_done.fetch_add(1, Ordering::Relaxed) + 1; + if num_done.is_multiple_of(100) || num_done == num_total { + info!("published {num_done}/{num_total} dependencies"); + } + anyhow::Ok(()) + }, + ) + .context(info_span!("publishing loader dependencies")) + .await?; + // write processed loader profiles + for (path, data) in &profile_files { + app.public_blobs + .put(path, data, ContentType::Json) + .context(info_span!("writing loader profile", %path)) + .await?; + } + info!(num_profiles = profile_files.len(), "wrote loader profiles"); + // write manifest (overall catalog), after all referenced files are uploaded + let path = match loader { + FabriclikeLoader::Fabric => { + format!("fabric/v{format_version}/manifest.json") + } + FabriclikeLoader::Quilt => { + format!("quilt/v{format_version}/manifest.json") + } + }; + app.public_blobs + .put(&path, &json, ContentType::Json) + .context(info_span!("writing loader manifest", %path)) + .await?; + info!(%path, "wrote loader manifest"); + Ok(()) +} + +async fn fetch_catalog( + app: &AppState, + loader: FabriclikeLoader, +) -> Result<(FabriclikeCatalog, Vec)> { + let mut conn = app + .db + .connection() + .context(info_span!("acquiring connection")) + .await?; + // keep both queries on the same snapshot if extraction commits between them + let mut txn = conn + .transaction() + .context(info_span!("starting export snapshot")) + .await?; + toasty::sql::statement( + "SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY", + ) + .exec(&mut txn) + .context(info_span!("setting export snapshot isolation")) + .await?; + let catalog = FabriclikeCatalog::get_by_loader(&mut txn, loader) + .context(info_span!("fetching extracted loader catalog", ?loader)) + .await?; + let mut profiles = FabriclikeProfile::all() + .filter(FabriclikeProfile::fields().loader().eq(loader)) + .exec(&mut txn) + .context(info_span!("fetching extracted loader profiles", ?loader)) + .await?; + txn.commit() + .context(info_span!("finishing export snapshot")) + .await?; + profiles.sort_by_key(|profile| profile.position); + Ok((catalog, profiles)) +} + +fn prepare_profile( + mut profile: FabriclikeProfileMetadata, + public_maven: &Url, +) -> FabriclikeProfileMetadata { + for library in &mut profile.libraries { + library.url = Some(public_maven.to_string()); + } + profile +} + +/// Publishes a stored artifact from content-addressed storage to the Maven store. +pub async fn publish_artifact( + app: &AppState, + stored: FabriclikeStoredArtifact, +) -> Result<()> { + let bytes = app.cas + .get(stored.sha256) + .context(info_span!("reading stored Maven artifact", sha256 = %stored.sha256, coordinate = %stored.artifact.coordinate)) + .await?; + + app.maven + .put(&stored.artifact.coordinate, &bytes) + .context(info_span!("publishing Maven artifact", coordinate = %stored.artifact.coordinate)) + .await +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct Manifest { + game_versions: Vec, + #[serde(skip_serializing_if = "Vec::is_empty")] + version_groups: Vec, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct GameVersion { + id: String, + stable: bool, + #[serde(skip_serializing_if = "Option::is_none")] + version_group: Option, + loaders: Vec, +} + +#[derive(Debug, Serialize)] +struct LoaderVersion { + id: String, + url: Url, + stable: bool, +} + +#[derive(Debug, Serialize)] +struct VersionGroup { + id: String, + loaders: Vec, +} + +struct ProfileExport { + loader: LoaderVersion, + group: Option, + path: String, + json: Vec, +} + +fn profile_path( + loader: FabriclikeLoader, + format_version: u32, + group: Option<&str>, + version: &str, +) -> String { + let loader = match loader { + FabriclikeLoader::Fabric => "fabric", + FabriclikeLoader::Quilt => "quilt", + }; + match group { + None => format!("{loader}/v{format_version}/versions/{version}.json"), + Some(group) => format!( + "{loader}/v{format_version}/version-group/{group}/loader-version/{version}" + ), + } +} + +fn make_manifest( + games: IndexMap, + groups: &[FabriclikeProfileGroup], + profiles: Vec, +) -> Result { + let mut manifest = Manifest { + game_versions: Vec::new(), + version_groups: Vec::new(), + }; + let mut grouped_loaders: IndexMap, Vec> = + IndexMap::new(); + for profile in profiles { + grouped_loaders + .entry(profile.group) + .or_default() + .push(profile.loader); + } + let mut game_groups = HashMap::new(); + for group in groups { + let loaders = + grouped_loaders.shift_remove(&group.id).unwrap_or_default(); + if let Some(id) = &group.id { + manifest.version_groups.push(VersionGroup { + id: id.clone(), + loaders, + }); + } else { + manifest.game_versions.push(GameVersion { + id: GAME_VERSION_PLACEHOLDER.to_owned(), + stable: true, + version_group: None, + loaders, + }); + } + for game in &group.games { + ensure!( + game_groups.insert(game.clone(), group.id.clone()).is_none(), + "game version {game} belongs to multiple profile groups" + ); + } + } + for game in games.into_values() { + let version_group = + game_groups.remove(&game.id).with_context(|| { + format!("game version {} has no profile group", game.id) + })?; + manifest.game_versions.push(GameVersion { + id: game.id, + stable: game.stable, + version_group, + loaders: Vec::new(), + }); + } + Ok(manifest) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn stored_artifacts_preserve_flattened_provenance_and_require_content_hash() + { + let mut value = json!({ + "coordinate": "net.fabricmc:fabric-loader:0.19.5", + "repository": "https://maven.fabricmc.net/", + "source": "downloaded profile", + "repository_source": "profile library.url", + "sha256": "0000000000000000000000000000000000000000000000000000000000000000" + }); + let stored: FabriclikeStoredArtifact = + serde_json::from_value(value.clone()).unwrap(); + assert_eq!(serde_json::to_value(stored).unwrap(), value); + value.as_object_mut().unwrap().remove("sha256"); + assert!( + serde_json::from_value::(value).is_err() + ); + } + + fn group( + id: Option<&str>, + template: &str, + games: &[&str], + ) -> FabriclikeProfileGroup { + FabriclikeProfileGroup { + id: id.map(str::to_owned), + template: template.into(), + fallback_templates: Vec::new(), + games: games.iter().map(|game| (*game).into()).collect(), + profile_base_url: "https://meta.example.com/versions/loader/" + .parse() + .unwrap(), + } + } + + fn exported(group: Option<&str>, version: &str) -> ProfileExport { + ProfileExport { + loader: LoaderVersion { + id: version.into(), + url: "https://example.com/profile.json".parse().unwrap(), + stable: true, + }, + group: group.map(str::to_owned), + path: String::new(), + json: vec![], + } + } + + fn catalog_games(ids: &[&str]) -> IndexMap { + ids.iter() + .map(|id| { + ( + (*id).into(), + FabriclikeGameVersion { + id: (*id).into(), + stable: true, + }, + ) + }) + .collect() + } + + #[test] + fn universal_manifest_preserves_fabric_shape() { + let manifest = make_manifest( + catalog_games(&["1.21", "26.3"]), + &[group(None, "1.21", &["1.21", "26.3"])], + vec![exported(None, "0.19.5")], + ) + .unwrap(); + let json = serde_json::to_value(manifest).unwrap(); + assert_eq!(json["gameVersions"][0]["id"], GAME_VERSION_PLACEHOLDER); + assert_eq!(json["gameVersions"][0]["loaders"][0]["id"], "0.19.5"); + assert_eq!(json["gameVersions"][1]["loaders"], json!([])); + assert!(json["gameVersions"][1].get("versionGroup").is_none()); + assert!(json.get("versionGroups").is_none()); + } + + #[test] + fn grouped_manifest_has_quilt_group_references() { + let manifest = make_manifest( + catalog_games(&["26.3", "1.21"]), + &[ + group(Some("v1"), "1.21", &["1.21"]), + group(Some("v2"), "26.3", &["26.3"]), + ], + vec![ + exported(Some("v1"), "0.30.0"), + exported(Some("v2"), "0.30.0"), + ], + ) + .unwrap(); + let json = serde_json::to_value(manifest).unwrap(); + assert_eq!(json["gameVersions"].as_array().unwrap().len(), 2); + assert_eq!(json["gameVersions"][0]["versionGroup"], "v2"); + assert_eq!(json["gameVersions"][1]["versionGroup"], "v1"); + assert_eq!(json["versionGroups"][0]["id"], "v1"); + assert_eq!(json["versionGroups"][1]["loaders"][0]["id"], "0.30.0"); + assert_eq!(json["gameVersions"][0]["loaders"], json!([])); + } + + #[test] + fn profile_locations_match_published_formats() { + assert_eq!( + profile_path(FabriclikeLoader::Fabric, 0, None, "0.19.5"), + "fabric/v0/versions/0.19.5.json" + ); + assert_eq!( + profile_path(FabriclikeLoader::Quilt, 1, Some("v2"), "0.30.0"), + "quilt/v1/version-group/v2/loader-version/0.30.0" + ); + } + + #[test] + fn publication_applies_current_maven_url_without_changing_extracted_metadata() + { + let profile: FabriclikeProfileMetadata = serde_json::from_value(json!({ + "id": "fabric-loader-1-${modrinth.gameVersion}", + "inheritsFrom": "${modrinth.gameVersion}", + "releaseTime": "2026-01-01T00:00:00Z", + "time": "2026-01-01T00:00:00Z", + "type": "release", + "libraries": [ + {"name": "net.fabricmc:intermediary:${modrinth.gameVersion}", "url": "https://maven.fabricmc.net/"}, + {"name": "net.fabricmc:fabric-loader:1"} + ] + })).unwrap(); + let original = serde_json::to_value(&profile).unwrap(); + for base in [ + "https://first.example.com/maven/", + "https://second.example.com/meta/maven/", + ] { + let published = + prepare_profile(profile.clone(), &base.parse().unwrap()); + let mut expected = original.clone(); + for library in expected["libraries"].as_array_mut().unwrap() { + library["url"] = json!(base); + } + assert_eq!(serde_json::to_value(published).unwrap(), expected); + } + + } +} diff --git a/apps/meta-generator/src/export/forgelike.rs b/apps/meta-generator/src/export/forgelike.rs new file mode 100644 index 00000000000..cabbba353e8 --- /dev/null +++ b/apps/meta-generator/src/export/forgelike.rs @@ -0,0 +1,531 @@ +use std::{ + collections::{BTreeMap, HashSet}, + sync::atomic::{AtomicUsize, Ordering}, +}; + +use anyhow::{Context, Result, ensure}; +use futures::{TryStreamExt, stream}; +use serde::Serialize; +use tracing::{info, info_span}; +use tracing_anyhow::FutureContext; +use url::Url; + +use crate::{ + AppState, + model::{ + ForgelikeExtract, ForgelikeInstaller, ForgelikeLoader, + ForgelikeStoredArtifact, ProfileMetadata, + }, + store::ContentType, + upstream::Upstreams, + util::MavenCoordinate, +}; + +pub const FORMAT_VERSION: u32 = 0; + +/// Publishes stored dependencies, version profiles, and Forge-like catalogs. +pub async fn export(app: &AppState, upstreams: Upstreams) -> Result<()> { + let mut loaders = Vec::new(); + if upstreams.forge { + loaders.push(ForgelikeLoader::Forge); + } + if upstreams.neoforge { + loaders.push(ForgelikeLoader::Neoforge); + } + if loaders.is_empty() { + return Ok(()); + } + let mut conn = app + .db + .connection() + .context(info_span!("acquiring Forge-like export connection")) + .await?; + let mut txn = conn + .transaction() + .context(info_span!("starting Forge-like export snapshot")) + .await?; + toasty::sql::statement( + "SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY", + ) + .exec(&mut txn) + .context(info_span!("setting Forge-like export snapshot isolation")) + .await?; + let fields = ForgelikeInstaller::fields(); + let mut installers = ForgelikeInstaller::all() + .filter(fields.loader().in_list(loaders.clone())) + .exec(&mut txn) + .context(info_span!("fetching selected Forge-like installers")) + .await?; + installers.retain(|installer| !installer.is_blacklisted()); + for loader in &loaders { + ensure!( + installers + .iter() + .any(|installer| installer.loader == *loader), + "no imported {loader:?} installers; run import before exporting" + ); + } + let installer_hashes = installers + .iter() + .map(|installer| installer.sha256) + .collect::>(); + let extracts = ForgelikeExtract::all() + .filter( + ForgelikeExtract::fields() + .installer_sha256() + .in_list(installer_hashes.iter().copied().collect::>()), + ) + .exec(&mut txn) + .context(info_span!("fetching selected Forge-like extracts")) + .await?; + let extracted_hashes = extracts + .iter() + .map(|extract| extract.installer_sha256) + .collect::>(); + let missing = installer_hashes.difference(&extracted_hashes).count(); + ensure!( + missing == 0, + "{missing} Forge-like installers have no extracted metadata; run import before exporting" + ); + txn.commit() + .context(info_span!("finishing Forge-like export snapshot")) + .await?; + drop(conn); + + let artifacts = deduplicate_artifacts( + extracts + .iter() + .flat_map(|extract| extract.embedded_maven_artifacts.0.clone()), + )?; + let extracts = extracts + .into_iter() + .map(|extract| (extract.installer_sha256, extract)) + .collect::>(); + + installers.sort_by(|a, b| a.name.cmp(&b.name)); + // prepare profiles and catalogs before publishing any files + let mut profiles = BTreeMap::new(); + let mut catalogs = Vec::new(); + for loader in loaders { + let mut games = BTreeMap::>::new(); + for installer in installers + .iter() + .filter(|installer| installer.loader == loader) + { + let extract = &extracts[&installer.sha256]; + let version = loader_version(loader, installer.name.as_str()); + let path = profile_path(loader, version); + let profile = prepare_profile( + extract.metadata.0.clone(), + &extract.embedded_maven_artifacts.0, + |path| app.public_blobs.url_for(path), + )?; + ensure!( + !profile.inherits_from.is_empty(), + "installer {} has no game association", + installer.name + ); + let json = serde_json::to_vec(&profile) + .context("serializing Forge-like profile")?; + if let Some(existing) = profiles.get(&path) { + ensure!( + existing == &json, + "conflicting Forge-like profiles at {path}" + ); + } else { + profiles.insert(path.clone(), json); + } + let entries = games.entry(profile.inherits_from).or_default(); + if !entries.iter().any(|entry| entry.id == version) { + entries.push(LoaderVersion { + id: version.to_owned(), + url: app.public_blobs.url_for(&path), + stable: false, + }); + } + } + let manifest = make_manifest(games); + catalogs.push(( + format!("{}/v{FORMAT_VERSION}/manifest.json", loader_path(loader)), + serde_json::to_vec(&manifest) + .context("serializing Forge-like catalog")?, + )); + } + // publish dependencies before the profiles which reference them + let num_total = artifacts.len(); + let num_done = AtomicUsize::new(0); + info!( + num_artifacts = num_total, + "publishing Forge-like embedded artifacts" + ); + let num_done = &num_done; + stream::iter(artifacts.into_iter().map(Ok::<_, anyhow::Error>)) + .try_for_each_concurrent(app.concurrency.extract_files.get(), |artifact| async move { + let bytes = app.cas.get(artifact.sha256) + .context(info_span!("reading embedded Maven artifact", coordinate = %artifact.coordinate, sha256 = %artifact.sha256)) + .await?; + app.maven.put(&artifact.coordinate, &bytes) + .context(info_span!("publishing embedded Maven artifact", coordinate = %artifact.coordinate)) + .await?; + let done = num_done.fetch_add(1, Ordering::Relaxed) + 1; + if done.is_multiple_of(100) || done == num_total { + info!("published {done}/{num_total} Forge-like embedded artifacts"); + } + anyhow::Ok(()) + }) + .await?; + info!( + num_artifacts = num_total, + "finished publishing Forge-like embedded artifacts" + ); + // publish profiles before the catalogs which reference them + for (path, json) in profiles { + app.public_blobs + .put(&path, &json, ContentType::Json) + .context(info_span!("writing Forge-like profile", %path)) + .await?; + } + for (path, json) in catalogs { + app.public_blobs + .put(&path, &json, ContentType::Json) + .context(info_span!("writing Forge-like catalog", %path)) + .await?; + } + Ok(()) +} + +fn loader_path(loader: ForgelikeLoader) -> &'static str { + match loader { + ForgelikeLoader::Forge => "forge", + ForgelikeLoader::Neoforge => "neo", + } +} + +fn loader_version(loader: ForgelikeLoader, name: &str) -> &str { + match loader { + ForgelikeLoader::Forge => name.split('-').nth(1).unwrap_or(name), + ForgelikeLoader::Neoforge => { + let mut parts = name.split('-'); + let prefix = parts.next().unwrap_or(name); + match parts.next() { + Some(version) + if prefix.contains('.') + && prefix.bytes().all(|byte| { + byte.is_ascii_digit() || byte == b'.' + }) + && version.contains('.') + && version.bytes().all(|byte| { + byte.is_ascii_digit() || byte == b'.' + }) => + { + version + } + _ => name, + } + } + } +} + +fn compare_version_components(left: &str, right: &str) -> std::cmp::Ordering { + let mut left = left.split('.').map(|part| part.trim_start_matches('0')); + let mut right = right.split('.').map(|part| part.trim_start_matches('0')); + loop { + match (left.next(), right.next()) { + (Some(left), Some(right)) => { + let order = + left.len().cmp(&right.len()).then_with(|| left.cmp(right)); + if !order.is_eq() { + return order; + } + } + (None, None) => return std::cmp::Ordering::Equal, + (Some(_), None) => return std::cmp::Ordering::Greater, + (None, Some(_)) => return std::cmp::Ordering::Less, + } + } +} + +fn compare_loader_versions(left: &str, right: &str) -> std::cmp::Ordering { + let (left_version, left_suffix) = left + .split_once('-') + .map_or((left, None), |(version, suffix)| (version, Some(suffix))); + let (right_version, right_suffix) = right + .split_once('-') + .map_or((right, None), |(version, suffix)| (version, Some(suffix))); + compare_version_components(left_version, right_version) + .then_with(|| match (left_suffix, right_suffix) { + (None, Some(_)) => std::cmp::Ordering::Greater, + (Some(_), None) => std::cmp::Ordering::Less, + (Some(left), Some(right)) => left.cmp(right), + (None, None) => std::cmp::Ordering::Equal, + }) + .then_with(|| left.cmp(right)) +} + +fn profile_path(loader: ForgelikeLoader, version: &str) -> String { + format!( + "{}/v{FORMAT_VERSION}/versions/{version}.json", + loader_path(loader) + ) +} + +fn prepare_profile( + mut profile: ProfileMetadata, + artifacts: &[ForgelikeStoredArtifact], + url_for: impl Fn(&str) -> Url, +) -> Result { + let paths = artifacts + .iter() + .map(|artifact| artifact.coordinate.to_maven_path()) + .collect::>(); + for library in &mut profile.libraries { + let coordinate = library.name.parse::()?; + let path = coordinate.to_maven_path(); + if paths.contains(&path) { + library.url = Some(url_for("maven/").to_string()); + library.downloadable = true; + if let Some(artifact) = library + .downloads + .as_mut() + .and_then(|downloads| downloads.artifact.as_mut()) + { + artifact.url = Some(url_for(&path)); + } + } + } + Ok(profile) +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct Manifest { + game_versions: Vec, +} + +#[derive(Debug, Serialize)] +struct GameVersion { + id: String, + stable: bool, + loaders: Vec, +} + +#[derive(Debug, Serialize)] +struct LoaderVersion { + id: String, + url: Url, + stable: bool, +} + +fn make_manifest(games: BTreeMap>) -> Manifest { + Manifest { + game_versions: games + .into_iter() + .map(|(id, mut loaders)| { + // latest and the stable fallback both select the first loader + loaders.sort_by(|left, right| { + compare_loader_versions(&right.id, &left.id) + }); + GameVersion { + id, + stable: true, + loaders, + } + }) + .collect(), + } +} + +fn deduplicate_artifacts( + artifacts: impl IntoIterator, +) -> Result> { + let mut paths = BTreeMap::::new(); + for artifact in artifacts { + let path = artifact.coordinate.to_maven_path(); + if let Some(existing) = paths.get(&path) { + ensure!( + existing.sha256 == artifact.sha256, + "conflicting embedded Maven artifacts at {path}: {} and {}", + existing.sha256, + artifact.sha256 + ); + } else { + paths.insert(path, artifact); + } + } + Ok(paths.into_values().collect()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::util::Sha256; + + fn artifact(coordinate: &str, bytes: &[u8]) -> ForgelikeStoredArtifact { + ForgelikeStoredArtifact { + coordinate: coordinate.parse().unwrap(), + sha256: Sha256::from_digest(bytes), + } + } + + #[test] + fn version_names_and_paths_match_consumers() { + for (raw, expected) in [ + ("1.9-12.16.0.1880-1.9", "12.16.0.1880"), + ("1.10.2-12.18.1.2016-failtests", "12.18.1.2016"), + ("26.1-64.0.10", "64.0.10"), + ("26.1.1-64.0.10-26.1.1", "64.0.10"), + ] { + assert_eq!(loader_version(ForgelikeLoader::Forge, raw), expected); + } + for (raw, expected) in [ + ("1.20.1-47.1.74", "47.1.74"), + ("47.1.83", "47.1.83"), + ("20.2.29-beta", "20.2.29-beta"), + ("26.1.0.10-beta", "26.1.0.10-beta"), + ("26.1.1.10", "26.1.1.10"), + ] { + assert_eq!( + loader_version(ForgelikeLoader::Neoforge, raw), + expected + ); + } + assert_eq!( + profile_path(ForgelikeLoader::Forge, "49.0.1"), + "forge/v0/versions/49.0.1.json" + ); + assert_eq!( + profile_path(ForgelikeLoader::Neoforge, "26.1.0.10-beta"), + "neo/v0/versions/26.1.0.10-beta.json" + ); + assert_eq!( + profile_path( + ForgelikeLoader::Neoforge, + "26.1.0.0-alpha.1+snapshot-1" + ), + "neo/v0/versions/26.1.0.0-alpha.1+snapshot-1.json" + ); + } + + #[test] + fn catalog_serialization_preserves_game_order_and_sorts_loaders_latest_first() + { + let entry = |id: &str| LoaderVersion { + id: id.into(), + stable: false, + url: format!("https://meta.example/forge/v0/versions/{id}.json") + .parse() + .unwrap(), + }; + let manifest = make_manifest(BTreeMap::from([ + ("1.9".into(), vec![entry("12.1"), entry("12.2")]), + ("1.10.2".into(), vec![entry("13.1")]), + ])); + let value = serde_json::to_value(manifest).unwrap(); + assert_eq!(value["gameVersions"][0]["id"], "1.10.2"); + assert_eq!(value["gameVersions"][1]["loaders"][0]["id"], "12.2"); + assert_eq!(value["gameVersions"][1]["stable"], true); + assert_eq!(value["gameVersions"][1]["loaders"][0]["stable"], false); + assert!(value.get("versionGroups").is_none()); + assert!(value["gameVersions"][0].get("versionGroup").is_none()); + } + + #[test] + fn numeric_order_selects_latest_for_forge_and_neoforge() { + for (game, versions, expected) in [ + ( + "1.20.1", + vec!["47.1.9", "47.1.100", "47.1.10"], + vec!["47.1.100", "47.1.10", "47.1.9"], + ), + ( + "1.20.6", + vec!["20.6.9", "20.6.119", "20.6.119-beta", "20.6.120-beta"], + vec!["20.6.120-beta", "20.6.119", "20.6.119-beta", "20.6.9"], + ), + ( + "26.1", + vec!["26.1.0.9-beta", "26.1.0.10-beta", "26.1.0.10"], + vec!["26.1.0.10", "26.1.0.10-beta", "26.1.0.9-beta"], + ), + ] { + let entries = versions + .into_iter() + .map(|id| LoaderVersion { + id: id.into(), + stable: false, + url: "https://meta.example/profile.json".parse().unwrap(), + }) + .collect(); + let manifest = + make_manifest(BTreeMap::from([(game.into(), entries)])); + let loaders = &manifest.game_versions[0].loaders; + assert_eq!( + loaders + .iter() + .map(|loader| loader.id.as_str()) + .collect::>(), + expected + ); + + } + assert!( + compare_loader_versions( + "1.999999999999999999999999999999", + "1.1000000000000000000000000000000" + ) + .is_lt() + ); + } + + #[test] + fn rewrites_only_stored_dependencies_and_preserves_install_metadata() { + let profile: ProfileMetadata = serde_json::from_value(serde_json::json!({ + "id": "neo", "inheritsFrom": "26.1", "releaseTime": "2026-01-01T00:00:00Z", "time": "2026-01-01T00:00:00Z", "type": "release", + "libraries": [ + {"name": "g:a:1@jar", "include_in_classpath": false, "downloads": {"artifact": {"url": "https://upstream.example/a.jar", "sha1": "0000000000000000000000000000000000000000", "size": 12}}}, + {"name": "g:b:1", "downloadable": false, "downloads": {"artifact": {"url": "", "sha1": "0000000000000000000000000000000000000000", "size": 0}}}, + {"name": "g:c:1", "url": "https://libraries.minecraft.net/"} + ], + "data": {"BINPATCH": {"client": "/data/client.lzma", "server": "/data/server.lzma"}}, + "processors": [{"jar": "g:a:1", "classpath": [], "args": ["{BINPATCH}"], "sides": ["client"]}] + })).unwrap(); + let original = serde_json::to_value(&profile).unwrap(); + for base in ["https://meta.example/", "https://meta.example/prefix/"] { + let base = Url::parse(base).unwrap(); + let published = prepare_profile( + profile.clone(), + &[artifact("g:a:1", b"a")], + |path| base.join(path).unwrap(), + ) + .unwrap(); + let mut expected = original.clone(); + expected["libraries"][0]["url"] = + serde_json::json!(base.join("maven/").unwrap()); + expected["libraries"][0]["downloads"]["artifact"]["url"] = + serde_json::json!(base.join("maven/g/a/1/a-1.jar").unwrap()); + assert_eq!(serde_json::to_value(published).unwrap(), expected); + } + + } + + #[test] + fn deduplicates_coordinate_paths_including_default_extension() { + let artifacts = deduplicate_artifacts([ + artifact("net.example:library:1", b"same"), + artifact("net.example:library:1@jar", b"same"), + ]) + .unwrap(); + assert_eq!(artifacts.len(), 1); + } + + #[test] + fn rejects_conflicting_content_before_publication() { + assert!( + deduplicate_artifacts([ + artifact("net.example:library:1", b"first"), + artifact("net.example:library:1", b"second"), + ]) + .is_err() + ); + } +} diff --git a/apps/meta-generator/src/export/mod.rs b/apps/meta-generator/src/export/mod.rs new file mode 100644 index 00000000000..3af08163975 --- /dev/null +++ b/apps/meta-generator/src/export/mod.rs @@ -0,0 +1,64 @@ +//! Schemas and logic for generating manifest artifacts consumed by the Modrinth +//! App. + +use anyhow::{Result, ensure}; +use tracing::{info_span, warn}; +use tracing_anyhow::FutureContext; + +use crate::{AppState, upstream::Upstreams}; + +mod fabric; +mod fabriclike; +mod forgelike; +mod mojang; +mod quilt; + +pub async fn run(app: AppState, upstreams: Upstreams) -> Result<()> { + let mut failed = Vec::new(); + if upstreams.mojang { + mojang::export(&app) + .context(info_span!("exporting Mojang artifacts")) + .await + .inspect_err(|err| { + warn!("error: {err:?}"); + failed.push("Mojang"); + }) + .ok(); + } + + if upstreams.fabric { + fabric::export(&app) + .context(info_span!("exporting Fabric artifacts")) + .await + .inspect_err(|err| { + warn!("error: {err:?}"); + failed.push("Fabric"); + }) + .ok(); + } + + if upstreams.quilt { + quilt::export(&app) + .context(info_span!("exporting Quilt artifacts")) + .await + .inspect_err(|err| { + warn!("error: {err:?}"); + failed.push("Quilt"); + }) + .ok(); + } + + if upstreams.forge || upstreams.neoforge { + forgelike::export(&app, upstreams) + .context(info_span!("exporting Forge-like artifacts")) + .await + .inspect_err(|err| { + warn!("error: {err:?}"); + failed.push("Forge/NeoForge"); + }) + .ok(); + } + + ensure!(failed.is_empty(), "export failed for {}", failed.join(", ")); + Ok(()) +} diff --git a/apps/meta-generator/src/export/mojang.rs b/apps/meta-generator/src/export/mojang.rs new file mode 100644 index 00000000000..e8f2dfe86e0 --- /dev/null +++ b/apps/meta-generator/src/export/mojang.rs @@ -0,0 +1,140 @@ +use anyhow::{Result, bail, ensure}; +use tracing::{debug, info, info_span}; +use tracing_anyhow::FutureContext; + +use crate::{ + AppState, model, + store::ContentType, + upstream::mojang, + util::{Sha1, to_json_string}, +}; + +pub const FORMAT_VERSION: u32 = 0; + +struct VersionManifestExport { + version: mojang::Version, + json: String, + path: String, +} + +pub async fn export(app: &AppState) -> Result<()> { + let mut conn = app + .db + .connection() + .context(info_span!("acquiring connection")) + .await?; + + let rows = model::MinecraftVersion::all() + .exec(&mut conn) + .context(info_span!("fetching extracted Minecraft versions")) + .await?; + ensure!( + !rows.is_empty(), + "no extracted Minecraft versions; run Mojang extraction before exporting" + ); + info!("found {} extracted Minecraft versions", rows.len()); + + let latest_release = { + let mut all_latest = rows + .iter() + .filter(|row| row.is_latest_release == Some(true)) + .collect::>() + .into_iter(); + let (Some(latest), None) = (all_latest.next(), all_latest.next()) + else { + bail!( + "expected exactly one extracted latest release; \ + run Mojang extraction before exporting", + ); + }; + latest + }; + + let latest_snapshot = { + let mut all_latest = rows + .iter() + .filter(|row| row.is_latest_snapshot == Some(true)) + .collect::>() + .into_iter(); + let (Some(latest), None) = (all_latest.next(), all_latest.next()) + else { + bail!( + "expected exactly one extracted latest snapshot; \ + run Mojang extraction before exporting", + ); + }; + latest + }; + + let latest = mojang::Latest { + release: latest_release.name.clone(), + snapshot: latest_snapshot.name.clone(), + }; + info!(release = %latest.release, snapshot = %latest.snapshot, "found latest versions"); + + // prepare all version manifests before writing + let mut exports = rows + .into_iter() + .map(|mut row| { + let json = to_json_string(&row.manifest); + row.catalog_entry.0.sha1 = Sha1::from_digest(json.as_bytes()); + let path = format!( + "minecraft/v{FORMAT_VERSION}/versions/{}.json", + row.name + ); + row.catalog_entry.0.url = app.public_blobs.url_for(&path); + + VersionManifestExport { + version: row.catalog_entry.0, + json, + path, + } + }) + .collect::>(); + + // sort by latest-first, breaking ties by version ID + exports.sort_by_key(|export| { + ( + std::cmp::Reverse(export.version.release_time), + export.version.id.clone(), + ) + }); + + let uber_catalog = mojang::Catalog { + latest, + versions: exports + .iter() + .map(|export| export.version.clone()) + .collect(), + }; + let data = to_json_string(&uber_catalog); + + // write processed version manifests + let num_total = exports.len(); + for (index, export) in exports.iter().enumerate() { + let game_version = &export.version.id; + let path = &export.path; + app.public_blobs + .put(path, export.json.as_bytes(), ContentType::Json) + .context( + info_span!("writing manifest to blob store", %game_version, %path), + ) + .await?; + debug!(%path, "wrote version manifest to blob store"); + + let num_done = index + 1; + if num_done.is_multiple_of(100) || num_done == num_total { + info!("wrote {num_done}/{num_total} version manifests"); + } + } + + // write manifest (uber catalog) + let path = format!("minecraft/v{FORMAT_VERSION}/manifest.json"); + app.public_blobs + .put(&path, data.as_bytes(), ContentType::Json) + .context(info_span!("writing manifest to blob store", %path)) + .await?; + info!(%path, "wrote uber catalog (manifest) to blob store"); + + Ok(()) +} diff --git a/apps/meta-generator/src/export/quilt.rs b/apps/meta-generator/src/export/quilt.rs new file mode 100644 index 00000000000..3a0785209be --- /dev/null +++ b/apps/meta-generator/src/export/quilt.rs @@ -0,0 +1,9 @@ +use anyhow::Result; + +use crate::{AppState, export::fabriclike, model::FabriclikeLoader}; + +pub const FORMAT_VERSION: u32 = 1; + +pub async fn export(app: &AppState) -> Result<()> { + fabriclike::export(app, FabriclikeLoader::Quilt, FORMAT_VERSION).await +} diff --git a/apps/meta-generator/src/import/download.rs b/apps/meta-generator/src/import/download.rs new file mode 100644 index 00000000000..3a7f02c736f --- /dev/null +++ b/apps/meta-generator/src/import/download.rs @@ -0,0 +1,236 @@ +use std::any::type_name; + +use anyhow::{Context, Result, anyhow}; +use jiff::Timestamp; +use reqwest::IntoUrl; +use serde::de::DeserializeOwned; +use tokio::sync::Mutex; +use tracing::{info, info_span}; +use tracing_anyhow::FutureContext; + +use crate::{ + AppState, + import::{fabric, forge, mojang, neoforge, quilt}, + model::{self, DownloadRunId}, + store::BlobCas, + upstream::Upstreams, + util::{ + ErrorAccumulator, ResponseExt, Sha256, from_json_str, from_json_value, + }, +}; + +pub struct DownloadRunContext<'cx> { + pub http: &'cx reqwest::Client, + pub cas: &'cx BlobCas, + pub conn: Mutex<&'cx mut toasty::Connection>, + pub download_run_id: DownloadRunId, + pub download_concurrency: usize, + pub errors: ErrorAccumulator, +} + +pub async fn download_from_upstreams( + app: &AppState, + upstreams: Upstreams, +) -> Result { + let mut conn = app + .db + .connection() + .context(info_span!("acquiring db connection")) + .await?; + + let mut download_run = toasty::create!(model::DownloadRun { + started_at: Timestamp::now(), + }) + .exec(&mut conn) + .context(info_span!("creating download run")) + .await?; + info!(?download_run.id, "starting download run"); + + let mut cx = DownloadRunContext { + http: &app.http, + cas: &app.cas, + conn: Mutex::new(&mut conn), + download_run_id: download_run.id, + download_concurrency: app.concurrency.download.get(), + errors: ErrorAccumulator::new(), + }; + + let mut completed = Upstreams { + mojang: false, + fabric: false, + forge: false, + neoforge: false, + quilt: false, + }; + + if upstreams.mojang { + let errors_before = cx.errors.len(); + let result = mojang::download(&mut cx) + .context(info_span!("downloading Mojang upstream")) + .await; + completed.mojang = + download_succeeded(result, &cx.errors, errors_before); + } + + if upstreams.fabric { + let errors_before = cx.errors.len(); + let result = fabric::download(&mut cx) + .context(info_span!("downloading Fabric upstream")) + .await; + completed.fabric = + download_succeeded(result, &cx.errors, errors_before); + } + + if upstreams.neoforge { + let errors_before = cx.errors.len(); + let result = neoforge::download(&mut cx) + .context(info_span!("downloading NeoForge upstream")) + .await; + completed.neoforge = + download_succeeded(result, &cx.errors, errors_before); + } + + if upstreams.quilt { + let errors_before = cx.errors.len(); + let result = quilt::download(&mut cx) + .context(info_span!("downloading Quilt upstream")) + .await; + completed.quilt = download_succeeded(result, &cx.errors, errors_before); + } + + // do Forge last since we have to download a lot of installers + if upstreams.forge { + let errors_before = cx.errors.len(); + let result = forge::download(&mut cx) + .context(info_span!("downloading Forge upstream")) + .await; + completed.forge = download_succeeded(result, &cx.errors, errors_before); + } + + toasty::update!(download_run { + completed_at: Timestamp::now(), + errors: toasty::Json(cx.errors.finish()), + }) + .exec(&mut conn) + .context(info_span!("marking download run as completed")) + .await?; + + Ok(completed) +} + +fn download_succeeded( + result: Result<()>, + errors: &ErrorAccumulator, + errors_before: usize, +) -> bool { + if let Err(err) = result { + errors.push(&err); + return false; + } + errors.len() == errors_before +} + +impl<'cx> DownloadRunContext<'cx> { + pub async fn conn( + &self, + ) -> tokio::sync::MutexGuard<'_, &'cx mut toasty::Connection> { + self.conn.lock().await + } + + pub async fn download_blob(&self, url: impl IntoUrl) -> Result { + let url = url.into_url().context("converting to URL")?; + let bytes = async { + self.http + .get(url.clone()) + .send() + .await? + .error_for_status_ext() + .await? + .bytes() + .await + .map_err(anyhow::Error::from) + } + .context(info_span!("fetching bytes")) + .await?; + + let sha256 = self.insert_blob(url.as_str(), &bytes).await?; + Ok(sha256) + } + + pub async fn download_json( + &self, + url: impl IntoUrl, + ) -> Result<(T, Sha256)> { + let url = url.into_url().context("converting to URL")?; + let text = async { + self.http + .get(url.clone()) + .send() + .await? + .error_for_status_ext() + .await? + .text() + .await + .map_err(anyhow::Error::from) + } + .context(info_span!("fetching text")) + .await?; + let json = from_json_str::(&text) + .context("text is not valid JSON")?; + + let sha256 = self.insert_blob(url.as_str(), text.as_bytes()).await?; + + let t = from_json_value::(&json) + .with_context(|| anyhow!("parsing as `{}`", type_name::()))?; + Ok((t, sha256)) + } + + async fn insert_blob(&self, url: &str, data: &[u8]) -> Result { + let mut conn = self.conn.lock().await; + let mut txn = conn + .transaction() + .context(info_span!("starting transaction")) + .await?; + + let sha256 = self + .cas + .put(&mut txn, data) + .context(info_span!("storing blob")) + .await?; + toasty::create!(model::BlobDownload { + download_run_id: self.download_run_id, + sha256, + url, + }) + .exec(&mut txn) + .context(info_span!("inserting blob download")) + .await?; + + txn.commit() + .context(info_span!("committing transaction")) + .await?; + + Ok(sha256) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn download_success_requires_no_new_recorded_errors() { + let errors = ErrorAccumulator::new(); + assert!(download_succeeded(Ok(()), &errors, 0)); + errors.push(&anyhow!("failed to download a profile")); + assert!(!download_succeeded(Ok(()), &errors, 0)); + let previous = errors.len(); + assert!(download_succeeded(Ok(()), &errors, previous)); + assert!(!download_succeeded( + Err(anyhow!("failed to fetch catalog")), + &errors, + previous + )); + assert_eq!(errors.len(), previous + 1); + } +} diff --git a/apps/meta-generator/src/import/extract/fabric.rs b/apps/meta-generator/src/import/extract/fabric.rs new file mode 100644 index 00000000000..b7ce0b627c9 --- /dev/null +++ b/apps/meta-generator/src/import/extract/fabric.rs @@ -0,0 +1,134 @@ +use std::collections::HashMap; + +use anyhow::{Context, Result}; +use indexmap::IndexSet; +use tracing::{info, info_span}; +use tracing_anyhow::FutureContext; +use url::Url; + +use crate::{ + AppState, + import::extract::fabriclike::{ + self, Artifact, CatalogGame, CatalogLoader, CatalogSnapshot, + ProfileGroup, + }, + model::{self, FabriclikeLoader}, + upstream::fabric, + util::from_json_slice, +}; + +pub async fn extract(app: &AppState) -> Result<()> { + let mut conn = app + .db + .connection() + .context(info_span!("acquiring connection")) + .await?; + let mut rows = model::FabricCatalog::all() + .exec(&mut conn) + .context(info_span!("fetching Fabric catalogs")) + .await?; + info!("found {} downloaded catalogs", rows.len()); + + // fetch download runs where we downloaded these catalogs from; + // newer catalogs take precedence over older ones + let runs = model::DownloadRun::all() + .exec(&mut conn) + .context(info_span!("fetching download runs")) + .await? + .into_iter() + .map(|run| (run.id, run.started_at)) + .collect::>(); + rows.sort_by_key(|row| { + ( + std::cmp::Reverse(runs.get(&row.download_run_id).copied()), + row.id.0, + ) + }); + + // read each catalog from its blob in the store, + // and start assembling the artifacts we'll extract + let repository = Url::parse(crate::import::fabric::MAVEN_URL)?; + let mut catalogs = Vec::new(); + let mut games = IndexSet::new(); + for row in rows { + let bytes = app + .cas + .get(row.sha256) + .context( + info_span!("fetching Fabric catalog", sha256 = %row.sha256), + ) + .await?; + let catalog = from_json_slice::(&bytes) + .context("parsing Fabric catalog")?; + + for game in &catalog.game { + games.insert(game.version.to_string()); + } + + catalogs.push(CatalogSnapshot { + download_run_id: row.download_run_id, + sha256: row.sha256, + games: catalog + .game + .into_iter() + .map(|game| CatalogGame { + id: game.version.to_string(), + stable: game.stable, + }) + .collect(), + loaders: catalog + .loader + .into_iter() + .map(|loader| CatalogLoader { + id: loader.version.to_string(), + stable: loader.stable, + }) + .collect(), + mappings: + catalog + .intermediary + .into_iter() + .map(|mapping| Artifact { + coordinate: mapping.maven, + repository: repository.clone(), + source: Some(format!( + "{} (catalog SHA256 {})", + fabric::CATALOG_URL, + row.sha256 + )), + repository_source: + "catalog intermediary: loader Maven fallback" + .to_owned(), + }) + .collect(), + }); + } + + // setup version groups + // see README.md on what version groups are and why we need them + // Fabric only has one, since it takes a single version (`PROFILE_GAME_VERSION`) + // and templates that into all the json files. Quilt has more. + let group = ProfileGroup { + id: None, + template: crate::import::fabric::PROFILE_GAME_VERSION.to_owned(), + fallback_templates: Vec::new(), + games: games.into_iter().collect(), + profile_base_url: Url::parse(&format!( + "{}/loader/", + fabric::CATALOG_URL + ))?, + }; + + fabriclike::extract_catalogs( + app, + FabriclikeLoader::Fabric, + &repository, + catalogs, + vec![group], + ) + .context(info_span!( + "extracting catalogs", + loader = ?FabriclikeLoader::Fabric + )) + .await +} diff --git a/apps/meta-generator/src/import/extract/fabriclike.rs b/apps/meta-generator/src/import/extract/fabriclike.rs new file mode 100644 index 00000000000..b546cc84612 --- /dev/null +++ b/apps/meta-generator/src/import/extract/fabriclike.rs @@ -0,0 +1,1022 @@ +use std::collections::{HashMap, HashSet}; + +use anyhow::{Context, Result, anyhow, ensure}; +use futures::{StreamExt, TryStreamExt, stream}; +use indexmap::IndexMap; +use tracing::{debug, info, info_span, warn}; +use tracing_anyhow::FutureContext; +use url::Url; + +use crate::{ + AppState, model, + model::{ + FabriclikeLoader, FabriclikeStoredArtifact, GAME_VERSION_PLACEHOLDER, + }, + util::{MavenCoordinate, ResponseExt, Sha1, Sha256, from_json_slice}, +}; + +pub use crate::model::{ + FabriclikeArtifact as Artifact, FabriclikeGameVersion as CatalogGame, + FabriclikeProfileGroup as ProfileGroup, + FabriclikeProfileMetadata as ProfileMetadata, +}; + +/// Selects and normalizes downloaded profiles without publishing any files. +/// Catalog state and profiles are replaced atomically after all dependencies +/// have been downloaded into CAS. +pub async fn extract_catalogs( + app: &AppState, + loader: FabriclikeLoader, + fallback_maven: &Url, + catalogs: Vec, + groups: Vec, +) -> Result<()> { + // merge catalogs, keeping the newest metadata for each version; + // older catalogs can still contribute versions missing from newer ones + let required_loaders = catalogs + .first() + .into_iter() + .flat_map(|snapshot| { + snapshot.loaders.iter().map(|version| version.id.clone()) + }) + .collect::>(); + let mut games = IndexMap::new(); + let mut loaders = IndexMap::new(); + let mut artifacts = IndexMap::new(); + let mut run_priority = HashMap::new(); + for (priority, snapshot) in catalogs.into_iter().enumerate() { + run_priority + .entry(snapshot.download_run_id) + .or_insert(priority); + for game in snapshot.games { + games.entry(game.id.clone()).or_insert(game); + } + for version in snapshot.loaders { + loaders.entry(version.id.clone()).or_insert(version); + } + for mut artifact in snapshot.mappings { + if artifact.source.is_none() { + artifact.source = + Some(format!("catalog SHA256 {}", snapshot.sha256)); + } + artifacts + .entry(artifact.coordinate.to_maven_path()) + .or_insert(artifact); + } + } + ensure!( + !games.is_empty() && !loaders.is_empty() && !groups.is_empty(), + "no game versions, loaders, or profile groups to extract" + ); + // only fetch download records for the profiles we're going to use; + // include historical templates in case a loader has no current profile + let mut profile_urls = Vec::new(); + for group in &groups { + for version in loaders.keys() { + for template in std::iter::once(&group.template) + .chain(&group.fallback_templates) + { + profile_urls.push( + profile_source(group, template, version)?.to_string(), + ); + } + } + } + let mut conn = app + .db + .connection() + .context(info_span!("acquiring connection")) + .await?; + let mut downloads = model::BlobDownload::all() + .filter(model::BlobDownload::fields().url().in_list(profile_urls)) + .exec(&mut conn) + .context(info_span!("fetching downloaded profiles")) + .await?; + downloads.sort_by_key(|row| { + ( + run_priority + .get(&row.download_run_id) + .copied() + .unwrap_or(usize::MAX), + row.download_run_id.0, + ) + }); + let mut profile_hashes = HashMap::new(); + for download in downloads { + profile_hashes + .entry(download.url) + .or_insert(download.sha256); + } + // now we make all the loader profiles and collect their libraries + let mut profiles = Vec::new(); + for group in &groups { + for version in loaders.values() { + // prefer the current template, then try older ones from this group; + // normalize against the template we actually downloaded + let Some((template, source, sha256)) = select_profile( + loader, + group, + &version.id, + &profile_hashes, + required_loaders.contains(&version.id), + )? + else { + warn!( + ?loader, + loader_version = %version.id, + group = ?group.id, + "skipping historical loader without a downloaded profile" + ); + continue; + }; + let bytes = app.cas.get(sha256) + .context(info_span!("fetching loader profile", loader_version = %version.id, %source)) + .await?; + let profile = from_json_slice::(&bytes) + .context("parsing loader profile")?; + let (profile, dependencies) = normalize_profile( + profile, + template, + &group.games, + fallback_maven, + ) + .with_context(|| { + format!("normalizing profile {} from {source}", version.id) + })?; + for mut artifact in dependencies { + artifact.source = + Some(format!("{source} (profile SHA256 {sha256})")); + artifacts + .entry(artifact.coordinate.to_maven_path()) + .or_insert(artifact); + } + profiles.push(SelectedProfile { + loader_version: version.id.clone(), + stable: version.stable, + version_group: group.id.clone().unwrap_or_default(), + source_url: source.to_string(), + source_sha256: sha256, + template: template.to_owned(), + metadata: profile, + }); + } + } + validate_groups(&games, &groups)?; + drop(conn); + + // store dependencies before committing profiles which reference them; + // a failed download leaves the previous extracted dataset intact + let num_total = artifacts.len(); + let mut num_done = 0usize; + info!(num_artifacts = num_total, "downloading loader dependencies"); + let artifacts = stream::iter(artifacts.into_values()) + .map(|artifact| download_artifact(app, artifact)) + .buffered(app.concurrency.download.get()) + .inspect_ok(|_| { + num_done += 1; + if num_done.is_multiple_of(100) || num_done == num_total { + info!("downloaded {num_done}/{num_total} dependencies"); + } + }) + .try_collect::>() + .context(info_span!("downloading loader dependencies")) + .await?; + + let mut conn = app + .db + .connection() + .context(info_span!("acquiring connection")) + .await?; + let mut txn = conn + .transaction() + .context(info_span!("starting extraction transaction")) + .await?; + model::FabriclikeProfile::all() + .filter(model::FabriclikeProfile::fields().loader().eq(loader)) + .delete() + .exec(&mut txn) + .context(info_span!("clearing previous loader profiles", ?loader)) + .await?; + let num_profiles = profiles.len(); + for (position, profile) in profiles.into_iter().enumerate() { + model::FabriclikeProfile::create() + .loader(loader) + .version_group(profile.version_group) + .loader_version(profile.loader_version) + .stable(profile.stable) + .position( + i32::try_from(position).context("too many loader profiles")?, + ) + .source_url(profile.source_url) + .source_sha256(profile.source_sha256) + .template(profile.template) + .metadata(toasty::Json(profile.metadata)) + .exec(&mut txn) + .context(info_span!("storing extracted loader profile", ?loader)) + .await?; + } + model::FabriclikeCatalog::upsert_by_loader(loader) + .games(toasty::Json(games.into_values().collect::>())) + .groups(toasty::Json(groups)) + .artifacts(toasty::Json(artifacts)) + .exec(&mut txn) + .context(info_span!("storing extracted loader catalog", ?loader)) + .await?; + txn.commit() + .context(info_span!("committing extracted loader metadata", ?loader)) + .await?; + info!(?loader, num_profiles, "stored loader metadata"); + Ok(()) +} + +fn artifact_url(artifact: &Artifact) -> Result { + let mut url = artifact.repository.clone(); + + // old profiles reference HTTP repositories which now require HTTPS + if url.scheme() == "http" { + warn!( + %url, + coordinate = %artifact.coordinate, + source = artifact.source.as_deref().unwrap_or("unknown"), + repository = %artifact.repository, + repository_source = artifact.repository_source, + "upgraded Maven artifact URL from HTTP to HTTPS", + ); + url.set_scheme("https").map_err(|()| { + anyhow!("cannot upgrade Maven repository URL to HTTPS") + })?; + } + let path = artifact.coordinate.to_maven_path(); + let relative = path + .strip_prefix("maven/") + .context("missing Maven path prefix")?; + url.path_segments_mut() + .map_err(|()| { + anyhow!("repository URL cannot be a base: {}", artifact.repository) + })? + .pop_if_empty() + .extend(relative.split('/')); + Ok(url) +} + +async fn download_artifact( + app: &AppState, + artifact: Artifact, +) -> Result { + let url = artifact_url(&artifact)?; + + let mut checksum_url = url.clone(); + checksum_url.set_path(&format!("{}.sha256", url.path())); + + let response = app.http.get(checksum_url.clone()).send() + .context(info_span!("fetching Maven artifact checksum", url = %checksum_url, coordinate = %artifact.coordinate)) + .await?; + let use_sha1 = response.status() == reqwest::StatusCode::NOT_FOUND; + let response = if use_sha1 { + debug!(%url, coordinate = %artifact.coordinate, "Maven artifact SHA256 checksum unavailable; using SHA1"); + checksum_url.set_path(&format!("{}.sha1", url.path())); + app.http.get(checksum_url.clone()).send() + .context(info_span!("fetching Maven artifact SHA1 checksum", url = %checksum_url, coordinate = %artifact.coordinate)) + .await? + } else { + response + }; + + let checksum = async { + response + .error_for_status_ext() + .await? + .text() + .await + .map_err(anyhow::Error::from) + } + .context(info_span!("reading Maven artifact checksum", url = %checksum_url)) + .await?; + let (expected_sha256, expected_sha1) = if use_sha1 { + let mut digest = [0; 20]; + hex::decode_to_slice(checksum.trim(), &mut digest).with_context( + || format!("invalid Maven artifact SHA1 at {checksum_url}"), + )?; + (None, Some(Sha1(digest))) + } else { + let mut digest = [0; 32]; + hex::decode_to_slice(checksum.trim(), &mut digest).with_context( + || format!("invalid Maven artifact SHA256 at {checksum_url}"), + )?; + (Some(Sha256(digest)), None) + }; + + let mut conn = app + .db + .connection() + .context(info_span!("acquiring connection")) + .await?; + let query = model::BlobHash::all(); + let query = if let Some(sha256) = expected_sha256 { + query.filter(model::BlobHash::fields().sha256().eq(sha256)) + } else { + query + .filter(model::BlobHash::fields().sha1().eq(expected_sha1.unwrap())) + }; + let existing = query + .exec(&mut conn) + .context(info_span!("checking Maven artifact CAS hash")) + .await?; + drop(conn); + if let Some(existing) = existing.first() { + let expected_sha256 = existing.sha256; + app.cas.get(expected_sha256) + .context(info_span!("fetching cached Maven artifact", sha256 = %expected_sha256, coordinate = %artifact.coordinate)) + .await?; + return Ok(FabriclikeStoredArtifact { + artifact, + sha256: expected_sha256, + }); + } + + let bytes = async { + app.http + .get(url.clone()) + .send() + .await? + .error_for_status_ext() + .await? + .bytes() + .await + .map_err(anyhow::Error::from) + } + .context(info_span!("fetching Maven artifact", %url, coordinate = %artifact.coordinate, + source = artifact.source.as_deref().unwrap_or("unknown"), + repository = %artifact.repository, repository_source = artifact.repository_source, + scheme = url.scheme())) + .await?; + + if let Some(expected) = expected_sha256 { + ensure!( + Sha256::from_digest(&bytes) == expected, + "Maven artifact SHA256 mismatch for {url}: expected {expected}, got {}", + Sha256::from_digest(&bytes) + ); + } + if let Some(expected) = expected_sha1 { + ensure!( + Sha1::from_digest(&bytes) == expected, + "Maven artifact SHA1 mismatch for {url}: expected {expected}, got {}", + Sha1::from_digest(&bytes) + ); + } + let mut conn = app + .db + .connection() + .context(info_span!("acquiring connection")) + .await?; + let sha256 = app.cas + .put(&mut conn, &bytes) + .context(info_span!("storing Maven artifact in CAS", coordinate = %artifact.coordinate)) + .await?; + Ok(FabriclikeStoredArtifact { artifact, sha256 }) +} + +fn validate_groups( + games: &IndexMap, + groups: &[ProfileGroup], +) -> Result<()> { + let mut assigned = std::collections::HashSet::new(); + let mut ids = std::collections::HashSet::new(); + for group in groups { + ensure!( + ids.insert(&group.id), + "duplicate profile group {:?}", + group.id + ); + for game in &group.games { + ensure!( + games.contains_key(game), + "profile group references unknown game {game}" + ); + ensure!( + assigned.insert(game), + "game version {game} belongs to multiple profile groups" + ); + } + } + for game in games.keys() { + ensure!( + assigned.contains(game), + "game version {game} has no profile group" + ); + } + Ok(()) +} + +/// Takes a downloaded loader profile, replaces its game-specific metadata with +/// placeholders, retaining source repositories for later publication. +/// +/// Returns the processed profile and the concrete artifacts needed by its games. +/// Callers should dedup artifacts by Maven path before mirroring them. +pub fn normalize_profile( + mut profile: ProfileMetadata, + template: &str, + games: &[String], + fallback_maven: &Url, +) -> Result<(ProfileMetadata, Vec)> { + if let Some(prefix) = profile.id.strip_suffix(template) { + profile.id = format!("{prefix}{GAME_VERSION_PLACEHOLDER}"); + } + profile.inherits_from = profile + .inherits_from + .replace(template, GAME_VERSION_PLACEHOLDER); + + let mut artifacts = Vec::new(); + for (index, library) in profile.libraries.iter_mut().enumerate() { + let name = library.name.clone(); + let coordinate = + name.parse::().with_context(|| { + format!("library {index} has an invalid Maven name") + })?; + + let repository_source = if name == "net.minecraft:launchwrapper:1.12" { + "launchwrapper override" + } else if library.url.is_some() { + "profile library.url" + } else { + "loader Maven fallback" + }; + let repository = if name == "net.minecraft:launchwrapper:1.12" { + // special case this because it's not available on Fabric's Maven repo + Url::parse("https://libraries.minecraft.net/")? + } else { + match &library.url { + None => fallback_maven.clone(), + Some(value) => Url::parse(value).with_context(|| { + anyhow!( + "library {index} has an invalid repository URL {value:?}" + ) + })?, + } + }; + + // rewrite the specific `template` (game version) + // to `GAME_VERSION_PLACEHOLDER`, so that app consumer + // can replace the placeholder to the real game version later. + // we only rewrite the coordinate if it's one of the ones + // that we know contains a game version + // (like fabric intermediary coordinate). + if matches!( + ( + coordinate.group_id.as_str(), + coordinate.artifact_id.as_str() + ), + ("net.fabricmc", "intermediary") | ("org.quiltmc", "hashed") + ) { + ensure!( + coordinate.version.as_str() == template, + "library {index} mapping version {} does not match template {template:?}", + coordinate.version + ); + for game in games { + let mut concrete = coordinate.clone(); + concrete.version = game.parse().with_context(|| { + format!( + "library {index} has an invalid game version {game:?}" + ) + })?; + artifacts.push(Artifact { + coordinate: concrete, + repository: repository.clone(), + source: None, + repository_source: repository_source.to_owned(), + }); + } + let classifier = coordinate + .classifier + .as_ref() + .map(|value| format!(":{value}")) + .unwrap_or_default(); + let extension = coordinate + .extension + .as_ref() + .map(|value| format!("@{value}")) + .unwrap_or_default(); + library.name = format!( + "{}:{}:{GAME_VERSION_PLACEHOLDER}{classifier}{extension}", + coordinate.group_id, coordinate.artifact_id + ); + } else { + artifacts.push(Artifact { + coordinate, + repository, + source: None, + repository_source: repository_source.to_owned(), + }); + } + } + Ok((profile, artifacts)) +} +pub struct CatalogLoader { + pub id: String, + pub stable: bool, +} + +pub struct CatalogSnapshot { + pub download_run_id: model::DownloadRunId, + pub sha256: Sha256, + pub games: Vec, + pub loaders: Vec, + pub mappings: Vec, +} + +struct SelectedProfile { + loader_version: String, + version_group: String, + stable: bool, + source_url: String, + source_sha256: Sha256, + template: String, + metadata: ProfileMetadata, +} + +fn profile_source( + group: &ProfileGroup, + template: &str, + loader_version: &str, +) -> Result { + let mut url = group.profile_base_url.clone(); + url.path_segments_mut() + .map_err(|()| anyhow!("profile base URL cannot contain path segments"))? + .pop_if_empty() + .extend([template, loader_version, "profile", "json"]); + Ok(url) +} + +fn select_profile<'a>( + loader: FabriclikeLoader, + group: &'a ProfileGroup, + loader_version: &str, + hashes: &HashMap, + required: bool, +) -> Result> { + for template in + std::iter::once(&group.template).chain(&group.fallback_templates) + { + let source = profile_source(group, template, loader_version)?; + if let Some(sha256) = hashes.get(source.as_str()) { + return Ok(Some((template.as_str(), source, *sha256))); + } + } + ensure!( + !required, + "no downloaded {loader:?} profile for {loader_version} in group {:?}", + group.id, + ); + Ok(None) +} + +#[cfg(test)] +mod tests { + use std::slice; + + use super::*; + use serde_json::{Value, json}; + + fn group(id: Option<&str>, template: &str, games: &[&str]) -> ProfileGroup { + ProfileGroup { + id: id.map(str::to_owned), + template: template.into(), + fallback_templates: Vec::new(), + games: games.iter().map(|game| (*game).into()).collect(), + profile_base_url: "https://meta.example.com/versions/loader/" + .parse() + .unwrap(), + } + } + + #[test] + fn source_urls_encode_game_and_loader_versions() { + assert_eq!( + profile_source( + &group(None, "1.14 Pre-Release 1", &[]), + "1.14 Pre-Release 1", + "0.4.1+build.128" + ) + .unwrap() + .as_str(), + "https://meta.example.com/versions/loader/1.14%20Pre-Release%201/0.4.1+build.128/profile/json" + ); + } + + #[test] + fn selects_current_template_before_historical_profiles() { + let mut group = group(Some("v2"), "26.3", &["26.3", "26.2"]); + group.fallback_templates = vec!["26.2".into()]; + let current = profile_source(&group, "26.3", "0.30.0").unwrap(); + let historical = profile_source(&group, "26.2", "0.30.0").unwrap(); + let current_hash = Sha256::from_digest(b"current"); + let historical_hash = Sha256::from_digest(b"historical"); + let mut hashes = + HashMap::from([(historical.to_string(), historical_hash)]); + let selected = select_profile( + FabriclikeLoader::Quilt, + &group, + "0.30.0", + &hashes, + true, + ) + .unwrap() + .unwrap(); + assert_eq!(selected, ("26.2", historical, historical_hash)); + hashes.insert(current.to_string(), current_hash); + let selected = select_profile( + FabriclikeLoader::Quilt, + &group, + "0.30.0", + &hashes, + true, + ) + .unwrap() + .unwrap(); + assert_eq!(selected, ("26.3", current, current_hash)); + } + + #[test] + fn missing_profiles_are_optional_only_for_historical_loaders() { + let legacy = group(Some("v1"), "1.21", &[]); + let modern = group(Some("v2"), "26.3", &[]); + let sha256 = Sha256::from_digest(b"historical profile"); + let hashes = HashMap::from([( + profile_source(&legacy, "1.21", "old").unwrap().to_string(), + sha256, + )]); + assert!( + select_profile( + FabriclikeLoader::Quilt, + &legacy, + "old", + &hashes, + false + ) + .unwrap() + .is_some() + ); + assert!( + select_profile( + FabriclikeLoader::Quilt, + &modern, + "old", + &hashes, + false + ) + .unwrap() + .is_none() + ); + assert!( + select_profile( + FabriclikeLoader::Quilt, + &modern, + "old", + &hashes, + true + ) + .is_err() + ); + let mut invalid = modern; + invalid.profile_base_url = + Url::parse("mailto:invalid@example.com").unwrap(); + assert!( + select_profile( + FabriclikeLoader::Quilt, + &invalid, + "old", + &hashes, + false + ) + .is_err() + ); + } + + #[test] + fn groups_cover_each_catalog_game_exactly_once() { + let games = ["1.21", "26.3"] + .into_iter() + .map(|id| { + ( + id.to_owned(), + CatalogGame { + id: id.into(), + stable: true, + }, + ) + }) + .collect(); + let legacy = group(Some("v1"), "1.21", &["1.21"]); + let modern = group(Some("v2"), "26.3", &["26.3"]); + assert!( + validate_groups(&games, &[legacy.clone(), modern.clone()]).is_ok() + ); + assert!(validate_groups(&games, slice::from_ref(&legacy)).is_err()); + assert!(validate_groups(&games, &[legacy.clone(), legacy]).is_err()); + assert!( + validate_groups( + &games, + &[group(None, "1.21", &["1.21", "26.3"]), modern] + ) + .is_err() + ); + assert!( + validate_groups( + &games, + &[group(None, "1.21", &["1.21", "26.3", "unknown"])] + ) + .is_err() + ); + } + + + fn normalize( + mut profile: Value, + games: &[String], + ) -> Result<(Value, Vec)> { + if let Some(object) = profile.as_object_mut() { + object.insert("releaseTime".into(), json!("2026-01-01T00:00:00Z")); + object.insert("time".into(), json!("2026-01-01T00:00:00Z")); + object.insert("type".into(), json!("release")); + } + let (profile, artifacts) = normalize_profile( + serde_json::from_value(profile)?, + "GAME", + games, + &Url::parse("https://maven.fabricmc.net/").unwrap(), + )?; + Ok((serde_json::to_value(profile)?, artifacts)) + } + + #[test] + fn profile_id_substitution_preserves_loader_version() { + let profile: ProfileMetadata = serde_json::from_value(json!({ + "id": "quilt-loader-0.26.3-26.3", + "inheritsFrom": "26.3", + "releaseTime": "2026-01-01T00:00:00Z", + "time": "2026-01-01T00:00:00Z", + "type": "release", + "libraries": [] + })) + .unwrap(); + let (profile, _) = normalize_profile( + profile, + "26.3", + &[], + &Url::parse("https://maven.quiltmc.org/repository/release/") + .unwrap(), + ) + .unwrap(); + assert_eq!( + profile.id, + format!("quilt-loader-0.26.3-{GAME_VERSION_PLACEHOLDER}") + ); + } + + #[test] + fn substitutes_templates_and_expands_concrete_artifacts() { + let (profile, artifacts) = normalize( + json!({ + "id": "fabric-loader-GAME", + "inheritsFrom": "GAME", + "libraries": [{"name": "net.fabricmc:intermediary:GAME"}] + }), + &["1.21.1".to_owned(), "1.14.2 Pre-Release 4".to_owned()], + ) + .unwrap(); + assert_eq!( + profile["id"], + format!("fabric-loader-{GAME_VERSION_PLACEHOLDER}") + ); + assert_eq!(profile["inheritsFrom"], GAME_VERSION_PLACEHOLDER); + assert_eq!( + profile["libraries"][0]["name"], + format!("net.fabricmc:intermediary:{GAME_VERSION_PLACEHOLDER}") + ); + + assert_eq!(artifacts.len(), 2); + assert_eq!( + artifacts[0].coordinate.to_string(), + "net.fabricmc:intermediary:1.21.1" + ); + assert_eq!( + artifacts[1].coordinate.to_string(), + "net.fabricmc:intermediary:1.14.2 Pre-Release 4" + ); + assert_eq!( + artifacts[0].repository.as_str(), + "https://maven.fabricmc.net/" + ); + } + + #[test] + fn only_mapping_coordinates_are_templated() { + let (profile, artifacts) = normalize( + json!({ + "id": "GAME", "inheritsFrom": "GAME", + "libraries": [ + {"name": "net.fabricmc:sponge-mixin:0.7.11.21"}, + {"name": "net.fabricmc:sponge-mixin:GAME"}, + {"name": "other:intermediary:GAME"}, + {"name": "net.fabricmc:intermediary:GAME:GAME@jar"}, + {"name": "org.quiltmc:hashed:GAME"} + ] + }), + &["1.21.1".into(), "26.4-snapshot-3".into()], + ) + .unwrap(); + assert_eq!(artifacts.len(), 7); + for (index, name) in [ + "net.fabricmc:sponge-mixin:0.7.11.21", + "net.fabricmc:sponge-mixin:GAME", + "other:intermediary:GAME", + ] + .iter() + .enumerate() + { + assert_eq!(profile["libraries"][index]["name"], *name); + assert_eq!(artifacts[index].coordinate.to_string(), *name); + } + assert_eq!( + profile["libraries"][3]["name"], + format!( + "net.fabricmc:intermediary:{GAME_VERSION_PLACEHOLDER}:GAME@jar" + ) + ); + assert_eq!( + artifacts[3].coordinate.to_string(), + "net.fabricmc:intermediary:1.21.1:GAME@jar" + ); + assert_eq!( + artifacts[4].coordinate.to_string(), + "net.fabricmc:intermediary:26.4-snapshot-3:GAME@jar" + ); + assert_eq!( + profile["libraries"][4]["name"], + format!("org.quiltmc:hashed:{GAME_VERSION_PLACEHOLDER}") + ); + } + + #[test] + fn rejects_mapping_versions_that_do_not_match_template() { + assert!( + normalize( + json!({ + "id": "GAME", "inheritsFrom": "GAME", + "libraries": [{"name": "net.fabricmc:intermediary:OTHER"}] + }), + &["1.21.1".into()] + ) + .is_err() + ); + } + + #[test] + fn launchwrapper_uses_minecraft_repository() { + let (profile, artifacts) = normalize( + json!({ + "id": "GAME", "inheritsFrom": "GAME", + "libraries": [{ + "name": "net.minecraft:launchwrapper:1.12", + "url": "https://maven.fabricmc.net/" + }] + }), + &[], + ) + .unwrap(); + assert_eq!( + artifacts[0].repository.as_str(), + "https://libraries.minecraft.net/" + ); + assert_eq!( + profile["libraries"][0]["url"], + "https://maven.fabricmc.net/" + ); + } + + #[test] + fn preserves_concrete_libraries_and_emits_daedalus_defaults() { + let (profile, artifacts) = normalize(json!({ + "id": "loader", "inheritsFrom": "1.21.1", + "libraries": [ + {"name": "org.quiltmc:quilt-loader:1", "url": "https://maven.quiltmc.org/repository/release/", "include_in_classpath": false, "downloadable": false, "rules": []}, + {"name": "net.fabricmc:fabric-loader:1"} + ] + }), &["1.21.1".to_owned(), "1.21.2".to_owned()]).unwrap(); + assert_eq!(artifacts.len(), 2); + assert_eq!( + artifacts[0].repository.as_str(), + "https://maven.quiltmc.org/repository/release/" + ); + assert_eq!( + artifacts[1].repository.as_str(), + "https://maven.fabricmc.net/" + ); + assert_eq!( + profile["libraries"][0]["name"], + "org.quiltmc:quilt-loader:1" + ); + assert_eq!(profile["libraries"][0]["include_in_classpath"], false); + assert_eq!(profile["libraries"][0]["downloadable"], false); + assert_eq!(profile["libraries"][0]["rules"], json!([])); + assert_eq!(profile["libraries"][1]["include_in_classpath"], true); + assert_eq!(profile["libraries"][1]["downloadable"], true); + assert!(profile.get("arguments").is_none()); + assert!(profile.get("data").is_none()); + } + + #[test] + fn rejects_invalid_profiles_and_names() { + for profile in [ + json!(null), + json!({"id": 1, "inheritsFrom": "GAME", "libraries": []}), + json!({"id": "GAME", "libraries": []}), + json!({"id": "GAME", "inheritsFrom": "GAME", "libraries": {}}), + json!({"id": "GAME", "inheritsFrom": "GAME", "libraries": [null]}), + json!({"id": "GAME", "inheritsFrom": "GAME", "libraries": [{"name": 1}]}), + json!({"id": "GAME", "inheritsFrom": "GAME", "libraries": [{"name": "bad/GAME:a:1"}]}), + json!({"id": "GAME", "inheritsFrom": "GAME", "libraries": [{"name": "g:a:${unknown}"}]}), + ] { + assert!(normalize(profile, &[]).is_err()); + } + } + + #[test] + fn typed_profiles_round_trip_templates_and_require_metadata() { + let value = json!({ + "id": "fabric-${modrinth.gameVersion}", + "inheritsFrom": "${modrinth.gameVersion}", + "releaseTime": "2026-01-01T00:00:00Z", + "time": "2026-01-01T00:00:00Z", + "type": "release", + "libraries": [{"name": "net.fabricmc:intermediary:${modrinth.gameVersion}"}] + }); + let profile: ProfileMetadata = + serde_json::from_value(value.clone()).unwrap(); + let output = serde_json::to_value(&profile).unwrap(); + assert_eq!( + output["libraries"][0]["name"], + value["libraries"][0]["name"] + ); + assert!(output["libraries"][0].get("downloads").is_none()); + let mut missing = value; + missing.as_object_mut().unwrap().remove("releaseTime"); + assert!(serde_json::from_value::(missing).is_err()); + } + + #[test] + fn artifact_urls_upgrade_http_repositories() { + let artifact = Artifact { + coordinate: "com.github.zafarkhaja:java-semver:0.9.0" + .parse() + .unwrap(), + repository: "http://repo.maven.apache.org/maven2/".parse().unwrap(), + source: Some("downloaded profile".into()), + repository_source: "profile library.url".into(), + }; + assert_eq!( + artifact_url(&artifact).unwrap().as_str(), + "https://repo.maven.apache.org/maven2/com/github/zafarkhaja/java-semver/0.9.0/java-semver-0.9.0.jar" + ); + } + + #[test] + fn artifact_urls_append_encoded_segments_without_store_prefix() { + let artifact = Artifact { + coordinate: "net.fabricmc:intermediary:1.14.2 Pre-Release 4" + .parse() + .unwrap(), + repository: "https://example.com/repository/release" + .parse() + .unwrap(), + source: None, + repository_source: "test".into(), + }; + assert_eq!( + artifact_url(&artifact).unwrap().as_str(), + "https://example.com/repository/release/net/fabricmc/intermediary/1.14.2%20Pre-Release%204/intermediary-1.14.2%20Pre-Release%204.jar" + ); + } + + #[test] + fn records_profile_repository_selection() { + let (_, artifacts) = normalize( + json!({ + "id": "GAME", "inheritsFrom": "GAME", + "libraries": [{ + "name": "com.github.zafarkhaja:java-semver:0.9.0", + "url": "http://repo.maven.apache.org/maven2/" + }] + }), + &[], + ) + .unwrap(); + assert_eq!(artifacts[0].repository_source, "profile library.url"); + assert_eq!( + artifacts[0].repository.as_str(), + "http://repo.maven.apache.org/maven2/" + ); + } +} diff --git a/apps/meta-generator/src/import/extract/mod.rs b/apps/meta-generator/src/import/extract/mod.rs new file mode 100644 index 00000000000..2220e6ba7d2 --- /dev/null +++ b/apps/meta-generator/src/import/extract/mod.rs @@ -0,0 +1,56 @@ +//! Logic for how to parse information downloaded from upstreams, and extract +//! it into some more usable, structured form (DB rows). + +pub mod fabric; +pub mod fabriclike; +pub mod mojang; +pub mod quilt; + +use anyhow::{Result, ensure}; +use tracing::{info_span, warn}; +use tracing_anyhow::FutureContext; + +use crate::{AppState, upstream::Upstreams}; + +pub async fn extract(app: &AppState, upstreams: Upstreams) -> Result<()> { + let mut failed = Vec::new(); + if upstreams.mojang { + mojang::extract(app) + .context(info_span!("extracting Mojang metadata")) + .await + .inspect_err(|err| { + warn!("error: {err:?}"); + failed.push("Mojang"); + }) + .ok(); + } + + if upstreams.fabric { + fabric::extract(app) + .context(info_span!("extracting Fabric metadata")) + .await + .inspect_err(|err| { + warn!("error: {err:?}"); + failed.push("Fabric"); + }) + .ok(); + } + + if upstreams.quilt { + quilt::extract(app) + .context(info_span!("extracting Quilt metadata")) + .await + .inspect_err(|err| { + warn!("error: {err:?}"); + failed.push("Quilt"); + }) + .ok(); + } + + ensure!( + failed.is_empty(), + "metadata extraction failed for {}", + failed.join(", ") + ); + Ok(()) +} diff --git a/apps/meta-generator/src/import/extract/mojang.rs b/apps/meta-generator/src/import/extract/mojang.rs new file mode 100644 index 00000000000..8380412a215 --- /dev/null +++ b/apps/meta-generator/src/import/extract/mojang.rs @@ -0,0 +1,334 @@ +use std::collections::{HashMap, HashSet}; + +use anyhow::{Context, Result, bail, ensure}; +use futures::{StreamExt, TryStreamExt, stream}; +use indexmap::IndexMap; +use tracing::{info, info_span}; +use tracing_anyhow::FutureContext; + +use crate::{ + AppState, model, + store::BlobCas, + upstream::mojang, + util::{Sha1, Sha256, from_json_slice}, +}; + +pub async fn extract(app: &AppState) -> Result<()> { + let mut conn = app + .db + .connection() + .context(info_span!("acquiring connection")) + .await?; + let mut rows = model::MojangCatalog::all() + .exec(&mut conn) + .context(info_span!("fetching Mojang catalogs")) + .await?; + info!("found {} downloaded catalogs", rows.len()); + + // fetch runs, so that newer catalogs take precedence over older ones + let runs = model::DownloadRun::all() + .exec(&mut conn) + .context(info_span!("fetching download runs")) + .await? + .into_iter() + .map(|run| (run.id, run.started_at)) + .collect::>(); + rows.sort_by_key(|row| { + ( + std::cmp::Reverse(runs.get(&row.download_run_id).copied()), + row.id.0, + ) + }); + + // read each catalog from its blob in the store; + // preserve query order while reading concurrently + let catalogs = stream::iter(rows) + .map(|row| async move { + let bytes = app + .cas + .get(row.sha256) + .context( + info_span!("fetching Mojang catalog", sha256 = %row.sha256), + ) + .await?; + let catalog = from_json_slice::(&bytes) + .context("parsing Mojang catalog")?; + anyhow::Ok((row.sha256, catalog)) + }) + .buffered(app.concurrency.download.get()) + .try_collect::>() + .await?; + let candidate_sha1s = catalogs + .iter() + .flat_map(|(_, catalog)| { + catalog.versions.iter().map(|version| version.sha1) + }) + .collect::>() + .into_iter() + .collect::>(); + let available_sha1s = model::BlobHash::all() + .select(model::BlobHash::fields().sha1()) + .filter(model::BlobHash::fields().sha1().in_list(candidate_sha1s)) + .exec(&mut conn) + .context(info_span!("fetching available version manifest hashes")) + .await? + .into_iter() + .collect::>(); + let (latest, versions) = select_versions(catalogs, &available_sha1s)?; + info!("found {} unique versions", versions.len()); + + // now we make all the version manifests; + // finish reading and patching before changing any selected rows + let num_total = versions.len(); + let mut extracted = Vec::with_capacity(num_total); + for (source_catalog_sha256, catalog_entry) in versions { + let (source_manifest_sha256, manifest) = make_version_manifest(&mut conn, &app.cas, &catalog_entry) + .context(info_span!("extracting version manifest", version = %catalog_entry.id, %source_catalog_sha256)) + .await?; + extracted.push(( + source_catalog_sha256, + source_manifest_sha256, + catalog_entry, + manifest, + )); + let num_done = extracted.len(); + if num_done.is_multiple_of(100) || num_done == num_total { + info!("extracted {num_done}/{num_total} version manifests"); + } + } + info!("inserting into database"); + + // write the selected metadata together, so a failed extraction doesn't + // leave a mix of old and new rows or conflicting latest-version flags + let mut txn = conn + .transaction() + .context(info_span!("starting extraction transaction")) + .await?; + + let previous_latest = model::MinecraftVersion::all() + .filter( + model::MinecraftVersion::fields() + .is_latest_release() + .is_some() + .or(model::MinecraftVersion::fields() + .is_latest_snapshot() + .is_some()), + ) + .exec(&mut txn) + .context(info_span!("fetching previous latest versions")) + .await?; + for mut row in previous_latest { + toasty::update!(row { + is_latest_release: None::, + is_latest_snapshot: None::, + }) + .exec(&mut txn) + .context(info_span!("unsetting previous latest versions")) + .await?; + } + + for ( + source_catalog_sha256, + source_manifest_sha256, + catalog_entry, + manifest, + ) in extracted + { + let name = catalog_entry.id.clone(); + model::MinecraftVersion::upsert_by_name(name.clone()) + .is_latest_release((name == latest.release).then_some(true)) + .is_latest_snapshot((name == latest.snapshot).then_some(true)) + .catalog_entry(toasty::Json(catalog_entry)) + .manifest(toasty::Json(manifest)) + .source_catalog_sha256(source_catalog_sha256) + .source_manifest_sha256(source_manifest_sha256) + .exec(&mut txn) + .context(info_span!("upserting Minecraft version", %name)) + .await?; + } + txn.commit() + .context(info_span!("committing extracted Minecraft metadata")) + .await?; + + info!( + num_versions = num_total, + release = %latest.release, + snapshot = %latest.snapshot, + "stored Minecraft metadata" + ); + Ok(()) +} + +/// Makes a list of versions from a list of catalogs (ordered newest-first), by +/// taking the newest available revision for each unique version name. +/// Every version in the newest catalog must have a downloaded source blob. +fn select_versions( + catalogs: Vec<(Sha256, mojang::Catalog)>, + available_sha1s: &HashSet, +) -> Result<(mojang::Latest, Vec<(Sha256, mojang::Version)>)> { + let (_, latest) = + catalogs.first().context("no downloaded Mojang catalogs")?; + let latest = latest.latest.clone(); + + let mut versions = IndexMap::new(); + for (priority, (sha256, catalog)) in catalogs.into_iter().enumerate() { + for version in catalog.versions { + if !available_sha1s.contains(&version.sha1) { + ensure!( + priority != 0, + "no stored version manifest for {} with SHA-1 {} in newest Mojang catalog", + version.id, + version.sha1 + ); + continue; + } + versions + .entry(version.id.clone()) + .or_insert((sha256, version)); + } + } + + ensure!( + versions.contains_key(&latest.release), + "catalog says its latest release is {}, which doesn't exist in version list", + latest.release + ); + ensure!( + versions.contains_key(&latest.snapshot), + "catalog says its latest snapshot is {}, which doesn't exist in version list", + latest.snapshot + ); + + Ok((latest, versions.into_values().collect())) +} + +/// Pulls a catalog entry's source manifest from [`BlobCas`] and patches its +/// libraries. +/// +/// Returns: +/// - patched manifest hash +/// - patched manifest +async fn make_version_manifest( + conn: &mut toasty::Connection, + cas: &BlobCas, + version: &mojang::Version, +) -> Result<(Sha256, mojang::VersionManifest)> { + let matching_hashes = model::BlobHash::all() + .select(model::BlobHash::fields().sha256()) + .filter(model::BlobHash::fields().sha1().eq(version.sha1)) + .exec(conn) + .context( + info_span!("resolving version manifest hash", sha1 = %version.sha1), + ) + .await?; + let sha256 = match matching_hashes.as_slice() { + [sha256] => *sha256, + [] => bail!( + "no stored version manifest for {} with SHA-1 {}", + version.id, + version.sha1 + ), + _ => bail!( + "ambiguous version manifest for {}: multiple blobs have SHA-1 {}", + version.id, + version.sha1 + ), + }; + let bytes = cas + .get(sha256) + .context(info_span!("fetching version blob", %sha256)) + .await?; + + let mut manifest = from_json_slice::(&bytes) + .context("parsing version manifest")?; + + // post-process the version's libraries; + // they need some patching to work with launchers + // see `mojang::library` + manifest.libraries = manifest + .libraries + .into_iter() + .flat_map(crate::import::mojang::patch_library) + .collect(); + + Ok((sha256, manifest)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn version(id: &str, bytes: &[u8]) -> mojang::Version { + serde_json::from_value(serde_json::json!({ + "id": id, + "type": "release", + "url": "https://example.com/version.json", + "time": "2026-01-01T00:00:00Z", + "releaseTime": "2026-01-01T00:00:00Z", + "sha1": Sha1::from_digest(bytes).to_string(), + "complianceLevel": 1 + })) + .unwrap() + } + + fn catalog(versions: Vec) -> (Sha256, mojang::Catalog) { + ( + Sha256::from_digest(b"catalog"), + mojang::Catalog { + latest: serde_json::from_value(serde_json::json!({ + "release": "current", "snapshot": "current" + })) + .unwrap(), + versions, + }, + ) + } + + #[test] + fn historical_selection_uses_newest_available_revision() { + let current = version("current", b"current"); + let missing = version("historical", b"missing revision"); + let available = version("historical", b"available revision"); + let older = version("historical", b"older revision"); + let never_downloaded = version("absent", b"never downloaded"); + let hashes = HashSet::from([current.sha1, available.sha1, older.sha1]); + let (latest, selected) = select_versions( + vec![ + catalog(vec![current]), + catalog(vec![missing, never_downloaded]), + catalog(vec![available.clone()]), + catalog(vec![older]), + ], + &hashes, + ) + .unwrap(); + assert_eq!(latest.release, selected[0].1.id); + assert_eq!(latest.snapshot, selected[0].1.id); + assert_eq!(selected.len(), 2); + assert_eq!(selected[1].1.sha1, available.sha1); + } + + #[test] + fn newest_catalog_cannot_fall_back_to_downloaded_historical_revision() { + let current = version("current", b"current"); + let missing = version("required", b"missing new revision"); + let old = version("required", b"downloaded old revision"); + let hashes = HashSet::from([current.sha1, old.sha1]); + assert!( + select_versions( + vec![catalog(vec![current, missing]), catalog(vec![old]),], + &hashes + ) + .is_err() + ); + } + + #[test] + fn latest_versions_must_exist_in_available_selection() { + assert!( + select_versions(vec![catalog(Vec::new())], &HashSet::new()) + .is_err() + ); + assert!(select_versions(Vec::new(), &HashSet::new()).is_err()); + } +} diff --git a/apps/meta-generator/src/import/extract/quilt.rs b/apps/meta-generator/src/import/extract/quilt.rs new file mode 100644 index 00000000000..f0d8b85d947 --- /dev/null +++ b/apps/meta-generator/src/import/extract/quilt.rs @@ -0,0 +1,153 @@ +use std::collections::HashMap; + +use anyhow::{Context, Result}; +use indexmap::IndexMap; +use tracing::{info, info_span}; +use tracing_anyhow::FutureContext; +use url::Url; + +use crate::{ + AppState, + import::extract::fabriclike::{ + self, CatalogGame, CatalogLoader, CatalogSnapshot, ProfileGroup, + }, + model::{self, FabriclikeLoader}, + upstream::{ + fabriclike::{GameVersion, GameVersionName}, + quilt, + }, + util::from_json_slice, +}; + +pub async fn extract(app: &AppState) -> Result<()> { + let mut conn = app + .db + .connection() + .context(info_span!("acquiring connection")) + .await?; + let mut rows = model::QuiltCatalog::all() + .exec(&mut conn) + .context(info_span!("fetching Quilt catalogs")) + .await?; + info!(num_catalogs = rows.len(), "found downloaded Quilt catalogs"); + + // fetch download runs where we downloaded these catalogs from; + // newer catalogs take precedence over older ones + let runs = model::DownloadRun::all() + .exec(&mut conn) + .context(info_span!("fetching download runs")) + .await? + .into_iter() + .map(|run| (run.id, run.started_at)) + .collect::>(); + rows.sort_by_key(|row| { + ( + std::cmp::Reverse(runs.get(&row.download_run_id).copied()), + row.id.0, + ) + }); + + // read each catalog from its blob in the store, + // and start assembling the artifacts we'll extract + let repository = Url::parse(quilt::MAVEN_URL)?; + let mut games = IndexMap::new(); + let mut catalogs = Vec::with_capacity(rows.len()); + for row in rows { + let bytes = app + .cas + .get(row.sha256) + .context(info_span!("fetching Quilt catalog", sha256 = %row.sha256)) + .await?; + let catalog = from_json_slice::(&bytes) + .context("parsing Quilt catalog")?; + + for game in &catalog.game { + games + .entry(game.version.clone()) + .or_insert_with(|| game.clone()); + } + + catalogs.push(CatalogSnapshot { + download_run_id: row.download_run_id, + sha256: row.sha256, + games: catalog + .game + .into_iter() + .map(|game| CatalogGame { + id: game.version.to_string(), + stable: game.stable, + }) + .collect(), + loaders: catalog + .loader + .into_iter() + .filter(|loader| { + loader.version.0 + != crate::import::quilt::SKIPPED_LOADER_VERSION + }) + .map(|loader| CatalogLoader { + id: loader.version.to_string(), + stable: loader.stable, + }) + .collect(), + mappings: Vec::new(), + }); + } + // older loaders may only have been downloaded with an older template; + // keep those templates as fallbacks, within the same version group + let mut historical_templates: IndexMap> = + IndexMap::new(); + for catalog in &catalogs { + let snapshot_games = catalog + .games + .iter() + .map(|game| GameVersion { + version: GameVersionName(game.id.clone()), + stable: game.stable, + }) + .collect::>(); + for group in crate::import::quilt::profile_groups(&snapshot_games) { + historical_templates + .entry(group.id) + .or_default() + .entry(group.template.to_string()) + .or_insert(()); + } + } + let games = games.into_values().collect::>(); + let profile_base_url = Url::parse(quilt::PROFILE_BASE_URL)?; + // use the downloader's grouping rules; + // legacy and modern games need separate profiles for the same loader + let groups = crate::import::quilt::profile_groups(&games) + .into_iter() + .map(|group| ProfileGroup { + fallback_templates: historical_templates + .shift_remove(&group.id) + .unwrap_or_default() + .into_keys() + .filter(|template| template != &group.template.0) + .collect(), + id: Some(group.id), + template: group.template.to_string(), + games: group + .games + .into_iter() + .map(|game| game.to_string()) + .collect(), + profile_base_url: profile_base_url.clone(), + }) + .collect(); + + fabriclike::extract_catalogs( + app, + FabriclikeLoader::Quilt, + &repository, + catalogs, + groups, + ) + .context(info_span!( + "extracting catalogs", + loader = ?FabriclikeLoader::Quilt + )) + .await +} diff --git a/apps/meta-generator/src/import/fabric.rs b/apps/meta-generator/src/import/fabric.rs new file mode 100644 index 00000000000..b7bccacabaf --- /dev/null +++ b/apps/meta-generator/src/import/fabric.rs @@ -0,0 +1,72 @@ +use std::sync::atomic::{self, AtomicUsize}; + +use anyhow::{Result, ensure}; +use futures::{StreamExt, stream}; +use tracing::{info, info_span}; +use tracing_anyhow::FutureContext; + +use crate::{ + import::download::DownloadRunContext, + model, + upstream::{ + fabric::{CATALOG_URL, Catalog, LoaderVersion}, + fabriclike::{GameLoaderProfile, LoaderVersionName}, + }, +}; + +pub const PROFILE_GAME_VERSION: &str = "1.21"; +pub const MAVEN_URL: &str = "https://maven.fabricmc.net/"; + +pub fn profile_url(loader_version: &LoaderVersionName) -> String { + format!( + "{CATALOG_URL}/loader/{PROFILE_GAME_VERSION}/{loader_version}/profile/json" + ) +} + +pub async fn download(cx: &mut DownloadRunContext<'_>) -> Result<()> { + let initial_errors = cx.errors.len(); + let (catalog, sha256) = cx + .download_json::(CATALOG_URL) + .context(info_span!("fetching catalog")) + .await?; + info!( + num_game_versions = catalog.game.len(), + num_loader_versions = catalog.loader.len(), + "downloaded Fabric catalog" + ); + + let num_total = catalog.loader.len(); + let num_done = AtomicUsize::new(0); + let task = async |loader_version: LoaderVersion| { + let version_name = &loader_version.version; + let url = profile_url(version_name); + + cx.download_json::(&url) + .await + .inspect_err(|err| cx.errors.push(err)) + .ok(); + + let num_done = num_done.fetch_add(1, atomic::Ordering::SeqCst) + 1; + if num_done.is_multiple_of(10) || num_done == num_total { + info!("downloaded {num_done}/{num_total} loader profiles"); + } + }; + stream::iter(catalog.loader) + .for_each_concurrent(cx.download_concurrency, task) + .await; + + ensure!( + cx.errors.len() == initial_errors, + "Fabric item downloads failed; catalog not registered" + ); + + toasty::create!(model::FabricCatalog { + download_run_id: cx.download_run_id, + sha256, + }) + .exec(*cx.conn().await) + .context(info_span!("inserting catalog")) + .await?; + + Ok(()) +} diff --git a/apps/meta-generator/src/import/forge.rs b/apps/meta-generator/src/import/forge.rs new file mode 100644 index 00000000000..24474392fd9 --- /dev/null +++ b/apps/meta-generator/src/import/forge.rs @@ -0,0 +1,190 @@ +use std::sync::atomic::{self, AtomicUsize}; + +use anyhow::{Context, Result, bail, ensure}; +use futures::{StreamExt, stream}; +use tracing::{info, info_span}; +use tracing_anyhow::FutureContext; + +use crate::{ + import::download::DownloadRunContext, + model::{self, ForgelikeInstallerName}, + upstream::forge::{CATALOG_URL, Catalog}, +}; + +pub async fn download(cx: &mut DownloadRunContext<'_>) -> Result<()> { + let initial_errors = cx.errors.len(); + let (catalog, sha256) = cx + .download_json::(CATALOG_URL) + .context(info_span!("fetching catalog")) + .await?; + let raw_cataloged_versions = + catalog.0.values().map(Vec::len).sum::(); + + let mut cataloged_versions = Vec::new(); + let mut skipped_pre_installer_versions = 0usize; + for name in catalog.0.into_values().flatten() { + if !has_supported_installer(&name) + .with_context(|| format!("checking installer support for {name}"))? + { + skipped_pre_installer_versions += 1; + continue; + } + if !BLACKLIST.contains(&name.as_str()) { + cataloged_versions.push(name); + } + } + + cataloged_versions.sort(); + info!( + raw_cataloged_versions, + skipped_pre_installer_versions, + num_cataloged_versions = cataloged_versions.len(), + "downloaded Forge catalog" + ); + + let existing_versions = model::ForgelikeInstaller::all() + .filter( + model::ForgelikeInstaller::fields() + .loader() + .eq(model::ForgelikeLoader::Forge), + ) + .select(model::ForgelikeInstaller::fields().name()) + .exec(*cx.conn().await) + .context(info_span!("fetching existing Forge installer records")) + .await?; + let missing_versions = cataloged_versions + .iter() + .filter(|name| !existing_versions.contains(name)) + .collect::>(); + info!( + "catalog contains {} versions, of which {} are missing; downloading installers", + cataloged_versions.len(), + missing_versions.len() + ); + + let num_total = missing_versions.len(); + let num_done = AtomicUsize::new(0); + let task = async |installer_name: &ForgelikeInstallerName| { + let installer_url = format!( + "https://maven.minecraftforge.net/net/minecraftforge/forge/{installer_name}/forge-{installer_name}-installer.jar" + ); + + async { + let sha256 = cx + .download_blob(installer_url) + .context(info_span!("downloading installer", %installer_name)) + .await?; + + toasty::create!(model::ForgelikeInstaller { + loader: model::ForgelikeLoader::Forge, + name: installer_name, + download_run_id: cx.download_run_id, + sha256, + }) + .exec(*cx.conn().await) + .context(info_span!("inserting installer record")) + .await?; + + anyhow::Ok(()) + } + .await + .inspect_err(|err| cx.errors.push(err)) + .ok(); + + let num_done = num_done.fetch_add(1, atomic::Ordering::SeqCst) + 1; + if num_done.is_multiple_of(10) || num_done == num_total { + info!("downloaded {num_done}/{num_total} version installers"); + } + }; + stream::iter(missing_versions) + .for_each_concurrent(cx.download_concurrency, task) + .await; + + ensure!( + cx.errors.len() == initial_errors, + "Forge item downloads failed; catalog not registered" + ); + + toasty::create!(model::ForgeCatalog { + download_run_id: cx.download_run_id, + sha256, + }) + .exec(*cx.conn().await) + .context(info_span!("inserting catalog")) + .await?; + + Ok(()) +} + +/// If this installer version has an installer JAR which we can process. +/// +/// Very old Forge versions don't have an installer JAR, so we don't support +/// them. +fn has_supported_installer(name: &ForgelikeInstallerName) -> Result { + let (minecraft, remaining) = name + .as_str() + .split_once('-') + .context("missing Forge version separator")?; + ensure!(!minecraft.is_empty(), "missing Minecraft version"); + let version = remaining + .split('-') + .next() + .context("missing Forge version")?; + let components = version + .split('.') + .map(|part| { + part.parse::() + .context("invalid numeric Forge version component") + }) + .collect::>>()?; + + let version_parts = match ( + <[u32; 4]>::try_from(components.as_slice()), + <[u32; 3]>::try_from(components.as_slice()), + ) { + (Ok([a, b, c, d]), _) => [a, b, c, d], + (Err(_), Ok([a, b, c])) => [a, b, c, 0], + (Err(_), Err(_)) => bail!("expected 3 or 4 Forge version components"), + }; + Ok(version_parts >= [7, 8, 0, 684]) +} + +/// Specific Forge installer versions which we can't download or process. +const BLACKLIST: &[&str] = &[ + // Not supported due to `data` field being `[]` even though the type is a map + "1.12.2-14.23.5.2851", + // Malformed Archives + "1.6.1-8.9.0.749", + "1.6.1-8.9.0.751", + "1.6.4-9.11.1.960", + "1.6.4-9.11.1.961", + "1.6.4-9.11.1.963", + "1.6.4-9.11.1.964", +]; + +#[cfg(test)] +mod tests { + use super::{ForgelikeInstallerName, has_supported_installer}; + + #[test] + fn installer_support_uses_numeric_release_boundary() { + for (name, supported) in [ + ("1.1-1.3.3.26", false), + ("1.5.2-7.8.0.683", false), + ("1.5.2-7.8.0.684", true), + ("1.5.2-7.8.1.1", true), + ("1.7.10-10.13.4.1614-1.7.10", true), + ("1.10.2-12.18.1.2016-failtests", true), + ("1.21.8-58.1.22", true), + ] { + assert_eq!( + has_supported_installer( + &name.parse::().unwrap() + ) + .unwrap(), + supported, + "{name}" + ); + } + } +} diff --git a/apps/meta-generator/src/import/forgelike/data.rs b/apps/meta-generator/src/import/forgelike/data.rs new file mode 100644 index 00000000000..bfccacaf280 --- /dev/null +++ b/apps/meta-generator/src/import/forgelike/data.rs @@ -0,0 +1,228 @@ +use std::collections::{HashMap, HashSet}; + +use anyhow::{Context, Result, ensure}; + +use crate::{ + model::{ForgelikeInstallerName, ForgelikeLoader, ProfileMetadata}, + upstream::mojang::Library, + util::MavenCoordinate, +}; + +/// Rewrites installer-relative data into libraries without storing their bytes. +/// All paths and coordinates are validated before the profile is changed. +pub(super) fn normalize( + profile: &mut ProfileMetadata, + loader: ForgelikeLoader, + name: &ForgelikeInstallerName, + entries: &HashSet<&str>, +) -> Result> { + let Some(mut data) = profile.data.clone() else { + return Ok(Vec::new()); + }; + let loader = match loader { + ForgelikeLoader::Forge => "forge", + ForgelikeLoader::Neoforge => "neoforge", + }; + let mut artifacts = Vec::new(); + let mut paths = HashMap::new(); + let mut coordinates = HashMap::new(); + let mut libraries = Vec::new(); + let mut keys = data.keys().cloned().collect::>(); + keys.sort(); + for key in keys { + let entry = data.get_mut(&key).expect("existing data key"); + for (side, value) in + [("client", &mut entry.client), ("server", &mut entry.server)] + { + let Some(path) = value.strip_prefix('/') else { + continue; + }; + ensure!( + entries.contains(path), + "missing installer data for {key} ({side}): {value}" + ); + if let Some(reference) = paths.get(path) { + *value = String::clone(reference); + continue; + } + let filename = + path.rsplit('/').next().context("missing data filename")?; + let mut parts = filename.split('.'); + let stem = parts.next().context("missing data filename stem")?; + let extension = + parts.next().context("missing data filename extension")?; + let coordinate: MavenCoordinate = format!( + "com.modrinth.daedalus:{loader}-installer-extracts:{}:{stem}@{extension}", + name.as_str() + ) + .parse() + .with_context(|| format!("invalid installer data coordinate for {key} ({side}): {value}"))?; + let maven_path = coordinate.to_maven_path(); + if let Some(existing) = + coordinates.insert(maven_path.clone(), path.to_owned()) + { + ensure!( + existing == path, + "installer data coordinate collision: {existing} and {path}" + ); + } + ensure!( + !profile.libraries.iter().any(|library| library + .name + .parse::() + .is_ok_and( + |existing| existing.to_maven_path() == maven_path + )), + "installer data coordinate conflicts with existing library: {coordinate}" + ); + let reference = format!("[{coordinate}]"); + paths.insert(path.to_owned(), reference.clone()); + libraries.push(Library { + name: coordinate.to_string(), + downloads: None, + extract: None, + url: None, + natives: None, + rules: None, + checksums: None, + include_in_classpath: false, + downloadable: true, + }); + artifacts.push((coordinate, path.to_owned())); + *value = reference; + } + } + profile.data = Some(data); + profile.libraries.extend(libraries); + Ok(artifacts) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn profile(client: &str, server: &str) -> ProfileMetadata { + serde_json::from_value(json!({ + "id": "test", "inheritsFrom": "1.20.3", "type": "release", + "time": "2024-01-01T00:00:00Z", "releaseTime": "2024-01-01T00:00:00Z", + "libraries": [], "data": {"BINPATCH": {"client": client, "server": server}} + })).unwrap() + } + + fn normalize_forge( + profile: &mut ProfileMetadata, + entries: &[&str], + ) -> Result> { + normalize( + profile, + ForgelikeLoader::Forge, + &"1.20.3-49.0.1".parse::().unwrap(), + &entries.iter().copied().collect(), + ) + } + + #[test] + fn sides_and_coordinate_names_match_daedalus() { + let mut profile = profile("/data/client.lzma", "/data/server.lzma"); + let artifacts = normalize_forge( + &mut profile, + &["data/client.lzma", "data/server.lzma"], + ) + .unwrap(); + assert_eq!(artifacts.len(), 2); + let data = &profile.data.as_ref().unwrap()["BINPATCH"]; + assert_eq!( + data.client, + "[com.modrinth.daedalus:forge-installer-extracts:1.20.3-49.0.1:client@lzma]" + ); + assert_eq!( + data.server, + "[com.modrinth.daedalus:forge-installer-extracts:1.20.3-49.0.1:server@lzma]" + ); + assert_eq!(artifacts[0].1, "data/client.lzma"); + assert!(profile.libraries.iter().all(|lib| lib.downloadable + && !lib.include_in_classpath + && lib.url.is_none() + && lib.downloads.is_none())); + } + + #[test] + fn repeated_files_are_deduplicated_across_sides_and_keys() { + let mut profile = profile("/data/client.lzma", "/data/client.lzma"); + let entry = profile.data.as_ref().unwrap()["BINPATCH"].clone(); + profile.data.as_mut().unwrap().insert("OTHER".into(), entry); + assert_eq!( + normalize_forge(&mut profile, &["data/client.lzma"]) + .unwrap() + .len(), + 1 + ); + assert_eq!(profile.libraries.len(), 1); + let data = profile.data.as_ref().unwrap(); + assert_eq!(data["BINPATCH"].client, data["OTHER"].server); + } + + #[test] + fn nonpaths_and_absent_data_are_preserved() { + for value in [ + "[net.example:artifact:1]", + "'literal'", + "{MINECRAFT_JAR}", + "123", + "", + "relative/file.jar", + ] { + let mut profile = profile(value, value); + let before = serde_json::to_value(&profile).unwrap(); + assert!(normalize_forge(&mut profile, &[]).unwrap().is_empty()); + assert_eq!(serde_json::to_value(profile).unwrap(), before); + } + let mut profile = profile("", ""); + profile.data = None; + assert!(normalize_forge(&mut profile, &[]).unwrap().is_empty()); + assert!(profile.data.is_none()); + } + + #[test] + fn neoforge_and_multidot_names_match_daedalus() { + let mut profile = profile("/data/client.extra.lzma", "'scalar'"); + let artifacts = normalize( + &mut profile, + ForgelikeLoader::Neoforge, + &"21.0.1-beta".parse::().unwrap(), + &HashSet::from(["data/client.extra.lzma"]), + ) + .unwrap(); + assert_eq!( + artifacts[0].0.to_string(), + "com.modrinth.daedalus:neoforge-installer-extracts:21.0.1-beta:client@extra" + ); + assert_eq!(profile.data.unwrap()["BINPATCH"].server, "'scalar'"); + } + + #[test] + fn invalid_or_missing_data_does_not_change_profile() { + for (client, server, entries) in [ + ( + "/data/client.lzma", + "/missing.lzma", + vec!["data/client.lzma"], + ), + ("/data/noextension", "", vec!["data/noextension"]), + ("/data/.lzma", "", vec!["data/.lzma"]), + ("/data/client.", "", vec!["data/client."]), + ("/", "", vec![]), + ( + "/a/client.lzma", + "/b/client.lzma", + vec!["a/client.lzma", "b/client.lzma"], + ), + ] { + let mut profile = profile(client, server); + let before = serde_json::to_value(&profile).unwrap(); + assert!(normalize_forge(&mut profile, &entries).is_err()); + assert_eq!(serde_json::to_value(profile).unwrap(), before); + } + } +} diff --git a/apps/meta-generator/src/import/forgelike/mod.rs b/apps/meta-generator/src/import/forgelike/mod.rs new file mode 100644 index 00000000000..5dae495b9e5 --- /dev/null +++ b/apps/meta-generator/src/import/forgelike/mod.rs @@ -0,0 +1,760 @@ +use std::{ + any::type_name, + collections::{HashMap, HashSet}, +}; + +use anyhow::{Context, Result, anyhow, ensure}; +use futures::{StreamExt, TryStreamExt, stream}; +use jiff::Timestamp; +use serde::{Deserialize, de::DeserializeOwned}; +use tracing::{debug, info, info_span, warn}; +use tracing_anyhow::FutureContext; + +use crate::{ + AppState, model, + upstream::{self, mojang}, + util::{ + MavenCoordinate, ResponseExt, Sha1, from_json_slice, from_json_value, + }, +}; + +mod data; + +pub async fn extract_installers( + app: &AppState, + upstreams: upstream::Upstreams, +) -> Result<()> { + let mut loaders = Vec::new(); + if upstreams.forge { + loaders.push(model::ForgelikeLoader::Forge); + } + if upstreams.neoforge { + loaders.push(model::ForgelikeLoader::Neoforge); + } + if loaders.is_empty() { + return Ok(()); + } + let mut conn = app + .db + .connection() + .context(info_span!("acquiring extraction connection")) + .await?; + let extracted_hashes = model::ForgelikeExtract::all() + .select(model::ForgelikeExtract::fields().installer_sha256()); + let mut unprocessed_installers = model::ForgelikeInstaller::all() + .filter( + model::ForgelikeInstaller::fields() + .sha256() + .in_query(extracted_hashes) + .not(), + ) + .filter( + model::ForgelikeInstaller::fields() + .loader() + .in_list(loaders), + ) + .exec(&mut conn) + .context(info_span!("fetching unprocessed Forge-like installers")) + .await?; + unprocessed_installers.retain(|installer| !installer.is_blacklisted()); + info!( + "found {} installers left to process", + unprocessed_installers.len() + ); + + drop(conn); + let mut num_done = 0usize; + let mut num_failed = 0usize; + let mut seen = HashSet::new(); + for installer in unprocessed_installers { + if !seen.insert(installer.sha256) { + continue; + } + let name = installer.name.clone(); + let sha256 = installer.sha256; + if let Err(err) = extract_from_installer(app, installer) + .context(info_span!("extracting from installer", %name, %sha256)) + .await + { + num_failed += 1; + warn!(%name, %sha256, "installer extraction failed: {err:?}"); + } + + num_done += 1; + if num_done.is_multiple_of(10) { + info!("processed {num_done} installers"); + } + } + + info!( + num_done, + num_failed, "finished Forge-like installer extraction" + ); + ensure!( + num_failed == 0, + "failed to extract {num_failed} Forge-like installers" + ); + Ok(()) +} + +#[derive(Debug, Deserialize)] +struct ModernInstallProfile { + data: HashMap, + libraries: Vec, + processors: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct LegacyInstallProfile { + install: LegacyInstall, + version_info: LegacyVersionInfo, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct LegacyInstall { + minecraft: String, + path: MavenCoordinate, + file_path: String, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct LegacyVersionInfo { + id: String, + #[serde(deserialize_with = "crate::util::deserialize_timestamp")] + release_time: Timestamp, + #[serde(deserialize_with = "crate::util::deserialize_timestamp")] + time: Timestamp, + main_class: Option, + minecraft_arguments: Option, + libraries: Vec, + #[serde(rename = "type")] + ty: mojang::VersionType, +} + +impl LegacyInstallProfile { + fn into_metadata(self) -> model::ProfileMetadata { + let version = self.version_info; + let arguments = version.minecraft_arguments.as_ref().map(|args| { + HashMap::from([( + mojang::ArgumentType::Game, + args.split(' ') + .map(|arg| mojang::Argument::Normal(arg.to_owned())) + .collect(), + )]) + }); + + model::ProfileMetadata { + id: version.id, + inherits_from: self.install.minecraft, + release_time: version.release_time, + time: version.time, + main_class: version.main_class, + minecraft_arguments: version.minecraft_arguments, + arguments, + libraries: version.libraries, + ty: version.ty, + data: None, + processors: None, + } + } +} + +type ZipFileReader = async_zip::base::read::mem::ZipFileReader; + +async fn extract_from_installer( + app: &AppState, + installer: model::ForgelikeInstaller, +) -> Result<()> { + let installer_blob = app + .cas + .get(installer.sha256) + .context(info_span!("fetching installer blob")) + .await?; + let zip = ZipFileReader::new(installer_blob) + .context(info_span!("opening installer zip for reading")) + .await?; + + let install_profile = + read_json::(&zip, "install_profile.json") + .context(info_span!("reading install profile")) + .await? + .context("installer is missing install_profile.json")?; + let mut legacy_artifact = None; + let mut version = + match read_json::(&zip, "version.json") + .context(info_span!("reading installer version")) + .await? + { + Some(mut version) => { + let profile = + from_json_value::(&install_profile) + .context("parsing modern install profile")?; + version.data = Some(profile.data); + version.processors = Some(profile.processors); + version.libraries.extend(profile.libraries.into_iter().map( + |mut library| { + library.include_in_classpath = false; + library + }, + )); + version + } + None => { + let profile = + from_json_value::(&install_profile) + .context( + "parsing legacy install profile without version.json", + )?; + legacy_artifact = Some(( + profile.install.path.clone(), + profile.install.file_path.clone(), + )); + profile.into_metadata() + } + }; + let modern = legacy_artifact.is_none(); + let embedded_entries = zip + .file() + .entries() + .iter() + .map(|entry| entry.filename().as_str()) + .collect::, _>>()?; + let data_artifacts = data::normalize( + &mut version, + installer.loader, + &installer.name, + &embedded_entries, + )?; + let referenced_artifacts = referenced_maven_artifacts(&version.libraries)?; + let mut embedded_maven_artifacts = { + referenced_artifacts + .into_iter() + .filter(|coordinate| { + embedded_entries.contains(coordinate.to_maven_path().as_str()) + }) + .map(|coordinate| { + let path = coordinate.to_maven_path(); + (coordinate, path) + }) + .collect::>() + }; + if let Some((coordinate, path)) = legacy_artifact { + embedded_maven_artifacts.retain(|(existing, _)| { + existing.to_maven_path() != coordinate.to_maven_path() + }); + embedded_maven_artifacts.push((coordinate, path)); + } + + embedded_maven_artifacts.extend(data_artifacts); + + debug!( + %version.id, + num_embedded_maven_artifacts = embedded_maven_artifacts.len(), + "read installer metadata" + ); + + let embedded_paths = embedded_maven_artifacts + .iter() + .map(|(coordinate, _)| coordinate.to_maven_path()) + .collect::>(); + let mut external_artifacts = Vec::new(); + for library in &mut version.libraries { + let coordinate: MavenCoordinate = library.name.parse()?; + let source = + classify_library(library, &coordinate, &embedded_paths, modern)?; + normalize_library(library, &source); + if let LibrarySource::External { urls, hashes } = source { + external_artifacts.push((coordinate, urls, hashes)); + } + } + if modern { + embedded_maven_artifacts.retain(|(coordinate, _)| { + !external_artifacts.iter().any(|(external, _, _)| { + external.to_maven_path() == coordinate.to_maven_path() + }) + }); + } + + let mut stored_artifacts = stream::iter(embedded_maven_artifacts) + .map(|(coordinate, path)| { + let zip = &zip; + async move { + let bytes = read_file(zip, &path) + .context(info_span!( + "extracting embedded Maven artifact", + path + )) + .await? + .context( + "embedded Maven artifact is missing from installer", + )?; + let mut conn = app.db.connection() + .context(info_span!("acquiring artifact connection")) + .await?; + let sha256 = app.cas + .put(&mut conn, &bytes) + .context(info_span!("storing embedded Maven artifact in CAS", path, %coordinate)) + .await?; + anyhow::Ok(model::ForgelikeStoredArtifact { coordinate, sha256 }) + } + }) + .buffered(app.concurrency.extract_files.get()) + .try_collect::>() + .await?; + + let fallback = match installer.loader { + model::ForgelikeLoader::Forge => "https://maven.minecraftforge.net/", + model::ForgelikeLoader::Neoforge => { + "https://maven.neoforged.net/releases/" + } + }; + let downloaded = stream::iter(external_artifacts) + .map(|(coordinate, mut urls, hashes)| async move { + if urls.len() > 1 { + urls.push(repository_artifact_url(fallback, &coordinate)?); + } + download_library(app, coordinate, urls, hashes).await + }) + .buffered(app.concurrency.download.get()) + .try_collect::>() + .await?; + stored_artifacts.extend(downloaded); + + let mut conn = app + .db + .connection() + .context(info_span!("acquiring extract connection")) + .await?; + model::ForgelikeExtract::upsert_by_installer_sha256(installer.sha256) + .metadata(toasty::Json(version)) + .embedded_maven_artifacts(toasty::Json(stored_artifacts)) + .or_ignore() + .exec(&mut conn) + .context(info_span!("storing Forge-like installer extract")) + .await?; + Ok(()) +} + +#[derive(Debug)] +enum LibrarySource { + External { + urls: Vec, + hashes: Vec, + }, + Embedded, + Generated, + Upstream, +} + +fn classify_library( + library: &mojang::Library, + coordinate: &MavenCoordinate, + embedded_paths: &HashSet, + modern: bool, +) -> Result { + let embedded = embedded_paths.contains(&coordinate.to_maven_path()); + if !modern && embedded { + return Ok(LibrarySource::Embedded); + } + if modern + && let Some(artifact) = library + .downloads + .as_ref() + .and_then(|downloads| downloads.artifact.as_ref()) + { + if let Some(url) = &artifact.url { + return Ok(LibrarySource::External { + urls: vec![url.clone()], + hashes: vec![artifact.sha1], + }); + } + return Ok(if embedded { + LibrarySource::Embedded + } else { + LibrarySource::Generated + }); + } + if let Some(repository) = + library.url.as_deref().filter(|url| !url.is_empty()) + { + let url = repository_artifact_url(repository, coordinate)?; + if !modern && url.host_str() == Some("libraries.minecraft.net") { + return Ok(LibrarySource::Upstream); + } + let hashes = library + .checksums + .as_ref() + .map(|checksums| { + checksums + .iter() + .map(|checksum| { + let mut digest = [0; 20]; + hex::decode_to_slice(checksum, &mut digest)?; + Ok::<_, hex::FromHexError>(Sha1(digest)) + }) + .collect::, _>>() + }) + .transpose() + .context("parsing library SHA1 checksums")? + .unwrap_or_default(); + let mut urls = vec![url]; + if modern { + urls.push(repository_artifact_url( + "https://libraries.minecraft.net/", + coordinate, + )?); + } + urls.push(repository_artifact_url( + "https://maven.creeperhost.net/", + coordinate, + )?); + return Ok(LibrarySource::External { urls, hashes }); + } + Ok(if embedded { + LibrarySource::Embedded + } else if modern { + LibrarySource::Generated + } else { + LibrarySource::Upstream + }) +} + +fn normalize_library(library: &mut mojang::Library, source: &LibrarySource) { + match source { + LibrarySource::External { .. } | LibrarySource::Embedded => { + library.downloadable = true + } + LibrarySource::Generated => library.downloadable = false, + LibrarySource::Upstream => {} + } +} + +fn repository_artifact_url( + repository: &str, + coordinate: &MavenCoordinate, +) -> Result { + let mut url = url::Url::parse(repository) + .context("parsing library repository URL")?; + if url.scheme() == "http" { + url.set_scheme("https").map_err(|()| { + anyhow!("cannot upgrade library repository to HTTPS") + })?; + } + let path = coordinate.to_maven_path(); + url.path_segments_mut() + .map_err(|()| anyhow!("library repository cannot be a base URL"))? + .pop_if_empty() + .extend( + path.strip_prefix("maven/") + .context("missing Maven path prefix")? + .split('/'), + ); + Ok(url) +} + +fn verify_library_hashes(bytes: &[u8], hashes: &[Sha1]) -> Result<()> { + let actual = Sha1::from_digest(bytes); + ensure!( + hashes.is_empty() || hashes.contains(&actual), + "library SHA1 mismatch: got {actual}, expected one of {hashes:?}" + ); + Ok(()) +} + +async fn download_library( + app: &AppState, + coordinate: MavenCoordinate, + urls: Vec, + hashes: Vec, +) -> Result { + if !hashes.is_empty() { + let mut conn = app.db.connection().await?; + let existing = model::BlobHash::all() + .filter(model::BlobHash::fields().sha1().in_list(hashes.clone())) + .exec(&mut conn) + .await?; + drop(conn); + for blob in existing { + if let Ok(bytes) = app.cas.get(blob.sha256).await { + verify_library_hashes(&bytes, &hashes)?; + return Ok(model::ForgelikeStoredArtifact { + coordinate, + sha256: blob.sha256, + }); + } + } + } + let mut failures = Vec::new(); + for url in urls { + let result = async { + let bytes = app + .http + .get(url.clone()) + .send() + .await? + .error_for_status_ext() + .await? + .bytes() + .await?; + verify_library_hashes(&bytes, &hashes)?; + anyhow::Ok(bytes) + } + .await; + match result { + Ok(bytes) => { + let mut conn = app.db.connection().await?; + let sha256 = app.cas.put(&mut conn, &bytes).await?; + return Ok(model::ForgelikeStoredArtifact { + coordinate, + sha256, + }); + } + Err(error) => failures.push(format!("{url}: {error:#}")), + } + } + Err(anyhow!( + "failed to download library {coordinate}: {}", + failures.join("; ") + )) +} + +fn referenced_maven_artifacts( + libraries: &[mojang::Library], +) -> Result> { + let mut seen = HashSet::new(); + let mut coordinates = Vec::new(); + for library in libraries { + let coordinate: MavenCoordinate = + library.name.parse().with_context(|| { + format!("parsing referenced library {}", library.name) + })?; + let path = coordinate.to_maven_path(); + if seen.insert(path) { + coordinates.push(coordinate); + } + } + Ok(coordinates) +} + +async fn read_file(zip: &ZipFileReader, path: &str) -> Result>> { + let index = zip + .file() + .entries() + .iter() + .position(|entry| entry.filename().as_str().ok() == Some(path)); + let Some(index) = index else { + return Ok(None); + }; + + let mut reader = zip + .reader_with_entry(index) + .context(info_span!("opening installer entry", path)) + .await?; + let mut bytes = Vec::new(); + reader + .read_to_end_checked(&mut bytes) + .context(info_span!("reading installer entry", path)) + .await?; + Ok(Some(bytes)) +} + +async fn read_json( + zip: &ZipFileReader, + path: &str, +) -> Result> { + let Some(bytes) = read_file(zip, path).await? else { + return Ok(None); + }; + let t = from_json_slice::(&bytes).with_context(|| { + anyhow!("entry {path} must be a `{}`", type_name::()) + })?; + Ok(Some(t)) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn library(value: serde_json::Value) -> mojang::Library { + serde_json::from_value(value).unwrap() + } + + fn classify( + library: &mojang::Library, + embedded: bool, + modern: bool, + ) -> LibrarySource { + let coordinate = library.name.parse::().unwrap(); + let paths = if embedded { + HashSet::from([coordinate.to_maven_path()]) + } else { + HashSet::new() + }; + classify_library(library, &coordinate, &paths, modern).unwrap() + } + + #[test] + fn direct_artifact_takes_precedence_over_repository_and_embedding() { + let lib = library(json!({ + "name": "g:a:1", "url": "https://repo.example/", + "downloads": { "artifact": { + "url": "https://files.example/a.jar", "size": 1, + "sha1": "0000000000000000000000000000000000000000" + }} + })); + let LibrarySource::External { urls, hashes } = + classify(&lib, true, true) + else { + panic!("expected external artifact") + }; + assert_eq!(urls.len(), 1); + assert_eq!(urls[0].as_str(), "https://files.example/a.jar"); + assert_eq!(hashes, vec![Sha1([0; 20])]); + } + + #[test] + fn empty_artifact_url_does_not_fall_back_to_repository() { + for url in [json!(""), json!(null)] { + let lib = library(json!({ + "name": "g:a:1", "url": "https://repo.example/", + "downloads": { "artifact": { + "url": url, "size": 0, + "sha1": "0000000000000000000000000000000000000000" + }} + })); + assert!(matches!( + classify(&lib, true, true), + LibrarySource::Embedded + )); + assert!(matches!( + classify(&lib, false, true), + LibrarySource::Generated + )); + } + } + + #[test] + fn repository_fallbacks_and_legacy_mojang_behavior() { + let lib = library( + json!({ "name": "g:a:1", "url": "https://libraries.minecraft.net/" }), + ); + assert!(matches!( + classify(&lib, false, false), + LibrarySource::Upstream + )); + let LibrarySource::External { urls, .. } = classify(&lib, true, true) + else { + panic!("expected external artifact") + }; + assert_eq!(urls.len(), 3); + assert_eq!( + urls[0].as_str(), + "https://libraries.minecraft.net/g/a/1/a-1.jar" + ); + assert!(matches!( + classify(&lib, true, false), + LibrarySource::Embedded + )); + } + + #[test] + fn normalization_preserves_processor_classpath_and_marks_generated_artifacts() + { + let mut lib = + library(json!({ "name": "g:a:1", "include_in_classpath": false })); + for url in [None, Some("")] { + lib.url = url.map(str::to_owned); + let source = classify(&lib, false, true); + normalize_library(&mut lib, &source); + assert!(!lib.downloadable); + assert!(!lib.include_in_classpath); + let source = classify(&lib, true, true); + normalize_library(&mut lib, &source); + assert!(lib.downloadable); + assert!(!lib.include_in_classpath); + } + normalize_library( + &mut lib, + &LibrarySource::External { + urls: Vec::new(), + hashes: Vec::new(), + }, + ); + assert!(lib.downloadable); + assert!(!lib.include_in_classpath); + } + + #[test] + fn repository_url_preserves_prefix_and_normalizes_maven_path() { + let coordinate = "net.example:library:1:client@zip".parse().unwrap(); + for repository in [ + "http://repo.example/releases", + "https://repo.example/releases/", + ] { + assert_eq!( + repository_artifact_url(repository, &coordinate) + .unwrap() + .as_str(), + "https://repo.example/releases/net/example/library/1/library-1-client.zip" + ); + } + assert!(repository_artifact_url("invalid", &coordinate).is_err()); + } + + #[test] + fn supplied_checksums_are_validated() { + let expected = Sha1::from_digest(b"expected"); + assert!(verify_library_hashes(b"expected", &[expected]).is_ok()); + assert!(verify_library_hashes(b"wrong", &[expected]).is_err()); + assert!(verify_library_hashes(b"anything", &[]).is_ok()); + let mut lib = library( + json!({ "name": "g:a:1", "url": "https://repo.example/", "checksums": [expected.to_string()] }), + ); + let LibrarySource::External { hashes, .. } = + classify(&lib, false, true) + else { + panic!("expected external artifact") + }; + assert_eq!(hashes, vec![expected]); + lib.checksums = Some(vec!["invalid".to_owned()]); + assert!( + classify_library( + &lib, + &lib.name.parse().unwrap(), + &HashSet::new(), + true + ) + .is_err() + ); + } + + #[test] + fn legacy_install_has_explicit_source_path_and_coordinate() { + let install: LegacyInstall = serde_json::from_value(json!({ + "minecraft": "1.7.10", + "path": "net.minecraftforge:forge:1.7.10-10.13.4.1614", + "filePath": "forge-universal.jar" + })) + .unwrap(); + assert_eq!(install.file_path, "forge-universal.jar"); + assert_eq!( + install.path.to_maven_path(), + "maven/net/minecraftforge/forge/1.7.10-10.13.4.1614/forge-1.7.10-10.13.4.1614.jar" + ); + } + + #[test] + fn legacy_install_requires_valid_coordinate_and_source_path() { + for value in [ + json!({"minecraft": "1.7.10", "path": "invalid", "filePath": "forge.jar"}), + json!({"minecraft": "1.7.10", "path": "g:a:1"}), + json!({"minecraft": "1.7.10", "filePath": "forge.jar"}), + ] { + assert!(serde_json::from_value::(value).is_err()); + } + } +} diff --git a/apps/meta-generator/src/import/mod.rs b/apps/meta-generator/src/import/mod.rs new file mode 100644 index 00000000000..50fc0862675 --- /dev/null +++ b/apps/meta-generator/src/import/mod.rs @@ -0,0 +1,54 @@ +use anyhow::{Result, ensure}; +use tracing::{info_span, warn}; +use tracing_anyhow::FutureContext; + +use crate::{AppState, upstream::Upstreams}; + +mod download; +mod extract; +pub mod fabric; +pub mod forge; +pub mod forgelike; +pub mod mojang; +pub mod neoforge; +pub mod quilt; + +pub async fn run(app: AppState, upstreams: Upstreams) -> Result<()> { + let completed = download::download_from_upstreams(&app, upstreams) + .context(info_span!("downloading import sources")) + .await?; + let mut failed = [ + (upstreams.mojang, completed.mojang, "Mojang"), + (upstreams.fabric, completed.fabric, "Fabric"), + (upstreams.forge, completed.forge, "Forge"), + (upstreams.neoforge, completed.neoforge, "NeoForge"), + (upstreams.quilt, completed.quilt, "Quilt"), + ] + .into_iter() + .filter(|(selected, completed, _)| *selected && !completed) + .map(|(_, _, name)| format!("downloading {name}")) + .collect::>(); + + // don't replace selected metadata for upstreams whose downloads failed + extract::extract(&app, completed) + .context(info_span!("extracting imported metadata")) + .await + .inspect_err(|err| { + warn!("error: {err:?}"); + failed.push("extracting metadata".into()); + }) + .ok(); + if completed.forge || completed.neoforge { + forgelike::extract_installers(&app, completed) + .context(info_span!("extracting imported installers")) + .await + .inspect_err(|err| { + warn!("error: {err:?}"); + failed.push("extracting installers".into()); + }) + .ok(); + } + + ensure!(failed.is_empty(), "import failed: {}", failed.join(", ")); + Ok(()) +} diff --git a/apps/meta-generator/src/import/mojang/library/library-patches.json b/apps/meta-generator/src/import/mojang/library/library-patches.json new file mode 100644 index 00000000000..df750e02b30 --- /dev/null +++ b/apps/meta-generator/src/import/mojang/library/library-patches.json @@ -0,0 +1,2887 @@ +[ + { + "_comment": "Only allow osx-arm64 for existing LWJGL 3.3.2/3.3.3", + "match": [ + "org.lwjgl:lwjgl-freetype-natives-macos-arm64:3.3.2", + "org.lwjgl:lwjgl-glfw-natives-macos-arm64:3.3.2", + "org.lwjgl:lwjgl-jemalloc-natives-macos-arm64:3.3.2", + "org.lwjgl:lwjgl-openal-natives-macos-arm64:3.3.2", + "org.lwjgl:lwjgl-opengl-natives-macos-arm64:3.3.2", + "org.lwjgl:lwjgl-stb-natives-macos-arm64:3.3.2", + "org.lwjgl:lwjgl-tinyfd-natives-macos-arm64:3.3.2", + "org.lwjgl:lwjgl-natives-macos-arm64:3.3.2", + "org.lwjgl:lwjgl-freetype-natives-macos-arm64:3.3.3", + "org.lwjgl:lwjgl-glfw-natives-macos-arm64:3.3.3", + "org.lwjgl:lwjgl-jemalloc-natives-macos-arm64:3.3.3", + "org.lwjgl:lwjgl-openal-natives-macos-arm64:3.3.3", + "org.lwjgl:lwjgl-opengl-natives-macos-arm64:3.3.3", + "org.lwjgl:lwjgl-stb-natives-macos-arm64:3.3.3", + "org.lwjgl:lwjgl-tinyfd-natives-macos-arm64:3.3.3", + "org.lwjgl:lwjgl-natives-macos-arm64:3.3.3" + ], + "override": { + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + } + }, + { + "_comment": "Only allow windows-arm64 for existing LWJGL 3.3.2/3.3.3", + "match": [ + "org.lwjgl:lwjgl-freetype-natives-windows-arm64:3.3.2", + "org.lwjgl:lwjgl-glfw-natives-windows-arm64:3.3.2", + "org.lwjgl:lwjgl-jemalloc-natives-windows-arm64:3.3.2", + "org.lwjgl:lwjgl-openal-natives-windows-arm64:3.3.2", + "org.lwjgl:lwjgl-opengl-natives-windows-arm64:3.3.2", + "org.lwjgl:lwjgl-stb-natives-windows-arm64:3.3.2", + "org.lwjgl:lwjgl-tinyfd-natives-windows-arm64:3.3.2", + "org.lwjgl:lwjgl-natives-windows-arm64:3.3.2", + "org.lwjgl:lwjgl-freetype-natives-windows-arm64:3.3.3", + "org.lwjgl:lwjgl-glfw-natives-windows-arm64:3.3.3", + "org.lwjgl:lwjgl-jemalloc-natives-windows-arm64:3.3.3", + "org.lwjgl:lwjgl-openal-natives-windows-arm64:3.3.3", + "org.lwjgl:lwjgl-opengl-natives-windows-arm64:3.3.3", + "org.lwjgl:lwjgl-stb-natives-windows-arm64:3.3.3", + "org.lwjgl:lwjgl-tinyfd-natives-windows-arm64:3.3.3", + "org.lwjgl:lwjgl-natives-windows-arm64:3.3.3" + ], + "override": { + "rules": [ + { + "action": "allow", + "os": { + "name": "windows-arm64" + } + } + ] + } + }, + { + "_comment": "Add missing tinyfd to the broken LWJGL 3.2.2 variant", + "match": ["org.lwjgl:lwjgl:3.2.2"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "fcbe606c8f8da6f8f9a05e2c540eb1ee8632b0e9", + "size": 7092, + "url": "https://libraries.minecraft.net/org/lwjgl/lwjgl-tinyfd/3.2.2/lwjgl-tinyfd-3.2.2.jar" + } + }, + "name": "org.lwjgl:lwjgl-tinyfd:3.2.2" + }, + { + "downloads": { + "artifact": { + "sha1": "fcbe606c8f8da6f8f9a05e2c540eb1ee8632b0e9", + "size": 7092, + "url": "https://libraries.minecraft.net/org/lwjgl/lwjgl-tinyfd/3.2.2/lwjgl-tinyfd-3.2.2.jar" + }, + "classifiers": { + "natives-linux": { + "sha1": "39e35b161c130635d9c8918ce04e887a30c5b687", + "size": 38804, + "url": "https://libraries.minecraft.net/org/lwjgl/lwjgl-tinyfd/3.2.2/lwjgl-tinyfd-3.2.2-natives-linux.jar" + }, + "natives-macos": { + "sha1": "46d0798228b8a28e857a2a0f02310fd6ba2a4eab", + "size": 42136, + "url": "https://libraries.minecraft.net/org/lwjgl/lwjgl-tinyfd/3.2.2/lwjgl-tinyfd-3.2.2-natives-macos.jar" + }, + "natives-windows": { + "sha1": "e9115958773644e863332a6a06488d26f9e1fc9f", + "size": 208314, + "url": "https://libraries.minecraft.net/org/lwjgl/lwjgl-tinyfd/3.2.2/lwjgl-tinyfd-3.2.2-natives-windows.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-tinyfd:3.2.2", + "natives": { + "linux": "natives-linux", + "osx": "natives-macos", + "windows": "natives-windows" + } + } + ], + "patchAdditionalLibraries": true + }, + { + "_comment": "Add additional library just for osx-arm64. No override needed", + "match": ["ca.weblite:java-objc-bridge:1.0.0"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "369a83621e3c65496348491e533cb97fe5f2f37d", + "size": 91947, + "url": "https://github.com/MinecraftMachina/Java-Objective-C-Bridge/releases/download/1.1.0-mmachina.1/java-objc-bridge-1.1.jar" + } + }, + "name": "ca.weblite:java-objc-bridge:1.1.0-mmachina.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add additional classifiers for jinput-platform", + "match": ["net.java.jinput:jinput-platform:2.0.5"], + "override": { + "downloads": { + "classifiers": { + "natives-osx-arm64": { + "sha1": "5189eb40db3087fb11ca063b68fa4f4c20b199dd", + "size": 10031, + "url": "https://github.com/r58Playz/jinput-m1/raw/main/plugins/OSX/bin/jinput-platform-2.0.5.jar" + }, + "natives-linux-arm64": { + "sha1": "42b388ccb7c63cec4e9f24f4dddef33325f8b212", + "size": 10932, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-2.9.4/jinput-platform-2.0.5-natives-linux.jar" + }, + "natives-linux-arm32": { + "sha1": "f3c455b71c5146acb5f8a9513247fc06db182fd5", + "size": 4521, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-2.9.4/jinput-platform-2.0.5-natives-linux.jar" + } + } + }, + "natives": { + "linux-arm64": "natives-linux-arm64", + "linux-arm32": "natives-linux-arm32", + "osx-arm64": "natives-osx-arm64" + } + } + }, + { + "_comment": "Use a newer version on osx-arm64", + "match": [ + "com.mojang:text2speech:1.0.10", + "com.mojang:text2speech:1.5", + "com.mojang:text2speech:1.6", + "com.mojang:text2speech:1.7", + "com.mojang:text2speech:1.10.1", + "com.mojang:text2speech:1.10.3", + "com.mojang:text2speech:1.11.2" + ], + "override": { + "rules": [ + { + "action": "allow" + }, + { + "action": "disallow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "f378f889797edd7df8d32272c06ca80a1b6b0f58", + "size": 13164, + "url": "https://libraries.minecraft.net/com/mojang/text2speech/1.11.3/text2speech-1.11.3.jar" + } + }, + "name": "com.mojang:text2speech:1.11.3", + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Use a newer version on osx-arm64, linux-arm64, and linux-arm32", + "match": [ + "org.lwjgl.lwjgl:lwjgl:2.9.3", + "org.lwjgl.lwjgl:lwjgl:2.9.1-nightly-20131120", + "org.lwjgl.lwjgl:lwjgl:2.9.1-nightly-20131017", + "org.lwjgl.lwjgl:lwjgl:2.9.1-nightly-20130708-debug3", + "org.lwjgl.lwjgl:lwjgl:2.9.1" + ], + "override": { + "rules": [ + { + "action": "allow" + }, + { + "action": "disallow", + "os": { + "name": "osx-arm64" + } + }, + { + "action": "disallow", + "os": { + "name": "linux-arm64" + } + }, + { + "action": "disallow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "697517568c68e78ae0b4544145af031c81082dfe", + "size": 1047168, + "url": "https://libraries.minecraft.net/org/lwjgl/lwjgl/lwjgl/2.9.4-nightly-20150209/lwjgl-2.9.4-nightly-20150209.jar" + } + }, + "name": "org.lwjgl.lwjgl:lwjgl:2.9.4-nightly-20150209", + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + }, + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + }, + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Use a newer version on osx-arm64, linux-arm64, and linux-arm32", + "match": [ + "org.lwjgl.lwjgl:lwjgl_util:2.9.3", + "org.lwjgl.lwjgl:lwjgl_util:2.9.1-nightly-20131120", + "org.lwjgl.lwjgl:lwjgl_util:2.9.1-nightly-20131017", + "org.lwjgl.lwjgl:lwjgl_util:2.9.1-nightly-20130708-debug3", + "org.lwjgl.lwjgl:lwjgl_util:2.9.1" + ], + "override": { + "rules": [ + { + "action": "allow" + }, + { + "action": "disallow", + "os": { + "name": "osx-arm64" + } + }, + { + "action": "disallow", + "os": { + "name": "linux-arm64" + } + }, + { + "action": "disallow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "d51a7c040a721d13efdfbd34f8b257b2df882ad0", + "size": 173887, + "url": "https://libraries.minecraft.net/org/lwjgl/lwjgl/lwjgl_util/2.9.4-nightly-20150209/lwjgl_util-2.9.4-nightly-20150209.jar" + } + }, + "name": "org.lwjgl.lwjgl:lwjgl_util:2.9.4-nightly-20150209", + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + }, + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + }, + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Use a newer patched version on osx-arm64, linux-arm64, and linux-arm32", + "match": [ + "org.lwjgl.lwjgl:lwjgl-platform:2.9.4-nightly-20150209", + "org.lwjgl.lwjgl:lwjgl-platform:2.9.3", + "org.lwjgl.lwjgl:lwjgl-platform:2.9.1-nightly-20131120", + "org.lwjgl.lwjgl:lwjgl-platform:2.9.1-nightly-20131017", + "org.lwjgl.lwjgl:lwjgl-platform:2.9.1-nightly-20130708-debug3", + "org.lwjgl.lwjgl:lwjgl-platform:2.9.1" + ], + "override": { + "downloads": { + "classifiers": { + "natives-osx-arm64": { + "sha1": "eff546c0b319d6ffc7a835652124c18089c67f36", + "size": 488316, + "url": "https://github.com/MinecraftMachina/lwjgl/releases/download/2.9.4-20150209-mmachina.2/lwjgl-platform-2.9.4-nightly-20150209-natives-osx.jar" + }, + "natives-linux-arm64": { + "sha1": "63ac7da0f4a4785c7eadc0f8edc1e9dcc4dd08cb", + "size": 579979, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-2.9.4/lwjgl-platform-2.9.4-nightly-20150209-natives-linux.jar" + }, + "natives-linux-arm32": { + "sha1": "fa483e540a9a753a5ffbb23dcf7879a5bf752611", + "size": 475177, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-2.9.4/lwjgl-platform-2.9.4-nightly-20150209-natives-linux.jar" + } + } + }, + "natives": { + "linux-arm64": "natives-linux-arm64", + "linux-arm32": "natives-linux-arm32", + "osx-arm64": "natives-osx-arm64" + } + } + }, + { + "_comment": "Use a newer patched version on osx-arm64, linux-arm64, and linux-arm32", + "match": [ + "org.lwjgl:lwjgl-glfw:3.2.2", + "org.lwjgl:lwjgl-glfw:3.2.1", + "org.lwjgl:lwjgl-glfw:3.1.6", + "org.lwjgl:lwjgl-glfw:3.1.2" + ], + "override": { + "rules": [ + { + "action": "allow" + }, + { + "action": "disallow", + "os": { + "name": "linux-arm64" + } + }, + { + "action": "disallow", + "os": { + "name": "linux-arm32" + } + }, + { + "action": "disallow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "155d175037efc76630940c197ca6dea2b17d7e18", + "size": 108691, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-glfw.jar" + } + }, + "name": "org.lwjgl:lwjgl-glfw:3.2.2-gman64.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "155d175037efc76630940c197ca6dea2b17d7e18", + "size": 108691, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-glfw.jar" + }, + "classifiers": { + "natives-linux-arm64": { + "sha1": "074ad243761147df0d060fbefc814614d2ff75cc", + "size": 85072, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-glfw-natives-linux-arm64.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-glfw:3.2.2-gman64.1", + "natives": { + "linux-arm64": "natives-linux-arm64" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "99e9a39fa8ed4167e3ff9e04d47eb32c9e69804d", + "size": 108691, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-glfw.jar" + } + }, + "name": "org.lwjgl:lwjgl-glfw:3.2.2-gman32.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "99e9a39fa8ed4167e3ff9e04d47eb32c9e69804d", + "size": 108691, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-glfw.jar" + }, + "classifiers": { + "natives-linux-arm32": { + "sha1": "4265f2fbe3b9d642591165165a17cf406cf7b98e", + "size": 80186, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-glfw-natives-linux-arm32.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-glfw:3.2.2-gman32.1", + "natives": { + "linux-arm32": "natives-linux-arm32" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "e9a101bca4fa30d26b21b526ff28e7c2d8927f1b", + "size": 130128, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-glfw.jar" + } + }, + "name": "org.lwjgl:lwjgl-glfw:3.3.1-mmachina.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "e9a101bca4fa30d26b21b526ff28e7c2d8927f1b", + "size": 130128, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-glfw.jar" + }, + "classifiers": { + "natives-osx-arm64": { + "sha1": "71d793d0a5a42e3dfe78eb882abc2523a2c6b496", + "size": 129076, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-glfw-natives-macos-arm64.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-glfw:3.3.1-mmachina.1", + "natives": { + "osx-arm64": "natives-osx-arm64" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Use a newer patched version on osx-arm64, linux-arm64, and linux-arm32", + "match": [ + "org.lwjgl:lwjgl-jemalloc:3.2.2", + "org.lwjgl:lwjgl-jemalloc:3.2.1", + "org.lwjgl:lwjgl-jemalloc:3.1.6", + "org.lwjgl:lwjgl-jemalloc:3.1.2" + ], + "override": { + "rules": [ + { + "action": "allow" + }, + { + "action": "disallow", + "os": { + "name": "linux-arm64" + } + }, + { + "action": "disallow", + "os": { + "name": "linux-arm32" + } + }, + { + "action": "disallow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "cc04eec29b2fa8c298791af9800a3766d9617954", + "size": 33790, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-jemalloc.jar" + } + }, + "name": "org.lwjgl:lwjgl-jemalloc:3.2.2-gman64.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "cc04eec29b2fa8c298791af9800a3766d9617954", + "size": 33790, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-jemalloc.jar" + }, + "classifiers": { + "natives-linux-arm64": { + "sha1": "762d7d80c9cdf3a3f3fc80c8a5f86612255edfe0", + "size": 156343, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-jemalloc-patched-natives-linux-arm64.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-jemalloc:3.2.2-gman64.2", + "natives": { + "linux-arm64": "natives-linux-arm64" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "8224ae2e8fc6d8e1a0fc7d84dc917aa3c440620c", + "size": 33790, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-jemalloc.jar" + } + }, + "name": "org.lwjgl:lwjgl-jemalloc:3.2.2-gman32.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "8224ae2e8fc6d8e1a0fc7d84dc917aa3c440620c", + "size": 33790, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-jemalloc.jar" + }, + "classifiers": { + "natives-linux-arm32": { + "sha1": "9163a2a5559ef87bc13ead8fea84417ea3928748", + "size": 134237, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-jemalloc-natives-linux-arm32.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-jemalloc:3.2.2-gman32.1", + "natives": { + "linux-arm32": "natives-linux-arm32" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "4fb94224378d3588d52d2beb172f2eeafea2d546", + "size": 36976, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-jemalloc.jar" + } + }, + "name": "org.lwjgl:lwjgl-jemalloc:3.3.1-mmachina.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "4fb94224378d3588d52d2beb172f2eeafea2d546", + "size": 36976, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-jemalloc.jar" + }, + "classifiers": { + "natives-osx-arm64": { + "sha1": "b0be721188d2e7195798780b1c5fe7eafe8091c1", + "size": 103478, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-jemalloc-natives-macos-arm64.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-jemalloc:3.3.1-mmachina.1", + "natives": { + "osx-arm64": "natives-osx-arm64" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Use a newer patched version on osx-arm64, linux-arm64, and linux-arm32", + "match": [ + "org.lwjgl:lwjgl-openal:3.2.2", + "org.lwjgl:lwjgl-openal:3.2.1", + "org.lwjgl:lwjgl-openal:3.1.6", + "org.lwjgl:lwjgl-openal:3.1.2" + ], + "override": { + "rules": [ + { + "action": "allow" + }, + { + "action": "disallow", + "os": { + "name": "linux-arm64" + } + }, + { + "action": "disallow", + "os": { + "name": "linux-arm32" + } + }, + { + "action": "disallow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "6dfce9dc6a9629c75b2ae01a8df7e7be80ba0261", + "size": 79582, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-openal.jar" + } + }, + "name": "org.lwjgl:lwjgl-openal:3.2.2-gman64.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "6dfce9dc6a9629c75b2ae01a8df7e7be80ba0261", + "size": 79582, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-openal.jar" + }, + "classifiers": { + "natives-linux-arm64": { + "sha1": "948e415b5b2a2c650c25b377a4a9f443b21ce92e", + "size": 469432, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-openal-natives-linux-arm64.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-openal:3.2.2-gman64.1", + "natives": { + "linux-arm64": "natives-linux-arm64" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "304f0571fd5971621ee6da86a4c1e90f6f52e2ee", + "size": 79582, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-openal.jar" + } + }, + "name": "org.lwjgl:lwjgl-openal:3.2.2-gman32.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "304f0571fd5971621ee6da86a4c1e90f6f52e2ee", + "size": 79582, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-openal.jar" + }, + "classifiers": { + "natives-linux-arm32": { + "sha1": "ecbc981fdd996492a1f6334f003ed62e5a8c0cd5", + "size": 398418, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-openal-natives-linux-arm32.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-openal:3.2.2-gman32.1", + "natives": { + "linux-arm32": "natives-linux-arm32" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "d48e753d85916fc8a200ccddc709b36e3865cc4e", + "size": 88880, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-openal.jar" + } + }, + "name": "org.lwjgl:lwjgl-openal:3.3.1-mmachina.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "d48e753d85916fc8a200ccddc709b36e3865cc4e", + "size": 88880, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-openal.jar" + }, + "classifiers": { + "natives-osx-arm64": { + "sha1": "6b80fc0b982a0723b141e88859c42d6f71bd723f", + "size": 346131, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-openal-natives-macos-arm64.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-openal:3.3.1-mmachina.1", + "natives": { + "osx-arm64": "natives-osx-arm64" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Use a newer patched version on osx-arm64, linux-arm64, and linux-arm32", + "match": [ + "org.lwjgl:lwjgl-opengl:3.2.2", + "org.lwjgl:lwjgl-opengl:3.2.1", + "org.lwjgl:lwjgl-opengl:3.1.6", + "org.lwjgl:lwjgl-opengl:3.1.2" + ], + "override": { + "rules": [ + { + "action": "allow" + }, + { + "action": "disallow", + "os": { + "name": "linux-arm64" + } + }, + { + "action": "disallow", + "os": { + "name": "linux-arm32" + } + }, + { + "action": "disallow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "198bc2f72e0b2eb401eb6f5999aea52909b31ac4", + "size": 937609, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-opengl.jar" + } + }, + "name": "org.lwjgl:lwjgl-opengl:3.2.2-gman64.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "198bc2f72e0b2eb401eb6f5999aea52909b31ac4", + "size": 937609, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-opengl.jar" + }, + "classifiers": { + "natives-linux-arm64": { + "sha1": "bd40897077bf7d12f562da898b18ac2c68e1f9d7", + "size": 56109, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-opengl-natives-linux-arm64.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-opengl:3.2.2-gman64.1", + "natives": { + "linux-arm64": "natives-linux-arm64" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "9762ae928d02147e716cd82e929b74a97ea9600a", + "size": 937609, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-opengl.jar" + } + }, + "name": "org.lwjgl:lwjgl-opengl:3.2.2-gman32.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "9762ae928d02147e716cd82e929b74a97ea9600a", + "size": 937609, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-opengl.jar" + }, + "classifiers": { + "natives-linux-arm32": { + "sha1": "3af5599c74dd76dd8dbb567b3f9b4963a6abeed5", + "size": 56388, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-opengl-natives-linux-arm32.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-opengl:3.2.2-gman32.1", + "natives": { + "linux-arm32": "natives-linux-arm32" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "962c2a8d2a8cdd3b89de3d78d766ab5e2133c2f4", + "size": 929233, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-opengl.jar" + } + }, + "name": "org.lwjgl:lwjgl-opengl:3.3.1-mmachina.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "962c2a8d2a8cdd3b89de3d78d766ab5e2133c2f4", + "size": 929233, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-opengl.jar" + }, + "classifiers": { + "natives-osx-arm64": { + "sha1": "bb575058e0372f515587b5d2d04ff7db185f3ffe", + "size": 41667, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-opengl-natives-macos-arm64.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-opengl:3.3.1-mmachina.1", + "natives": { + "osx-arm64": "natives-osx-arm64" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Use a newer patched version on osx-arm64, linux-arm64, and linux-arm32", + "match": [ + "org.lwjgl:lwjgl-stb:3.2.2", + "org.lwjgl:lwjgl-stb:3.2.1", + "org.lwjgl:lwjgl-stb:3.1.6", + "org.lwjgl:lwjgl-stb:3.1.2" + ], + "override": { + "rules": [ + { + "action": "allow" + }, + { + "action": "disallow", + "os": { + "name": "linux-arm64" + } + }, + { + "action": "disallow", + "os": { + "name": "linux-arm32" + } + }, + { + "action": "disallow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "46a5735f3eb9d17eb5dcbdd5afa194066d2a6555", + "size": 104075, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-stb.jar" + } + }, + "name": "org.lwjgl:lwjgl-stb:3.2.2-gman64.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "46a5735f3eb9d17eb5dcbdd5afa194066d2a6555", + "size": 104075, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-stb.jar" + }, + "classifiers": { + "natives-linux-arm64": { + "sha1": "077efa7d7ea41b32df5c6078e912e724cccd06db", + "size": 202038, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-stb-natives-linux-arm64.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-stb:3.2.2-gman64.1", + "natives": { + "linux-arm64": "natives-linux-arm64" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "ea979b0af45b8e689f5f47c989aa8550c148d8a2", + "size": 104075, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-stb.jar" + } + }, + "name": "org.lwjgl:lwjgl-stb:3.2.2-gman32.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "ea979b0af45b8e689f5f47c989aa8550c148d8a2", + "size": 104075, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-stb.jar" + }, + "classifiers": { + "natives-linux-arm32": { + "sha1": "ec9d70aaebd0ff76dfeecf8f00b56118bf3706b1", + "size": 149387, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-stb-natives-linux-arm32.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-stb:3.2.2-gman32.1", + "natives": { + "linux-arm32": "natives-linux-arm32" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "703e4b533e2542560e9f94d6d8bd148be1c1d572", + "size": 113273, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-stb.jar" + } + }, + "name": "org.lwjgl:lwjgl-stb:3.3.1-mmachina.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "703e4b533e2542560e9f94d6d8bd148be1c1d572", + "size": 113273, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-stb.jar" + }, + "classifiers": { + "natives-osx-arm64": { + "sha1": "98f0ad956c754723ef354d50057cc30417ef376a", + "size": 178409, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-stb-natives-macos-arm64.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-stb:3.3.1-mmachina.1", + "natives": { + "osx-arm64": "natives-osx-arm64" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Use a newer patched version on osx-arm64, linux-arm64, and linux-arm32", + "match": [ + "org.lwjgl:lwjgl-tinyfd:3.2.2", + "org.lwjgl:lwjgl-tinyfd:3.2.1", + "org.lwjgl:lwjgl-tinyfd:3.1.6", + "org.lwjgl:lwjgl-tinyfd:3.1.2" + ], + "override": { + "rules": [ + { + "action": "allow" + }, + { + "action": "disallow", + "os": { + "name": "linux-arm64" + } + }, + { + "action": "disallow", + "os": { + "name": "linux-arm32" + } + }, + { + "action": "disallow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "3a75b9811607633bf33c978f53964df1534a4bc1", + "size": 5571, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-tinyfd.jar" + } + }, + "name": "org.lwjgl:lwjgl-tinyfd:3.2.2-gman64.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "3a75b9811607633bf33c978f53964df1534a4bc1", + "size": 5571, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-tinyfd.jar" + }, + "classifiers": { + "natives-linux-arm64": { + "sha1": "37c744ca289b5d7ae155d79e39029488b3254e5b", + "size": 37893, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-tinyfd-natives-linux-arm64.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-tinyfd:3.2.2-gman64.1", + "natives": { + "linux-arm64": "natives-linux-arm64" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "a8c09f5b7fa24bd53ec329c231b566497a163d5b", + "size": 5571, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-tinyfd.jar" + } + }, + "name": "org.lwjgl:lwjgl-tinyfd:3.2.2-gman32.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "a8c09f5b7fa24bd53ec329c231b566497a163d5b", + "size": 5571, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-tinyfd.jar" + }, + "classifiers": { + "natives-linux-arm32": { + "sha1": "82d16054ada6633297a3108fb6d8bae98800c76f", + "size": 41663, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-tinyfd-natives-linux-arm32.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-tinyfd:3.2.2-gman32.1", + "natives": { + "linux-arm32": "natives-linux-arm32" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "1203660b3131cbb8681b17ce6437412545be95e0", + "size": 6802, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-tinyfd.jar" + } + }, + "name": "org.lwjgl:lwjgl-tinyfd:3.3.1-mmachina.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "1203660b3131cbb8681b17ce6437412545be95e0", + "size": 6802, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-tinyfd.jar" + }, + "classifiers": { + "natives-osx-arm64": { + "sha1": "015b931a2daba8f0c317d84c9d14e8e98ae56e0c", + "size": 41384, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-tinyfd-natives-macos-arm64.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-tinyfd:3.3.1-mmachina.1", + "natives": { + "osx-arm64": "natives-osx-arm64" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Use a newer patched version on osx-arm64, linux-arm64, and linux-arm32", + "match": [ + "org.lwjgl:lwjgl:3.2.2", + "org.lwjgl:lwjgl:3.2.1", + "org.lwjgl:lwjgl:3.1.6", + "org.lwjgl:lwjgl:3.1.2" + ], + "override": { + "rules": [ + { + "action": "allow" + }, + { + "action": "disallow", + "os": { + "name": "linux-arm64" + } + }, + { + "action": "disallow", + "os": { + "name": "linux-arm32" + } + }, + { + "action": "disallow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "360899386df83d6a8407844a94478607af937f97", + "size": 318833, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-core.jar" + } + }, + "name": "org.lwjgl:lwjgl:3.2.2-gman64.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "360899386df83d6a8407844a94478607af937f97", + "size": 318833, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-core.jar" + }, + "classifiers": { + "natives-linux-arm64": { + "sha1": "612efd57d12b2e48e554858eb35e7e2eb46ebb4c", + "size": 87121, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-natives-linux-arm64.jar" + } + } + }, + "name": "org.lwjgl:lwjgl:3.2.2-gman64.1", + "natives": { + "linux-arm64": "natives-linux-arm64" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "16ea3934fca417368250d1ddac01a30c1809d317", + "size": 318413, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-core.jar" + } + }, + "name": "org.lwjgl:lwjgl:3.2.2-gman32.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "16ea3934fca417368250d1ddac01a30c1809d317", + "size": 318413, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-core.jar" + }, + "classifiers": { + "natives-linux-arm32": { + "sha1": "6bd0b37fef777a309936a72dc7f63126e8c79ea5", + "size": 90296, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-natives-linux-arm32.jar" + } + } + }, + "name": "org.lwjgl:lwjgl:3.2.2-gman32.1", + "natives": { + "linux-arm32": "natives-linux-arm32" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "8e664dd69ad7bbcf2053da23efc7848e39e498db", + "size": 719038, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl.jar" + } + }, + "name": "org.lwjgl:lwjgl:3.3.1-mmachina.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "8e664dd69ad7bbcf2053da23efc7848e39e498db", + "size": 719038, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl.jar" + }, + "classifiers": { + "natives-osx-arm64": { + "sha1": "984df31fadaab86838877b112e5b4e4f68a00ccf", + "size": 42693, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-natives-macos-arm64.jar" + } + } + }, + "name": "org.lwjgl:lwjgl:3.3.1-mmachina.1", + "natives": { + "osx-arm64": "natives-osx-arm64" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Only allow osx-arm64 for existing LWJGL 3.3.1", + "match": [ + "org.lwjgl:lwjgl-glfw-natives-macos-arm64:3.3.1", + "org.lwjgl:lwjgl-jemalloc-natives-macos-arm64:3.3.1", + "org.lwjgl:lwjgl-openal-natives-macos-arm64:3.3.1", + "org.lwjgl:lwjgl-opengl-natives-macos-arm64:3.3.1", + "org.lwjgl:lwjgl-stb-natives-macos-arm64:3.3.1", + "org.lwjgl:lwjgl-tinyfd-natives-macos-arm64:3.3.1", + "org.lwjgl:lwjgl-natives-macos-arm64:3.3.1" + ], + "override": { + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + } + }, + { + "_comment": "Only allow osx-arm64 for existing java-objc-bridge:1.1", + "match": ["ca.weblite:java-objc-bridge:1.1"], + "override": { + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + } + }, + { + "_comment": "Only allow windows-arm64 for existing LWJGL 3.3.1", + "match": [ + "org.lwjgl:lwjgl-glfw-natives-windows-arm64:3.3.1", + "org.lwjgl:lwjgl-jemalloc-natives-windows-arm64:3.3.1", + "org.lwjgl:lwjgl-openal-natives-windows-arm64:3.3.1", + "org.lwjgl:lwjgl-opengl-natives-windows-arm64:3.3.1", + "org.lwjgl:lwjgl-stb-natives-windows-arm64:3.3.1", + "org.lwjgl:lwjgl-tinyfd-natives-windows-arm64:3.3.1", + "org.lwjgl:lwjgl-natives-windows-arm64:3.3.1" + ], + "override": { + "rules": [ + { + "action": "allow", + "os": { + "name": "windows-arm64" + } + } + ] + } + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.1", + "match": ["org.lwjgl:lwjgl-glfw:3.3.1"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "513eb39b866d0fe131a18d5c517087805433b029", + "size": 112350, + "url": "https://build.lwjgl.org/release/3.3.1/bin/lwjgl-glfw/lwjgl-glfw-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-glfw-natives-linux-arm64:3.3.1-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.1", + "match": ["org.lwjgl:lwjgl-jemalloc:3.3.1"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "749be48a9b86ee2c3a2da5fd77511208adcfb33b", + "size": 159993, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.3.1/lwjgl-jemalloc-patched-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-jemalloc-natives-linux-arm64:3.3.1-gman64.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.1", + "match": ["org.lwjgl:lwjgl-openal:3.3.1"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "cf4e303257e82981b8b2e31bba3d7f8f7b8f42b2", + "size": 470743, + "url": "https://build.lwjgl.org/release/3.3.1/bin/lwjgl-openal/lwjgl-openal-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-openal-natives-linux-arm64:3.3.1-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.1", + "match": ["org.lwjgl:lwjgl-opengl:3.3.1"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "1c528fb258a6e63e8fceb4482d8db0f3af10a634", + "size": 57908, + "url": "https://build.lwjgl.org/release/3.3.1/bin/lwjgl-opengl/lwjgl-opengl-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-opengl-natives-linux-arm64:3.3.1-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.1", + "match": ["org.lwjgl:lwjgl-stb:3.3.1"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "8e8348a1813aad7f30aaf75ea197151ebb7beba9", + "size": 205491, + "url": "https://build.lwjgl.org/release/3.3.1/bin/lwjgl-stb/lwjgl-stb-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-stb-natives-linux-arm64:3.3.1-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.1", + "match": ["org.lwjgl:lwjgl-tinyfd:3.3.1"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "964f628b7a82fd909def086c0dd9a4b84bb259ae", + "size": 42654, + "url": "https://build.lwjgl.org/release/3.3.1/bin/lwjgl-tinyfd/lwjgl-tinyfd-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-tinyfd-natives-linux-arm64:3.3.1-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.1", + "match": ["org.lwjgl:lwjgl:3.3.1"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "b597401014acb7196c76d97e15a6288f54f1f692", + "size": 86308, + "url": "https://build.lwjgl.org/release/3.3.1/bin/lwjgl/lwjgl-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-natives-linux-arm64:3.3.1-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.1", + "match": ["org.lwjgl:lwjgl-glfw:3.3.1"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "816d935933f2dd743074c4e717cc25b55720f294", + "size": 104027, + "url": "https://build.lwjgl.org/release/3.3.1/bin/lwjgl-glfw/lwjgl-glfw-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-glfw-natives-linux-arm32:3.3.1-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.1", + "match": ["org.lwjgl:lwjgl-jemalloc:3.3.1"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "a96a6d6cb3876d7813fcee53c3c24f246aeba3b3", + "size": 136157, + "url": "https://build.lwjgl.org/release/3.3.1/bin/lwjgl-jemalloc/lwjgl-jemalloc-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-jemalloc-natives-linux-arm32:3.3.1-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.1", + "match": ["org.lwjgl:lwjgl-openal:3.3.1"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "ffbe35d7fa5ec9b7eca136a7c71f24d4025a510b", + "size": 400129, + "url": "https://build.lwjgl.org/release/3.3.1/bin/lwjgl-openal/lwjgl-openal-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-openal-natives-linux-arm32:3.3.1-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.1", + "match": ["org.lwjgl:lwjgl-opengl:3.3.1"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "e3550fa91097fd56e361b4370fa822220fef3595", + "size": 58474, + "url": "https://build.lwjgl.org/release/3.3.1/bin/lwjgl-opengl/lwjgl-opengl-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-opengl-natives-linux-arm32:3.3.1-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.1", + "match": ["org.lwjgl:lwjgl-stb:3.3.1"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "b08226bab162c06ae69337d8a1b0ee0a3fdf0b90", + "size": 153889, + "url": "https://build.lwjgl.org/release/3.3.1/bin/lwjgl-stb/lwjgl-stb-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-stb-natives-linux-arm32:3.3.1-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.1", + "match": ["org.lwjgl:lwjgl-tinyfd:3.3.1"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "d53d331e859217a61298fcbcf8d79137f3df345c", + "size": 48061, + "url": "https://build.lwjgl.org/release/3.3.1/bin/lwjgl-tinyfd/lwjgl-tinyfd-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-tinyfd-natives-linux-arm32:3.3.1-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.1", + "match": ["org.lwjgl:lwjgl:3.3.1"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "41a3c1dd15d6b964eb8196dde69720a3e3e5e969", + "size": 82374, + "url": "https://build.lwjgl.org/release/3.3.1/bin/lwjgl/lwjgl-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-natives-linux-arm32:3.3.1-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.2", + "match": ["org.lwjgl:lwjgl-freetype:3.3.2"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "896e7d9b8f60d7273f3d491c69270afc67ece3ce", + "size": 1073374, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-freetype/lwjgl-freetype-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-freetype-natives-linux-arm64:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.2", + "match": ["org.lwjgl:lwjgl-glfw:3.3.2"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "bc49e64bae0f7ff103a312ee8074a34c4eb034c7", + "size": 120168, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-glfw/lwjgl-glfw-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-glfw-natives-linux-arm64:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.2", + "match": ["org.lwjgl:lwjgl-jemalloc:3.3.2"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "5249f18a9ae20ea86c5816bc3107a888ce7a17d2", + "size": 206402, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-jemalloc/lwjgl-jemalloc-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-jemalloc-natives-linux-arm64:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.2", + "match": ["org.lwjgl:lwjgl-openal:3.3.2"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "22408980cc579709feaf9acb807992d3ebcf693f", + "size": 590865, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-openal/lwjgl-openal-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-openal-natives-linux-arm64:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.2", + "match": ["org.lwjgl:lwjgl-opengl:3.3.2"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "bb9eb56da6d1d549d6a767218e675e36bc568eb9", + "size": 58627, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-opengl/lwjgl-opengl-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-opengl-natives-linux-arm64:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.2", + "match": ["org.lwjgl:lwjgl-stb:3.3.2"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "11a380c37b0f03cb46db235e064528f84d736ff7", + "size": 207419, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-stb/lwjgl-stb-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-stb-natives-linux-arm64:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.2", + "match": ["org.lwjgl:lwjgl-tinyfd:3.3.2"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "93f8c5bc1984963cd79109891fb5a9d1e580373e", + "size": 43381, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-tinyfd/lwjgl-tinyfd-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-tinyfd-natives-linux-arm64:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.2", + "match": ["org.lwjgl:lwjgl:3.3.2"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "8bd89332c90a90e6bc4aa997a25c05b7db02c90a", + "size": 90795, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl/lwjgl-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-natives-linux-arm64:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.2", + "match": ["org.lwjgl:lwjgl-freetype:3.3.2"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "b7f77ceb951182659fd400437272aa7e96709968", + "size": 924657, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-freetype/lwjgl-freetype-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-freetype-natives-linux-arm32:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.2", + "match": ["org.lwjgl:lwjgl-glfw:3.3.2"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "5907d9a6b7c44fb0612a63bb1cff5992588f65be", + "size": 110067, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-glfw/lwjgl-glfw-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-glfw-natives-linux-arm32:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.2", + "match": ["org.lwjgl:lwjgl-jemalloc:3.3.2"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "9367437ce192e4d6f5725d53d85520644c0b0d6f", + "size": 177571, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-jemalloc/lwjgl-jemalloc-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-jemalloc-natives-linux-arm32:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.2", + "match": ["org.lwjgl:lwjgl-openal:3.3.2"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "7c82bbc33ef49ee4094b216c940db564b2998224", + "size": 503352, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-openal/lwjgl-openal-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-openal-natives-linux-arm32:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.2", + "match": ["org.lwjgl:lwjgl-opengl:3.3.2"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "821f9a2d1d583c44893f42b96f6977682b48a99b", + "size": 59265, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-opengl/lwjgl-opengl-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-opengl-natives-linux-arm32:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.2", + "match": ["org.lwjgl:lwjgl-stb:3.3.2"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "ca9333da184aade20757151f4615f1e27ca521ae", + "size": 154928, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-stb/lwjgl-stb-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-stb-natives-linux-arm32:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.2", + "match": ["org.lwjgl:lwjgl-tinyfd:3.3.2"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "807e220913aa0740449ff90d3b3d825cf5f359ed", + "size": 48788, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-tinyfd/lwjgl-tinyfd-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-tinyfd-natives-linux-arm32:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.2", + "match": ["org.lwjgl:lwjgl:3.3.2"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "afcbfaaa46f217e98a6da4208550f71de1f2a225", + "size": 89347, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl/lwjgl-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-natives-linux-arm32:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.3", + "match": ["org.lwjgl:lwjgl-freetype:3.3.3"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "498965aac06c4a0d42df1fbef6bacd05bde7f974", + "size": 1093516, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl-freetype/lwjgl-freetype-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-freetype-natives-linux-arm64:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.3", + "match": ["org.lwjgl:lwjgl-glfw:3.3.3"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "492a0f11f85b85899a6568f07511160c1b87cd38", + "size": 122159, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl-glfw/lwjgl-glfw-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-glfw-natives-linux-arm64:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.3", + "match": ["org.lwjgl:lwjgl-jemalloc:3.3.3"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "eff8b86798191192fe2cba2dc2776109f30c239d", + "size": 209315, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl-jemalloc/lwjgl-jemalloc-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-jemalloc-natives-linux-arm64:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.3", + "match": ["org.lwjgl:lwjgl-openal:3.3.3"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "ad8f302118a65bb8d615f8a2a680db58fb8f835e", + "size": 592963, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl-openal/lwjgl-openal-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-openal-natives-linux-arm64:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.3", + "match": ["org.lwjgl:lwjgl-opengl:3.3.3"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "2096f6b94b2d68745d858fbfe53aacf5f0c8074c", + "size": 58625, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl-opengl/lwjgl-opengl-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-opengl-natives-linux-arm64:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.3", + "match": ["org.lwjgl:lwjgl-stb:3.3.3"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "ddc177afc2be1ee8d93684b11363b80589a13fe1", + "size": 207418, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl-stb/lwjgl-stb-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-stb-natives-linux-arm64:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.3", + "match": ["org.lwjgl:lwjgl-tinyfd:3.3.3"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "2823a8c955c758d0954d282888075019ef99cec7", + "size": 43864, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl-tinyfd/lwjgl-tinyfd-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-tinyfd-natives-linux-arm64:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.3", + "match": ["org.lwjgl:lwjgl:3.3.3"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "f35d8b6ffe1ac1e3a5eb1d4e33de80f044ad5fd8", + "size": 91294, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl/lwjgl-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-natives-linux-arm64:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.3", + "match": ["org.lwjgl:lwjgl-freetype:3.3.3"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "7dd3b1f751571adaf2c4dc882bc675a5d1e796e6", + "size": 942636, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl-freetype/lwjgl-freetype-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-freetype-natives-linux-arm32:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.3", + "match": ["org.lwjgl:lwjgl-glfw:3.3.3"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "d9af485c32545b37dd5359b163161d42d7534dcf", + "size": 112560, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl-glfw/lwjgl-glfw-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-glfw-natives-linux-arm32:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.3", + "match": ["org.lwjgl:lwjgl-jemalloc:3.3.3"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "109b6931880d02d4e65ced38928a16e41d19873e", + "size": 178324, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl-jemalloc/lwjgl-jemalloc-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-jemalloc-natives-linux-arm32:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.3", + "match": ["org.lwjgl:lwjgl-openal:3.3.3"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "e1702aa09d20359d6cf5cb2999fa7685a785eca7", + "size": 505618, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl-openal/lwjgl-openal-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-openal-natives-linux-arm32:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.3", + "match": ["org.lwjgl:lwjgl-opengl:3.3.3"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "dbba17fc5ac0985d14a57c11f9537617d67b9952", + "size": 59263, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl-opengl/lwjgl-opengl-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-opengl-natives-linux-arm32:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.3", + "match": ["org.lwjgl:lwjgl-stb:3.3.3"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "1ae28ff044699ff29b0e980ffabd73fba8a664b3", + "size": 154931, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl-stb/lwjgl-stb-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-stb-natives-linux-arm32:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.3", + "match": ["org.lwjgl:lwjgl-tinyfd:3.3.3"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "c2a0a05c82c4b9f69ded0b6ad5f417addea78ce2", + "size": 49495, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl-tinyfd/lwjgl-tinyfd-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-tinyfd-natives-linux-arm32:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.3", + "match": ["org.lwjgl:lwjgl:3.3.3"], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "2075c51a80f0ef0f22ba616ba54007ac2b0debd4", + "size": 89565, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl/lwjgl-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-natives-linux-arm32:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Replace glfw from 3.3.1 with version from 3.3.2 to prevent stack smashing", + "match": ["org.lwjgl:lwjgl-glfw-natives-linux:3.3.1"], + "override": { + "downloads": { + "artifact": { + "sha1": "0766bb0e8e829598b1c8052fd8173c62af741c52", + "size": 115553, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-glfw/lwjgl-glfw-natives-linux.jar" + } + }, + "name": "org.lwjgl:lwjgl-glfw-natives-linux:3.3.2-lwjgl.1" + } + }, + { + "_comment": "Use newer JNA on macOS to prevent crashes due to faulty assertion", + "match": [ + "net.java.dev.jna:jna:5.6.0", + "net.java.dev.jna:jna:5.8.0", + "net.java.dev.jna:jna:5.9.0", + "net.java.dev.jna:jna:5.10.0", + "net.java.dev.jna:jna:5.12.1" + ], + "override": { + "rules": [ + { + "action": "allow" + }, + { + "action": "disallow", + "os": { + "name": "osx" + } + }, + { + "action": "disallow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "1200e7ebeedbe0d10062093f32925a912020e747", + "size": 1879325, + "url": "https://libraries.minecraft.net/net/java/dev/jna/jna/5.13.0/jna-5.13.0.jar" + } + }, + "name": "net.java.dev.jna:jna:5.13.0", + "rules": [ + { + "action": "allow", + "os": { + "name": "osx" + } + }, + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Use newer JNA on macOS to prevent crashes due to faulty assertion", + "match": [ + "net.java.dev.jna:jna-platform:5.6.0", + "net.java.dev.jna:jna-platform:5.8.0", + "net.java.dev.jna:jna-platform:5.9.0", + "net.java.dev.jna:jna-platform:5.10.0", + "net.java.dev.jna:jna-platform:5.12.1" + ], + "override": { + "rules": [ + { + "action": "allow" + }, + { + "action": "disallow", + "os": { + "name": "osx" + } + }, + { + "action": "disallow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "88e9a306715e9379f3122415ef4ae759a352640d", + "size": 1363209, + "url": "https://libraries.minecraft.net/net/java/dev/jna/jna-platform/5.13.0/jna-platform-5.13.0.jar" + } + }, + "name": "net.java.dev.jna:jna-platform:5.13.0", + "rules": [ + { + "action": "allow", + "os": { + "name": "osx" + } + }, + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + } + ] + } +] diff --git a/apps/meta-generator/src/import/mojang/library/mod.rs b/apps/meta-generator/src/import/mojang/library/mod.rs new file mode 100644 index 00000000000..a061acc19db --- /dev/null +++ b/apps/meta-generator/src/import/mojang/library/mod.rs @@ -0,0 +1,156 @@ +use std::{collections::HashMap, sync::LazyLock}; + +use serde::{Deserialize, Serialize}; + +use crate::{ + upstream::mojang::{ + Library, LibraryDownloads, LibraryExtract, OperatingSystem, Rule, + }, + util::from_json_slice, +}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LibraryPatch { + // #[serde(rename = "_comment")] + // pub _comment: String, + #[serde(rename = "match")] + pub matches: Vec, + pub additional_libraries: Option>, + #[serde(rename = "override")] + pub override_library: Option, + pub patch_additional_libraries: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PartialLibrary { + pub downloads: Option, + pub extract: Option, + pub name: Option, + pub url: Option, + pub natives: Option>, + pub rules: Option>, + pub checksums: Option>, + pub include_in_classpath: Option, +} + +/// Bundled copy of PrismLauncher's library patches. +/// +/// Sourced from +/// +/// The upstream repository is licensed under the Microsoft Public License (Ms-PL). +static LIBRARY_PATCHES: LazyLock> = LazyLock::new(|| { + from_json_slice(include_bytes!("library-patches.json")) + .expect("shouldn't fail to deserialize") +}); + +pub fn patch_library(mut library: Library) -> Vec { + let patches = LIBRARY_PATCHES.as_slice(); + + let mut libraries = Vec::new(); + let matching_patches = patches + .iter() + .filter(|patch| patch.matches.contains(&library.name)) + .collect::>(); + for patch in matching_patches { + if let Some(override_library) = &patch.override_library { + library = merge_partial_library(override_library.clone(), library); + } + if let Some(additional_libraries) = &patch.additional_libraries { + for additional_library in additional_libraries { + if patch.patch_additional_libraries.unwrap_or(false) { + libraries.extend(patch_library(additional_library.clone())); + } else { + libraries.push(additional_library.clone()); + } + } + } + } + libraries.push(library); + libraries +} + +pub fn merge_partial_library( + partial: PartialLibrary, + mut library: Library, +) -> Library { + if let Some(downloads) = partial.downloads { + if let Some(existing) = &mut library.downloads { + if let Some(artifact) = downloads.artifact { + existing.artifact = Some(artifact); + } + if let Some(classifiers) = downloads.classifiers { + if let Some(existing_classifiers) = &mut existing.classifiers { + existing_classifiers.extend(classifiers); + } else { + existing.classifiers = Some(classifiers); + } + } + } else { + library.downloads = Some(downloads); + } + } + if let Some(extract) = partial.extract { + library.extract = Some(extract); + } + if let Some(name) = partial.name { + library.name = name; + } + if let Some(url) = partial.url { + library.url = Some(url); + } + if let Some(natives) = partial.natives { + if let Some(existing) = &mut library.natives { + existing.extend(natives); + } else { + library.natives = Some(natives); + } + } + if let Some(rules) = partial.rules { + if let Some(existing) = &mut library.rules { + existing.extend(rules); + } else { + library.rules = Some(rules); + } + } + if let Some(checksums) = partial.checksums { + library.checksums = Some(checksums); + } + if let Some(include_in_classpath) = partial.include_in_classpath { + library.include_in_classpath = include_in_classpath; + } + library +} + +#[cfg(test)] +mod tests { + use super::patch_library; + use crate::upstream::mojang::Library; + + #[test] + fn bundled_patches_load_and_add_missing_dependencies() { + let library: Library = serde_json::from_value(serde_json::json!({ + "name": "org.lwjgl:lwjgl:3.2.2" + })) + .unwrap(); + let patched = patch_library(library); + assert!( + patched + .iter() + .any(|library| library.name == "org.lwjgl:lwjgl-tinyfd:3.2.2") + ); + assert_eq!(patched.last().unwrap().name, "org.lwjgl:lwjgl:3.2.2"); + } + + #[test] + fn unmatched_library_is_unchanged() { + let library: Library = serde_json::from_value(serde_json::json!({ + "name": "com.example:unmatched:1" + })) + .unwrap(); + let before = serde_json::to_value(&library).unwrap(); + let patched = patch_library(library); + assert_eq!(patched.len(), 1); + assert_eq!(serde_json::to_value(&patched[0]).unwrap(), before); + } +} diff --git a/apps/meta-generator/src/import/mojang/mod.rs b/apps/meta-generator/src/import/mojang/mod.rs new file mode 100644 index 00000000000..2f438cc2b4b --- /dev/null +++ b/apps/meta-generator/src/import/mojang/mod.rs @@ -0,0 +1,83 @@ +use std::sync::atomic::{self, AtomicUsize}; + +use anyhow::{Result, ensure}; +use futures::{StreamExt, stream}; +use tracing::{info, info_span}; +use tracing_anyhow::FutureContext; + +use crate::{ + import::download::DownloadRunContext, + model, + upstream::mojang::{CATALOG_URL, Catalog, Version}, +}; + +mod library; + +pub use library::patch_library; + +pub async fn download(cx: &mut DownloadRunContext<'_>) -> Result<()> { + let initial_errors = cx.errors.len(); + let (catalog, sha256) = cx + .download_json::(CATALOG_URL) + .context(info_span!("fetching catalog")) + .await?; + info!( + num_versions = catalog.versions.len(), + "downloaded Mojang catalog" + ); + + let cataloged_sha1s = catalog + .versions + .iter() + .map(|version| version.sha1) + .collect::>(); + let existing_sha1s = model::BlobHash::all() + .select(model::BlobHash::fields().sha1()) + .filter(model::BlobHash::fields().sha1().in_list(cataloged_sha1s)) + .exec(*cx.conn().await) + .context(info_span!("fetching existing sha1s")) + .await?; + let num_versions = catalog.versions.len(); + let missing_versions = catalog + .versions + .into_iter() + .filter(|version| !existing_sha1s.contains(&version.sha1)) + .collect::>(); + info!( + "catalog contains {num_versions} versions, of which {} are missing; downloading", + missing_versions.len() + ); + + let num_total = missing_versions.len(); + let num_done = AtomicUsize::new(0); + let task = async |version: Version| { + cx.download_blob(version.url.clone()) + .context(info_span!("downloading version", %version.id)) + .await + .inspect_err(|err| cx.errors.push(err)) + .ok(); + + let num_done = num_done.fetch_add(1, atomic::Ordering::SeqCst) + 1; + if num_done.is_multiple_of(100) || num_done == num_total { + info!("downloaded {num_done}/{num_total} versions"); + } + }; + stream::iter(missing_versions) + .for_each_concurrent(cx.download_concurrency, task) + .await; + + ensure!( + cx.errors.len() == initial_errors, + "Mojang item downloads failed; catalog not registered" + ); + + toasty::create!(model::MojangCatalog { + download_run_id: cx.download_run_id, + sha256, + }) + .exec(*cx.conn().await) + .context(info_span!("inserting catalog")) + .await?; + + Ok(()) +} diff --git a/apps/meta-generator/src/import/neoforge.rs b/apps/meta-generator/src/import/neoforge.rs new file mode 100644 index 00000000000..950282ff3ac --- /dev/null +++ b/apps/meta-generator/src/import/neoforge.rs @@ -0,0 +1,157 @@ +use std::{ + collections::HashSet, + sync::atomic::{self, AtomicUsize}, +}; + +use anyhow::{Result, ensure}; +use futures::{StreamExt, stream}; +use tracing::{info, info_span}; +use tracing_anyhow::FutureContext; + +use crate::{ + import::download::DownloadRunContext, + model::{self, ForgelikeInstallerName}, + upstream::neoforge::{Catalog, FORGE_CATALOG_URL, NEOFORGE_CATALOG_URL}, +}; + +pub async fn download(cx: &mut DownloadRunContext<'_>) -> Result<()> { + let initial_errors = cx.errors.len(); + let (forge_catalog, forge_sha256) = cx + .download_json::(FORGE_CATALOG_URL) + .context(info_span!("fetching Forge catalog")) + .await?; + let (neoforge_catalog, neoforge_sha256) = cx + .download_json::(NEOFORGE_CATALOG_URL) + .context(info_span!("fetching NeoForge catalog")) + .await?; + info!( + num_forge_versions = forge_catalog.versions.len(), + num_neoforge_versions = neoforge_catalog.versions.len(), + "downloaded NeoForge catalogs" + ); + + let mut seen = HashSet::new(); + let cataloged_versions = + forge_catalog + .versions + .into_iter() + .map(|version| version.0.parse().map(|name| (name, "forge"))) + .chain(neoforge_catalog.versions.into_iter().map(|version| { + version.0.parse().map(|name| (name, "neoforge")) + })) + .collect::>>()? + .into_iter() + .filter(|(name, _)| !model::NEOFORGE_BLACKLIST.contains(&name.as_str())) + .filter(|(name, _)| seen.insert(name.clone())) + .collect::>(); + let existing_versions = model::ForgelikeInstaller::all() + .filter( + model::ForgelikeInstaller::fields() + .loader() + .eq(model::ForgelikeLoader::Neoforge), + ) + .select(model::ForgelikeInstaller::fields().name()) + .exec(*cx.conn().await) + .context(info_span!("fetching existing NeoForge installer records")) + .await?; + let missing_versions = cataloged_versions + .iter() + .filter(|(name, _)| !existing_versions.contains(name)) + .collect::>(); + info!( + "catalogs contain {} distinct versions, of which {} are missing; downloading installers", + cataloged_versions.len(), + missing_versions.len() + ); + + let num_total = missing_versions.len(); + let num_done = AtomicUsize::new(0); + let task = async |(installer_name, artifact): &( + ForgelikeInstallerName, + &str, + )| { + let url = installer_url(artifact, installer_name); + async { + let sha256 = cx + .download_blob(url) + .context(info_span!("downloading installer", %installer_name, artifact)) + .await?; + toasty::create!(model::ForgelikeInstaller { + loader: model::ForgelikeLoader::Neoforge, + name: installer_name, + download_run_id: cx.download_run_id, + sha256, + }) + .exec(*cx.conn().await) + .context(info_span!("inserting installer record")) + .await?; + anyhow::Ok(()) + } + .await + .inspect_err(|err| cx.errors.push(err)) + .ok(); + + let num_done = num_done.fetch_add(1, atomic::Ordering::SeqCst) + 1; + if num_done.is_multiple_of(10) || num_done == num_total { + info!("downloaded {num_done}/{num_total} version installers"); + } + }; + stream::iter(missing_versions) + .for_each_concurrent(cx.download_concurrency, task) + .await; + + ensure!( + cx.errors.len() == initial_errors, + "NeoForge item downloads failed; catalogs not registered" + ); + toasty::create!(model::NeoforgeCatalog { + download_run_id: cx.download_run_id, + forge_sha256, + neoforge_sha256, + }) + .exec(*cx.conn().await) + .context(info_span!("inserting catalog")) + .await?; + + Ok(()) +} + +fn installer_url(artifact: &str, name: &ForgelikeInstallerName) -> String { + format!( + "https://maven.neoforged.net/releases/net/neoforged/{artifact}/{name}/{artifact}-{name}-installer.jar" + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn installer_urls_preserve_catalog_namespace_and_version() { + for (artifact, version, expected) in [ + ( + "forge", + "1.20.1-47.1.74", + "https://maven.neoforged.net/releases/net/neoforged/forge/1.20.1-47.1.74/forge-1.20.1-47.1.74-installer.jar", + ), + ( + "forge", + "47.1.82", + "https://maven.neoforged.net/releases/net/neoforged/forge/47.1.82/forge-47.1.82-installer.jar", + ), + ( + "neoforge", + "20.2.29-beta", + "https://maven.neoforged.net/releases/net/neoforged/neoforge/20.2.29-beta/neoforge-20.2.29-beta-installer.jar", + ), + ] { + assert_eq!( + installer_url( + artifact, + &version.parse::().unwrap() + ), + expected + ); + } + } +} diff --git a/apps/meta-generator/src/import/quilt.rs b/apps/meta-generator/src/import/quilt.rs new file mode 100644 index 00000000000..3f1c758fbd4 --- /dev/null +++ b/apps/meta-generator/src/import/quilt.rs @@ -0,0 +1,207 @@ +use std::sync::atomic::{self, AtomicUsize}; + +use anyhow::{Result, ensure}; +use futures::{StreamExt, stream}; +use tracing::{info, info_span}; +use tracing_anyhow::FutureContext; +use url::Url; + +use crate::{ + import::download::DownloadRunContext, + model, + upstream::{ + fabriclike::{ + GameLoaderProfile, GameVersion, GameVersionName, LoaderVersionName, + }, + quilt::{ + CATALOG_URL, Catalog, LoaderVersion, MetadataGroup, PROFILE_BASE_URL, + }, + }, +}; + +pub const SKIPPED_LOADER_VERSION: &str = "0.17.5-beta.4"; + +pub async fn download(cx: &mut DownloadRunContext<'_>) -> Result<()> { + let initial_errors = cx.errors.len(); + let (mut catalog, sha256) = cx + .download_json::(CATALOG_URL) + .context(info_span!("fetching catalog")) + .await?; + catalog + .loader + .retain(|loader| loader.version.0 != SKIPPED_LOADER_VERSION); + info!( + num_game_versions = catalog.game.len(), + num_loader_versions = catalog.loader.len(), + "downloaded Quilt catalog" + ); + + let groups = profile_groups(&catalog.game); + let template_game_versions = groups + .iter() + .map(|group| &group.template) + .collect::>(); + let num_profiles = template_game_versions.len() * catalog.loader.len(); + info!("downloading {num_profiles} loader profiles"); + + let template_pairs = template_game_versions + .iter() + .flat_map(|game_version| { + catalog.loader.iter().map(|loader| (*game_version, loader)) + }) + .collect::>(); + + let num_total = template_pairs.len(); + let num_done = AtomicUsize::new(0); + let task = + async |(game_version, loader): (&GameVersionName, &LoaderVersion)| { + let loader_version = &loader.version; + let url = profile_url(game_version, loader_version); + + cx.download_json::(&url) + .await + .inspect_err(|err| cx.errors.push(err)) + .ok(); + + let num_done = num_done.fetch_add(1, atomic::Ordering::SeqCst) + 1; + if num_done.is_multiple_of(10) || num_done == num_total { + info!("downloaded {num_done}/{num_total} loader profiles"); + } + }; + stream::iter(template_pairs) + .for_each_concurrent(cx.download_concurrency, task) + .await; + + ensure!( + cx.errors.len() == initial_errors, + "Quilt item downloads failed; catalog not registered" + ); + + toasty::create!(model::QuiltCatalog { + download_run_id: cx.download_run_id, + sha256, + }) + .exec(*cx.conn().await) + .context(info_span!("inserting catalog")) + .await?; + + Ok(()) +} + +pub fn profile_url( + game: &GameVersionName, + loader: &LoaderVersionName, +) -> String { + let mut url = + Url::parse(PROFILE_BASE_URL).expect("valid Quilt profile base URL"); + url.path_segments_mut() + .expect("Quilt profile URL supports path segments") + .pop_if_empty() + .extend([game.0.as_str(), loader.0.as_str(), "profile", "json"]); + url.to_string() +} + +pub fn profile_groups(game_versions: &[GameVersion]) -> Vec { + let (modern, legacy): (Vec<_>, Vec<_>) = game_versions + .iter() + .map(|game| game.version.clone()) + .partition(is_modern_game_version); + let mut groups = Vec::with_capacity(2); + if let Some(template) = legacy + .iter() + .find(|game| game.0 == "1.21") + .or_else(|| legacy.first()) + { + groups.push(MetadataGroup { + id: "v1".to_owned(), + template: template.clone(), + games: legacy, + }); + } + if let Some(template) = modern.first() { + groups.push(MetadataGroup { + id: "v2".to_owned(), + template: template.clone(), + games: modern, + }); + } + groups +} + +fn is_modern_game_version(game_version: &GameVersionName) -> bool { + game_version + .0 + .split(['.', 'w']) + .next() + .and_then(|major| major.parse::().ok()) + .is_some_and(|major| major >= 26) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn games(ids: &[&str]) -> Vec { + ids.iter() + .map(|id| GameVersion { + version: GameVersionName((*id).to_owned()), + stable: true, + }) + .collect() + } + + #[test] + fn groups_preserve_order_and_prefer_legacy_template() { + let groups = profile_groups(&games(&[ + "26.2", "1.21.11", "26w14a", "1.21", "25w10a", + ])); + assert_eq!(groups.len(), 2); + assert_eq!(groups[0].id, "v1"); + assert_eq!(groups[0].template.0, "1.21"); + assert_eq!( + groups[0].games, + games(&["1.21.11", "1.21", "25w10a"]) + .into_iter() + .map(|game| game.version) + .collect::>() + ); + assert_eq!(groups[1].id, "v2"); + assert_eq!(groups[1].template.0, "26.2"); + assert_eq!( + groups[1].games, + games(&["26.2", "26w14a"]) + .into_iter() + .map(|game| game.version) + .collect::>() + ); + } + + #[test] + fn legacy_template_falls_back_to_first_legacy_game() { + let groups = profile_groups(&games(&["26w14a", "1.20.6", "1.19"])); + assert_eq!(groups[0].template.0, "1.20.6"); + assert_eq!(groups[1].template.0, "26w14a"); + } + + #[test] + fn absent_groups_are_omitted() { + assert!(profile_groups(&[]).is_empty()); + let legacy = profile_groups(&games(&["1.21"])); + assert_eq!(legacy.len(), 1); + assert_eq!(legacy[0].id, "v1"); + let modern = profile_groups(&games(&["26w14a", "27.1"])); + assert_eq!(modern.len(), 1); + assert_eq!(modern[0].id, "v2"); + } + + #[test] + fn profile_url_matches_downloader_source() { + assert_eq!( + profile_url( + &GameVersionName("1.21".to_owned()), + &LoaderVersionName("0.30.0".to_owned()) + ), + "https://meta.quiltmc.org/v3/versions/loader/1.21/0.30.0/profile/json" + ); + } +} diff --git a/apps/meta-generator/src/lib.rs b/apps/meta-generator/src/lib.rs new file mode 100644 index 00000000000..c36951451c1 --- /dev/null +++ b/apps/meta-generator/src/lib.rs @@ -0,0 +1,170 @@ +use std::{sync::Arc, time::Duration}; + +use anyhow::{Context, Result}; +use figment::Figment; +use toasty::migration::MigrationSet; +use tracing::{info, info_span, level_filters::LevelFilter}; +use tracing_anyhow::FutureContext; +use tracing_subscriber::{ + EnvFilter, Layer, layer::SubscriberExt, util::SubscriberInitExt, +}; + +use crate::{ + config::Config, + store::{BlobCas, BlobStore, MavenStore, StoreVisibility}, + upstream::Upstreams, + util::fastly, +}; + +mod config; +mod export; +mod import; +mod store; +mod util; + +pub use modrinth_minecraft_meta::{model, upstream}; + +#[derive(Debug, clap::Parser)] +struct Cli { + /// Process Minecraft game version info from Mojang? + #[arg(long)] + mojang: bool, + /// Process Fabric loader info? + #[arg(long)] + fabric: bool, + /// Process Forge loader info? + #[arg(long)] + forge: bool, + /// Process NeoForge loader info? + #[arg(long)] + neoforge: bool, + /// Process Quilt loader info? + #[arg(long)] + quilt: bool, + #[command(subcommand)] + command: Command, +} + +#[derive(Debug, clap::Subcommand)] +enum Command { + /// Download manifests and sources from upstreams, and extract their + /// information into the local database + Import, + /// Take processed information from the database, produce JSON manifests + /// from them, and upload them to the public file store + Export, +} + +pub async fn main() -> Result<()> { + static MIGRATIONS: MigrationSet = toasty::embed_migrations!(); + + dotenvy::dotenv().ok(); + let cli = ::parse(); + tracing_subscriber::registry() + .with(tracing_anyhow::ErrorLayer::default()) + .with( + tracing_subscriber::fmt::layer() + .with_timer(tracing_subscriber::fmt::time::ChronoLocal::new( + "%H:%M".to_owned(), + )) + .with_target(false) + .with_writer(std::io::stderr) + .with_filter( + EnvFilter::builder() + .with_default_directive(LevelFilter::INFO.into()) + .from_env_lossy(), + ), + ) + .init(); + + let config = create_config()?; + + let http = reqwest::Client::builder() + .timeout(Duration::from_secs(30)) + .build() + .context("building HTTP client")?; + + let db = connect_to_db(&config).await?; + let report = MIGRATIONS + .apply(&db) + .context(info_span!("applying migrations")) + .await?; + info!("applied {} migrations", report.applied()); + + let fastly = if let Some(config) = config.fastly { + fastly::Client::new(http.clone(), config.base_url, config.key) + .map(|client| Some(Arc::new(client))) + .context("creating Fastly client")? + } else { + None + }; + + let public_blobs = BlobStore::new( + config.public_s3, + fastly.clone(), + StoreVisibility::Public, + ) + .context(info_span!("creating public blob store")) + .await?; + // private store is never behind a CDN, so we don't need to purge with Fastly + let private_blobs = + BlobStore::new(config.private_s3, None, StoreVisibility::Private) + .context(info_span!("creating private blob store")) + .await?; + let cas = BlobCas::new(private_blobs.clone()); + let maven = MavenStore::new(public_blobs.clone()); + + let app = AppState { + http, + db, + concurrency: config.concurrency, + cas, + public_blobs, + maven, + }; + + let upstreams = { + let all_upstreams = !cli.mojang + && !cli.fabric + && !cli.forge + && !cli.neoforge + && !cli.quilt; + Upstreams { + mojang: all_upstreams || cli.mojang, + fabric: all_upstreams || cli.fabric, + forge: all_upstreams || cli.forge, + neoforge: all_upstreams || cli.neoforge, + quilt: all_upstreams || cli.quilt, + } + }; + + match cli.command { + Command::Import => import::run(app, upstreams).await, + Command::Export => export::run(app, upstreams).await, + } +} + +#[derive(Debug)] +struct AppState { + http: reqwest::Client, + db: toasty::Db, + concurrency: config::Concurrency, + cas: BlobCas, + public_blobs: BlobStore, + maven: MavenStore, +} + +pub fn create_config() -> Result { + Figment::new() + .merge(figment::providers::Env::prefixed("MR_").split("__")) + .extract::() + .context("parsing config") +} + +pub async fn connect_to_db(config: &Config) -> Result { + toasty::Db::builder() + .models(toasty::models!(modrinth_minecraft_meta::*)) + .connect(config.database_url.as_str()) + .context(info_span!("connecting to database")) + .await +} diff --git a/apps/meta-generator/src/main.rs b/apps/meta-generator/src/main.rs new file mode 100644 index 00000000000..6227782c763 --- /dev/null +++ b/apps/meta-generator/src/main.rs @@ -0,0 +1,4 @@ +#[tokio::main] +async fn main() -> anyhow::Result<()> { + meta_generator::main().await +} diff --git a/apps/meta-generator/src/store/cas.rs b/apps/meta-generator/src/store/cas.rs new file mode 100644 index 00000000000..b66a0168a87 --- /dev/null +++ b/apps/meta-generator/src/store/cas.rs @@ -0,0 +1,54 @@ +use anyhow::Result; +use tracing::info_span; +use tracing_anyhow::FutureContext; + +use crate::{ + model, + store::{BlobStore, ContentType}, + util::{Sha1, Sha256}, +}; + +/// Stores arbitrary binary [`Vec`] blobs keyed by the binary data's +/// [`Sha256`] hash. +/// +/// This is a _content-addressed store_ (CAS) model. +#[derive(Debug, Clone)] +pub struct BlobCas { + imp: BlobStore, +} + +impl BlobCas { + #[must_use] + pub const fn new(imp: BlobStore) -> Self { + Self { imp } + } + + pub async fn get(&self, sha256: Sha256) -> Result> { + self.imp.get(&path_for(sha256)).await + } + + pub async fn put( + &self, + exec: &mut dyn toasty::Executor, + data: &[u8], + ) -> Result { + let sha256 = Sha256::from_digest(data); + let sha1 = Sha1::from_digest(data); + model::BlobHash::upsert_by_sha256(sha256) + .sha1(sha1) + .or_ignore() + .exec(exec) + .context(info_span!("upserting blob hash record")) + .await?; + self.imp + .put(&path_for(sha256), data, ContentType::Binary) + .await?; + Ok(sha256) + } +} + +fn path_for(sha256: Sha256) -> String { + let sha256 = sha256.to_string(); + let (first, _) = sha256.split_at(2); + format!("blobs/{first}/{sha256}") +} diff --git a/apps/meta-generator/src/store/cdn.rs b/apps/meta-generator/src/store/cdn.rs new file mode 100644 index 00000000000..3cae9ed0d63 --- /dev/null +++ b/apps/meta-generator/src/store/cdn.rs @@ -0,0 +1,53 @@ +use std::{pin::Pin, sync::Arc}; + +use anyhow::Result; +use tracing::info_span; +use tracing_anyhow::FutureContext; +use url::Url; + +use crate::{ + store::{ContentType, StoreOps}, + util::fastly, +}; + +#[derive(Debug)] +pub struct CdnStore { + imp: Box, + fastly: Arc, +} + +pub fn new(imp: Box, fastly: Arc) -> CdnStore { + CdnStore { imp, fastly } +} + +impl StoreOps for CdnStore { + fn url_for(&self, path: &str) -> Url { + self.imp.url_for(path) + } + + fn get<'a>( + &self, + path: &'a str, + ) -> Pin>> + 'a>> { + self.imp.get(path) + } + + fn put<'a>( + &self, + path: &'a str, + data: &'a [u8], + content_type: ContentType, + ) -> Pin> + 'a>> { + let fut = self.imp.put(path, data, content_type); + let fastly = self.fastly.clone(); + let url = self.imp.url_for(path); + Box::pin(async move { + fut.await?; + fastly + .purge_url(&url) + .context(info_span!("purging Fastly cache")) + .await?; + anyhow::Ok(()) + }) + } +} diff --git a/apps/meta-generator/src/store/fs.rs b/apps/meta-generator/src/store/fs.rs new file mode 100644 index 00000000000..809221873f5 --- /dev/null +++ b/apps/meta-generator/src/store/fs.rs @@ -0,0 +1,151 @@ +use std::pin::Pin; + +use anyhow::{Context, Result}; +use tokio::{fs, io::AsyncWriteExt}; +use tracing::{info_span, warn}; +use tracing_anyhow::FutureContext; +use url::Url; +use uuid::Uuid; + +use crate::store::{ContentType, StoreOps, StoreVisibility}; + +#[derive(Debug)] +pub struct FsStore { + root: Url, +} + +pub async fn new(visibility: StoreVisibility) -> Result { + let dirs = directories::ProjectDirs::from( + "com.modrinth", + "Modrinth", + "modrinth-minecraft-meta", + ) + .context("fetching project dirs")?; + let root = dirs.data_dir().join(match visibility { + StoreVisibility::Public => "public", + StoreVisibility::Private => "private", + }); + fs::create_dir_all(&root) + .context(info_span!("creating blobs dir")) + .await?; + let root = fs::canonicalize(root) + .context(info_span!("canonicalizing root dir")) + .await?; + let root = Url::from_directory_path(root) + .expect("canonicalized root is an absolute directory path"); + Ok(FsStore { root }) +} + +impl StoreOps for FsStore { + fn url_for(&self, path: &str) -> Url { + let mut url = self.root.clone(); + url.path_segments_mut() + .expect("root is a directory URL") + .pop_if_empty() + .extend(path.split('/')); + url + } + + fn get<'a>( + &self, + path: &'a str, + ) -> Pin>> + 'a>> { + // this is vulnerable to path traversal, + // but we always use wrappers on top of `StoreOps`, + // which must create well-formed, non-malicious paths. + let path = self + .root + .to_file_path() + .expect("root is a local file URL") + .join(path); + Box::pin(async move { + let data = fs::read(&path) + .context(info_span!("reading file", ?path)) + .await?; + Ok(data) + }) + } + + fn put<'a>( + &self, + path: &'a str, + data: &'a [u8], + _content_type: ContentType, + ) -> Pin> + 'a>> { + let path = self + .root + .to_file_path() + .expect("root is a local file URL") + .join(path); + Box::pin(async move { + let parent = path.parent().context("path has no parent")?; + fs::create_dir_all(&parent) + .context(info_span!("creating parent dir", ?parent)) + .await?; + let temp_path = parent.join(format!(".{}.tmp", Uuid::now_v7())); + let mut file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&temp_path) + .context(info_span!("creating temporary file", ?temp_path)) + .await?; + let result = async { + file.write_all(data) + .context(info_span!("writing temporary file", ?temp_path)) + .await?; + file.flush() + .context(info_span!("flushing temporary file", ?temp_path)) + .await?; + drop(file); + fs::rename(&temp_path, &path) + .context(info_span!("publishing file", ?temp_path, ?path)) + .await?; + Ok(()) + } + .await; + if result.is_err() + && let Err(err) = fs::remove_file(&temp_path).await + { + warn!(?temp_path, ?err, "failed to remove temporary file"); + } + result + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn urls_preserve_directory_slashes_and_encode_file_names() { + let store = FsStore { + root: Url::from_directory_path( + std::env::temp_dir().join("minecraft-meta-url-test"), + ) + .unwrap(), + }; + let directory = store.url_for("maven/"); + assert!(directory.as_str().ends_with("/maven/")); + assert_eq!( + directory.join("net/fabricmc/library.jar").unwrap(), + store.url_for("maven/net/fabricmc/library.jar") + ); + let file = + store.url_for("minecraft/v0/versions/3D Shareware v1.34.json"); + assert!(file.as_str().ends_with("/3D%20Shareware%20v1.34.json")); + assert!(!file.as_str().ends_with('/')); + assert!(!store.url_for("maven").as_str().ends_with('/')); + let literal = store.url_for("maven/file%20?#.jar"); + assert!(literal.query().is_none()); + assert!(literal.fragment().is_none()); + assert_eq!( + literal.to_file_path().unwrap(), + store + .root + .to_file_path() + .unwrap() + .join("maven/file%20?#.jar") + ); + } +} diff --git a/apps/meta-generator/src/store/maven.rs b/apps/meta-generator/src/store/maven.rs new file mode 100644 index 00000000000..316cd9edbf4 --- /dev/null +++ b/apps/meta-generator/src/store/maven.rs @@ -0,0 +1,27 @@ +use anyhow::Result; + +use crate::{ + store::{BlobStore, ContentType}, + util::MavenCoordinate, +}; + +#[derive(Debug, Clone)] +pub struct MavenStore { + imp: BlobStore, +} + +impl MavenStore { + #[must_use] + pub const fn new(imp: BlobStore) -> Self { + Self { imp } + } + + pub async fn put( + &self, + coordinate: &MavenCoordinate, + data: &[u8], + ) -> Result<()> { + let path = coordinate.to_maven_path(); + self.imp.put(&path, data, ContentType::Binary).await + } +} diff --git a/apps/meta-generator/src/store/mod.rs b/apps/meta-generator/src/store/mod.rs new file mode 100644 index 00000000000..79eb30a265f --- /dev/null +++ b/apps/meta-generator/src/store/mod.rs @@ -0,0 +1,111 @@ +mod cas; +mod cdn; +mod fs; +mod maven; +mod s3; + +pub use cas::BlobCas; +pub use maven::MavenStore; +use url::Url; + +use std::{fmt::Debug, pin::Pin, sync::Arc}; + +use anyhow::Result; + +use crate::{ + config::{self}, + util::fastly, +}; + +/// Stores arbitrary binary [`Vec`] blobs at a string `path`. +/// +/// If you're looking for content-addressed storage (store by a SHA256 hash), +/// see [`ContentStore`]. +#[derive(Debug, Clone)] +pub struct BlobStore { + imp: Arc, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ContentType { + Json, + Binary, +} + +impl ContentType { + pub const fn as_str(self) -> &'static str { + match self { + Self::Json => "application/json", + Self::Binary => "application/octet-stream", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StoreVisibility { + Public, + Private, +} + +impl BlobStore { + pub async fn new( + s3_config: Option, + fastly: Option>, + visibility: StoreVisibility, + ) -> Result { + let mut imp: Box = match s3_config { + Some(config) => Box::new(s3::new(&config)?), + None => Box::new(fs::new(visibility).await?), + }; + + if visibility == StoreVisibility::Public + && let Some(fastly) = fastly + { + imp = Box::new(cdn::new(imp, fastly)); + } + + Ok(Self { + imp: Arc::from(imp), + }) + } + + /// Get the [`Url`] for a file at a specific `path`. + pub fn url_for(&self, path: &str) -> Url { + self.imp.url_for(path) + } + + /// Get a blob at a `path`. + /// + /// The `path` here may be vulnerable to directory traversal if using the + /// filesystem store; it is the caller's responsibility to ensure this is + /// a well-formed path with no traversal. + pub async fn get(&self, path: &str) -> Result> { + self.imp.get(path).await + } + + /// Put a blob `data` at `path`. + pub async fn put( + &self, + path: &str, + data: &[u8], + content_type: ContentType, + ) -> Result<()> { + self.imp.put(path, data, content_type).await + } +} + +trait StoreOps: Send + Sync + Debug + 'static { + fn url_for(&self, path: &str) -> Url; + + fn get<'a>( + &self, + path: &'a str, + ) -> Pin>> + 'a>>; + + fn put<'a>( + &self, + path: &'a str, + data: &'a [u8], + content_type: ContentType, + ) -> Pin> + 'a>>; +} diff --git a/apps/meta-generator/src/store/s3.rs b/apps/meta-generator/src/store/s3.rs new file mode 100644 index 00000000000..7eeea0e3517 --- /dev/null +++ b/apps/meta-generator/src/store/s3.rs @@ -0,0 +1,137 @@ +use std::pin::Pin; + +use anyhow::{Context, Result, ensure}; +use derive_more::Debug; +use s3::{Bucket, Region, creds::Credentials}; +use tracing::info_span; +use tracing_anyhow::FutureContext; +use url::Url; + +use crate::{ + config::S3, + store::{ContentType, StoreOps}, +}; + +#[derive(Debug)] +pub struct S3Store { + #[debug(skip)] + bucket: Box, + base_url: Url, +} + +pub fn new(config: &S3) -> Result { + ensure!(!config.bucket.is_empty(), "S3 bucket must not be empty"); + + for (name, url) in [ + ("endpoint", &config.endpoint), + ("base_url", &config.base_url), + ] { + ensure!( + matches!(url.scheme(), "http" | "https") + && url.host_str().is_some(), + "S3 {name} must be an HTTP(S) URL" + ); + } + let credentials = + Credentials::default().context("loading S3 credentials")?; + let bucket = Bucket::new( + &config.bucket, + Region::Custom { + region: config.region.clone(), + endpoint: config.endpoint.to_string(), + }, + credentials, + ) + .context("creating S3 bucket client")?; + let bucket = if config.path_style { + bucket.with_path_style() + } else { + bucket + }; + Ok(S3Store { + bucket, + base_url: config.base_url.clone(), + }) +} + +impl StoreOps for S3Store { + fn url_for(&self, path: &str) -> Url { + let mut url = self.base_url.clone(); + url.path_segments_mut() + .expect("S3 base URL supports path segments") + .pop_if_empty() + .extend(path.split('/')); + url + } + + fn get<'a>( + &self, + path: &'a str, + ) -> Pin>> + 'a>> { + let bucket = self.bucket.clone(); + Box::pin(async move { + let response = bucket + .get_object(path) + .context(info_span!("reading S3 object", path)) + .await?; + ensure!( + (200..300).contains(&response.status_code()), + "S3 GET {path} returned HTTP {}", + response.status_code() + ); + Ok(response.to_vec()) + }) + } + + fn put<'a>( + &self, + path: &'a str, + data: &'a [u8], + content_type: ContentType, + ) -> Pin> + 'a>> { + let bucket = self.bucket.clone(); + Box::pin(async move { + let response = bucket + .put_object_with_content_type(path, data, content_type.as_str()) + .context(info_span!("writing S3 object", path)) + .await?; + ensure!( + (200..300).contains(&response.status_code()), + "S3 PUT {path} returned HTTP {}", + response.status_code() + ); + Ok(()) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn public_urls_encode_keys_and_preserve_prefix() { + let store = S3Store { + bucket: Bucket::new( + "test", + Region::Custom { + region: "test".into(), + endpoint: "https://s3.example.com".into(), + }, + Credentials::anonymous().unwrap(), + ) + .unwrap(), + base_url: "https://cdn.example.com/prefix/".parse().unwrap(), + }; + assert_eq!( + store + .url_for("minecraft/v0/versions/3D Shareware v1.34.json") + .as_str(), + "https://cdn.example.com/prefix/minecraft/v0/versions/3D%20Shareware%20v1.34.json" + ); + assert_eq!( + store.url_for("maven/a+b/file.jar").as_str(), + "https://cdn.example.com/prefix/maven/a+b/file.jar" + ); + } +} diff --git a/apps/meta-generator/src/util/error.rs b/apps/meta-generator/src/util/error.rs new file mode 100644 index 00000000000..a5451b736b9 --- /dev/null +++ b/apps/meta-generator/src/util/error.rs @@ -0,0 +1,32 @@ +use std::sync::Mutex; + +use tracing::warn; + +#[derive(Debug)] +pub struct ErrorAccumulator { + imp: Mutex>, +} + +impl ErrorAccumulator { + #[must_use] + pub const fn new() -> Self { + Self { + imp: Mutex::new(Vec::new()), + } + } + + pub fn push(&self, err: &anyhow::Error) { + warn!("error: {err:?}"); + let err = format!("{err:#}"); + let mut imp = self.imp.lock().expect("should not be poisoned"); + imp.push(err); + } + + pub fn len(&self) -> usize { + self.imp.lock().expect("should not be poisoned").len() + } + + pub fn finish(self) -> Vec { + self.imp.into_inner().expect("should not be poisoned") + } +} diff --git a/apps/meta-generator/src/util/fastly.rs b/apps/meta-generator/src/util/fastly.rs new file mode 100644 index 00000000000..ef639e2337c --- /dev/null +++ b/apps/meta-generator/src/util/fastly.rs @@ -0,0 +1,50 @@ +use anyhow::{Context, Result}; +use derive_more::Debug; +use reqwest::{IntoUrl, header::HeaderValue}; +use secrecy::{ExposeSecret, SecretString}; +use url::Url; + +use crate::util::ResponseExt; + +#[derive(Debug)] +pub struct Client { + http: reqwest::Client, + base: Url, + fastly_key: HeaderValue, +} + +impl Client { + pub fn new( + http: reqwest::Client, + base: impl IntoUrl, + fastly_key: SecretString, + ) -> Result { + let base = base.into_url().context("invalid base URL")?; + let mut fastly_key = HeaderValue::from_str(fastly_key.expose_secret()) + .context("invalid Fastly key")?; + fastly_key.set_sensitive(true); + Ok(Self { + http, + base, + fastly_key, + }) + } + + pub async fn purge_url(&self, cached_url: &Url) -> Result<()> { + let mut endpoint = self.base.join("purge/")?; + endpoint + .path_segments_mut() + .expect("API URL supports path segments") + .pop_if_empty() + .push(cached_url.as_str()); + + self.http + .post(endpoint) + .header("Fastly-Key", &self.fastly_key) + .send() + .await? + .error_for_status_ext() + .await?; + Ok(()) + } +} diff --git a/apps/meta-generator/src/util/http.rs b/apps/meta-generator/src/util/http.rs new file mode 100644 index 00000000000..30f676d4867 --- /dev/null +++ b/apps/meta-generator/src/util/http.rs @@ -0,0 +1,24 @@ +use anyhow::{Result, anyhow}; +use reqwest::Response; + +pub trait ResponseExt: Sized { + async fn error_for_status_ext(self) -> Result; +} + +impl ResponseExt for Response { + async fn error_for_status_ext(self) -> Result { + let status = self.status(); + if status.is_client_error() || status.is_server_error() { + let text = self + .text() + .await + .unwrap_or_else(|_| "(non-UTF-8 response)".to_string()); + Err(anyhow!( + "HTTP status client error ({}): {text}", + status.as_u16() + )) + } else { + Ok(self) + } + } +} diff --git a/apps/meta-generator/src/util/mod.rs b/apps/meta-generator/src/util/mod.rs new file mode 100644 index 00000000000..f25e0cdccce --- /dev/null +++ b/apps/meta-generator/src/util/mod.rs @@ -0,0 +1,7 @@ +mod error; +pub mod fastly; +mod http; + +pub use error::*; +pub use http::*; +pub use modrinth_minecraft_meta::util::*; diff --git a/apps/meta-generator/toasty/history.toml b/apps/meta-generator/toasty/history.toml new file mode 100644 index 00000000000..c5c0470bb26 --- /dev/null +++ b/apps/meta-generator/toasty/history.toml @@ -0,0 +1,16 @@ +version = 1 + +[[migrations]] +id = 3279592192024147812 +name = "20261007_111854_migration.sql" +snapshot_name = "20261007_111854_snapshot.toml" + +[[migrations]] +id = 4713685892546258052 +name = "20261009_104246_migration.sql" +snapshot_name = "20261009_104246_snapshot.toml" + +[[migrations]] +id = 5673943660847592223 +name = "20261009_155457_migration.sql" +snapshot_name = "20261009_155457_snapshot.toml" diff --git a/apps/meta-generator/toasty/migrations/20261007_111854_migration.sql b/apps/meta-generator/toasty/migrations/20261007_111854_migration.sql new file mode 100644 index 00000000000..c44a58f4ebb --- /dev/null +++ b/apps/meta-generator/toasty/migrations/20261007_111854_migration.sql @@ -0,0 +1,81 @@ +CREATE TYPE "forgelike_loader" AS ENUM ('forge', 'neoforge'); +CREATE TYPE "minecraft_loader" AS ENUM ('fabric', 'forge', 'neoforge', 'quilt'); +CREATE TABLE "forgelike_extracts" ( + "installer_sha256" TEXT NOT NULL, + "created_at" TIMESTAMPTZ(6) NOT NULL, + "metadata" JSON NOT NULL, + "embedded_maven_artifacts" JSON NOT NULL, + PRIMARY KEY ("installer_sha256") +); +CREATE TABLE "blob_hashes" ( + "sha256" TEXT NOT NULL, + "sha1" TEXT NOT NULL, + PRIMARY KEY ("sha256") +); +CREATE INDEX "index_blob_hashes_by_sha1" ON "blob_hashes" ("sha1"); +CREATE TABLE "quilt_catalogs" ( + "id" UUID NOT NULL, + "download_run_id" UUID NOT NULL, + "sha256" TEXT NOT NULL, + PRIMARY KEY ("id") +); +CREATE UNIQUE INDEX "index_quilt_catalogs_by_download_run_id" ON "quilt_catalogs" ("download_run_id"); +CREATE TABLE "mojang_catalogs" ( + "id" UUID NOT NULL, + "download_run_id" UUID NOT NULL, + "sha256" TEXT NOT NULL, + PRIMARY KEY ("id") +); +CREATE UNIQUE INDEX "index_mojang_catalogs_by_download_run_id" ON "mojang_catalogs" ("download_run_id"); +CREATE TABLE "download_runs" ( + "id" UUID NOT NULL, + "started_at" TIMESTAMPTZ(6) NOT NULL, + "completed_at" TIMESTAMPTZ(6), + "errors" TEXT, + PRIMARY KEY ("id") +); +CREATE TABLE "forgelike_installers" ( + "loader" "forgelike_loader" NOT NULL, + "name" TEXT NOT NULL, + "download_run_id" UUID NOT NULL, + "sha256" TEXT NOT NULL, + PRIMARY KEY ("loader", "name") +); +CREATE INDEX "index_forgelike_installers_by_sha256" ON "forgelike_installers" ("sha256"); +CREATE TABLE "fabric_catalogs" ( + "id" UUID NOT NULL, + "download_run_id" UUID NOT NULL, + "sha256" TEXT NOT NULL, + PRIMARY KEY ("id") +); +CREATE UNIQUE INDEX "index_fabric_catalogs_by_download_run_id" ON "fabric_catalogs" ("download_run_id"); +CREATE TABLE "profiles" ( + "loader" "minecraft_loader" NOT NULL, + "minecraft_version" TEXT NOT NULL, + "loader_version" TEXT NOT NULL, + "metadata" JSON NOT NULL, + PRIMARY KEY ("loader", "minecraft_version", "loader_version") +); +CREATE INDEX "index_profiles_by_minecraft_version" ON "profiles" ("minecraft_version"); +CREATE TABLE "neoforge_catalogs" ( + "id" UUID NOT NULL, + "download_run_id" UUID NOT NULL, + "forge_sha256" TEXT NOT NULL, + "neoforge_sha256" TEXT NOT NULL, + PRIMARY KEY ("id") +); +CREATE UNIQUE INDEX "index_neoforge_catalogs_by_download_run_id" ON "neoforge_catalogs" ("download_run_id"); +CREATE TABLE "blob_downloads" ( + "download_run_id" UUID NOT NULL, + "url" TEXT NOT NULL, + "sha256" TEXT NOT NULL, + PRIMARY KEY ("download_run_id", "url") +); +CREATE INDEX "index_blob_downloads_by_sha256" ON "blob_downloads" ("sha256"); +CREATE TABLE "forge_catalogs" ( + "id" UUID NOT NULL, + "download_run_id" UUID NOT NULL, + "sha256" TEXT NOT NULL, + PRIMARY KEY ("id") +); +CREATE UNIQUE INDEX "index_forge_catalogs_by_download_run_id" ON "forge_catalogs" ("download_run_id"); diff --git a/apps/meta-generator/toasty/migrations/20261009_104246_migration.sql b/apps/meta-generator/toasty/migrations/20261009_104246_migration.sql new file mode 100644 index 00000000000..9abc465de50 --- /dev/null +++ b/apps/meta-generator/toasty/migrations/20261009_104246_migration.sql @@ -0,0 +1,12 @@ +CREATE TABLE "minecraft_versions" ( + "name" TEXT NOT NULL, + "catalog_entry" JSON NOT NULL, + "manifest" JSON NOT NULL, + "source_catalog_sha256" TEXT NOT NULL, + "source_manifest_sha256" TEXT NOT NULL, + "is_latest_release" BOOLEAN, + "is_latest_snapshot" BOOLEAN, + PRIMARY KEY ("name") +); +CREATE UNIQUE INDEX "index_minecraft_versions_by_is_latest_release" ON "minecraft_versions" ("is_latest_release"); +CREATE UNIQUE INDEX "index_minecraft_versions_by_is_latest_snapshot" ON "minecraft_versions" ("is_latest_snapshot"); diff --git a/apps/meta-generator/toasty/migrations/20261009_155457_migration.sql b/apps/meta-generator/toasty/migrations/20261009_155457_migration.sql new file mode 100644 index 00000000000..9b93168f00d --- /dev/null +++ b/apps/meta-generator/toasty/migrations/20261009_155457_migration.sql @@ -0,0 +1,20 @@ +CREATE TYPE "fabriclike_loader" AS ENUM ('fabric', 'quilt'); +CREATE TABLE "fabriclike_profiles" ( + "loader" "fabriclike_loader" NOT NULL, + "version_group" TEXT NOT NULL, + "loader_version" TEXT NOT NULL, + "stable" BOOLEAN NOT NULL, + "position" INTEGER NOT NULL, + "source_url" TEXT NOT NULL, + "source_sha256" TEXT NOT NULL, + "template" TEXT NOT NULL, + "metadata" JSON NOT NULL, + PRIMARY KEY ("loader", "version_group", "loader_version") +); +CREATE TABLE "fabriclike_catalogs" ( + "loader" "fabriclike_loader" NOT NULL, + "games" JSON NOT NULL, + "groups" JSON NOT NULL, + "artifacts" JSON NOT NULL, + PRIMARY KEY ("loader") +); diff --git a/apps/meta-generator/toasty/snapshots/20261007_111854_snapshot.toml b/apps/meta-generator/toasty/snapshots/20261007_111854_snapshot.toml new file mode 100644 index 00000000000..a5fd53e2bc3 --- /dev/null +++ b/apps/meta-generator/toasty/snapshots/20261007_111854_snapshot.toml @@ -0,0 +1,428 @@ +version = 1 + +[schema] +[[schema.tables]] +id = 0 +name = "quilt_catalogs" +primary_key = { columns = [{ table = 0, index = 0 }], index = { table = 0, index = 0 } } + +[[schema.tables.columns]] +id = { table = 0, index = 0 } +name = "id" +ty = "Uuid" +storage_ty = "Uuid" + +[[schema.tables.columns]] +id = { table = 0, index = 1 } +name = "download_run_id" +ty = "Uuid" +storage_ty = "Uuid" + +[[schema.tables.columns]] +id = { table = 0, index = 2 } +name = "sha256" +ty = "String" +storage_ty = "Text" + +[[schema.tables.indices]] +id = { table = 0, index = 0 } +name = "index_quilt_catalogs_by_id" +on = 0 +columns = [{ column = { table = 0, index = 0 }, op = "Eq", scope = "Partition" }] +unique = true +primary_key = true + +[[schema.tables.indices]] +id = { table = 0, index = 1 } +name = "index_quilt_catalogs_by_download_run_id" +on = 0 +columns = [{ column = { table = 0, index = 1 }, op = "Eq", scope = "Partition" }] +unique = true + +[[schema.tables]] +id = 1 +name = "download_runs" +primary_key = { columns = [{ table = 1, index = 0 }], index = { table = 1, index = 0 } } + +[[schema.tables.columns]] +id = { table = 1, index = 0 } +name = "id" +ty = "Uuid" +storage_ty = "Uuid" + +[[schema.tables.columns]] +id = { table = 1, index = 1 } +name = "started_at" +ty = "Timestamp" +storage_ty = { Timestamp = 6 } + +[[schema.tables.columns]] +id = { table = 1, index = 2 } +name = "completed_at" +ty = "Timestamp" +storage_ty = { Timestamp = 6 } +nullable = true + +[[schema.tables.columns]] +id = { table = 1, index = 3 } +name = "errors" +ty = "String" +storage_ty = "Text" +nullable = true + +[[schema.tables.indices]] +id = { table = 1, index = 0 } +name = "index_download_runs_by_id" +on = 1 +columns = [{ column = { table = 1, index = 0 }, op = "Eq", scope = "Partition" }] +unique = true +primary_key = true + +[[schema.tables]] +id = 2 +name = "fabric_catalogs" +primary_key = { columns = [{ table = 2, index = 0 }], index = { table = 2, index = 0 } } + +[[schema.tables.columns]] +id = { table = 2, index = 0 } +name = "id" +ty = "Uuid" +storage_ty = "Uuid" + +[[schema.tables.columns]] +id = { table = 2, index = 1 } +name = "download_run_id" +ty = "Uuid" +storage_ty = "Uuid" + +[[schema.tables.columns]] +id = { table = 2, index = 2 } +name = "sha256" +ty = "String" +storage_ty = "Text" + +[[schema.tables.indices]] +id = { table = 2, index = 0 } +name = "index_fabric_catalogs_by_id" +on = 2 +columns = [{ column = { table = 2, index = 0 }, op = "Eq", scope = "Partition" }] +unique = true +primary_key = true + +[[schema.tables.indices]] +id = { table = 2, index = 1 } +name = "index_fabric_catalogs_by_download_run_id" +on = 2 +columns = [{ column = { table = 2, index = 1 }, op = "Eq", scope = "Partition" }] +unique = true + +[[schema.tables]] +id = 3 +name = "neoforge_catalogs" +primary_key = { columns = [{ table = 3, index = 0 }], index = { table = 3, index = 0 } } + +[[schema.tables.columns]] +id = { table = 3, index = 0 } +name = "id" +ty = "Uuid" +storage_ty = "Uuid" + +[[schema.tables.columns]] +id = { table = 3, index = 1 } +name = "download_run_id" +ty = "Uuid" +storage_ty = "Uuid" + +[[schema.tables.columns]] +id = { table = 3, index = 2 } +name = "forge_sha256" +ty = "String" +storage_ty = "Text" + +[[schema.tables.columns]] +id = { table = 3, index = 3 } +name = "neoforge_sha256" +ty = "String" +storage_ty = "Text" + +[[schema.tables.indices]] +id = { table = 3, index = 0 } +name = "index_neoforge_catalogs_by_id" +on = 3 +columns = [{ column = { table = 3, index = 0 }, op = "Eq", scope = "Partition" }] +unique = true +primary_key = true + +[[schema.tables.indices]] +id = { table = 3, index = 1 } +name = "index_neoforge_catalogs_by_download_run_id" +on = 3 +columns = [{ column = { table = 3, index = 1 }, op = "Eq", scope = "Partition" }] +unique = true + +[[schema.tables]] +id = 4 +name = "blob_hashes" +primary_key = { columns = [{ table = 4, index = 0 }], index = { table = 4, index = 0 } } + +[[schema.tables.columns]] +id = { table = 4, index = 0 } +name = "sha256" +ty = "String" +storage_ty = "Text" + +[[schema.tables.columns]] +id = { table = 4, index = 1 } +name = "sha1" +ty = "String" +storage_ty = "Text" + +[[schema.tables.indices]] +id = { table = 4, index = 0 } +name = "index_blob_hashes_by_sha256" +on = 4 +columns = [{ column = { table = 4, index = 0 }, op = "Eq", scope = "Partition" }] +unique = true +primary_key = true + +[[schema.tables.indices]] +id = { table = 4, index = 1 } +name = "index_blob_hashes_by_sha1" +on = 4 +columns = [{ column = { table = 4, index = 1 }, op = "Eq", scope = "Partition" }] + +[[schema.tables]] +id = 5 +name = "blob_downloads" +primary_key = { columns = [{ table = 5, index = 0 }, { table = 5, index = 1 }], index = { table = 5, index = 0 } } + +[[schema.tables.columns]] +id = { table = 5, index = 0 } +name = "download_run_id" +ty = "Uuid" +storage_ty = "Uuid" + +[[schema.tables.columns]] +id = { table = 5, index = 1 } +name = "url" +ty = "String" +storage_ty = "Text" + +[[schema.tables.columns]] +id = { table = 5, index = 2 } +name = "sha256" +ty = "String" +storage_ty = "Text" + +[[schema.tables.indices]] +id = { table = 5, index = 0 } +name = "index_blob_downloads_by_download_run_id_and_url" +on = 5 +columns = [{ column = { table = 5, index = 0 }, op = "Eq", scope = "Partition" }, { column = { table = 5, index = 1 }, op = "Eq", scope = "Local" }] +unique = true +primary_key = true + +[[schema.tables.indices]] +id = { table = 5, index = 1 } +name = "index_blob_downloads_by_sha256" +on = 5 +columns = [{ column = { table = 5, index = 2 }, op = "Eq", scope = "Partition" }] + +[[schema.tables]] +id = 6 +name = "forge_catalogs" +primary_key = { columns = [{ table = 6, index = 0 }], index = { table = 6, index = 0 } } + +[[schema.tables.columns]] +id = { table = 6, index = 0 } +name = "id" +ty = "Uuid" +storage_ty = "Uuid" + +[[schema.tables.columns]] +id = { table = 6, index = 1 } +name = "download_run_id" +ty = "Uuid" +storage_ty = "Uuid" + +[[schema.tables.columns]] +id = { table = 6, index = 2 } +name = "sha256" +ty = "String" +storage_ty = "Text" + +[[schema.tables.indices]] +id = { table = 6, index = 0 } +name = "index_forge_catalogs_by_id" +on = 6 +columns = [{ column = { table = 6, index = 0 }, op = "Eq", scope = "Partition" }] +unique = true +primary_key = true + +[[schema.tables.indices]] +id = { table = 6, index = 1 } +name = "index_forge_catalogs_by_download_run_id" +on = 6 +columns = [{ column = { table = 6, index = 1 }, op = "Eq", scope = "Partition" }] +unique = true + +[[schema.tables]] +id = 7 +name = "forgelike_installers" +primary_key = { columns = [{ table = 7, index = 0 }, { table = 7, index = 1 }], index = { table = 7, index = 0 } } + +[[schema.tables.columns]] +id = { table = 7, index = 0 } +name = "loader" +ty = "String" +storage_ty = { Enum = { name = "forgelike_loader", variants = [{ name = "forge" }, { name = "neoforge" }] } } + +[[schema.tables.columns]] +id = { table = 7, index = 1 } +name = "name" +ty = "String" +storage_ty = "Text" + +[[schema.tables.columns]] +id = { table = 7, index = 2 } +name = "download_run_id" +ty = "Uuid" +storage_ty = "Uuid" + +[[schema.tables.columns]] +id = { table = 7, index = 3 } +name = "sha256" +ty = "String" +storage_ty = "Text" + +[[schema.tables.indices]] +id = { table = 7, index = 0 } +name = "index_forgelike_installers_by_loader_and_name" +on = 7 +columns = [{ column = { table = 7, index = 0 }, op = "Eq", scope = "Partition" }, { column = { table = 7, index = 1 }, op = "Eq", scope = "Local" }] +unique = true +primary_key = true + +[[schema.tables.indices]] +id = { table = 7, index = 1 } +name = "index_forgelike_installers_by_sha256" +on = 7 +columns = [{ column = { table = 7, index = 3 }, op = "Eq", scope = "Partition" }] + +[[schema.tables]] +id = 8 +name = "forgelike_extracts" +primary_key = { columns = [{ table = 8, index = 0 }], index = { table = 8, index = 0 } } + +[[schema.tables.columns]] +id = { table = 8, index = 0 } +name = "installer_sha256" +ty = "String" +storage_ty = "Text" + +[[schema.tables.columns]] +id = { table = 8, index = 1 } +name = "created_at" +ty = "Timestamp" +storage_ty = { Timestamp = 6 } + +[[schema.tables.columns]] +id = { table = 8, index = 2 } +name = "metadata" +ty = "String" +storage_ty = "Json" + +[[schema.tables.columns]] +id = { table = 8, index = 3 } +name = "embedded_maven_artifacts" +ty = "String" +storage_ty = "Json" + +[[schema.tables.indices]] +id = { table = 8, index = 0 } +name = "index_forgelike_extracts_by_installer_sha256" +on = 8 +columns = [{ column = { table = 8, index = 0 }, op = "Eq", scope = "Partition" }] +unique = true +primary_key = true + +[[schema.tables]] +id = 9 +name = "mojang_catalogs" +primary_key = { columns = [{ table = 9, index = 0 }], index = { table = 9, index = 0 } } + +[[schema.tables.columns]] +id = { table = 9, index = 0 } +name = "id" +ty = "Uuid" +storage_ty = "Uuid" + +[[schema.tables.columns]] +id = { table = 9, index = 1 } +name = "download_run_id" +ty = "Uuid" +storage_ty = "Uuid" + +[[schema.tables.columns]] +id = { table = 9, index = 2 } +name = "sha256" +ty = "String" +storage_ty = "Text" + +[[schema.tables.indices]] +id = { table = 9, index = 0 } +name = "index_mojang_catalogs_by_id" +on = 9 +columns = [{ column = { table = 9, index = 0 }, op = "Eq", scope = "Partition" }] +unique = true +primary_key = true + +[[schema.tables.indices]] +id = { table = 9, index = 1 } +name = "index_mojang_catalogs_by_download_run_id" +on = 9 +columns = [{ column = { table = 9, index = 1 }, op = "Eq", scope = "Partition" }] +unique = true + +[[schema.tables]] +id = 10 +name = "profiles" +primary_key = { columns = [{ table = 10, index = 0 }, { table = 10, index = 1 }, { table = 10, index = 2 }], index = { table = 10, index = 0 } } + +[[schema.tables.columns]] +id = { table = 10, index = 0 } +name = "loader" +ty = "String" +storage_ty = { Enum = { name = "minecraft_loader", variants = [{ name = "fabric" }, { name = "forge" }, { name = "neoforge" }, { name = "quilt" }] } } + +[[schema.tables.columns]] +id = { table = 10, index = 1 } +name = "minecraft_version" +ty = "String" +storage_ty = "Text" + +[[schema.tables.columns]] +id = { table = 10, index = 2 } +name = "loader_version" +ty = "String" +storage_ty = "Text" + +[[schema.tables.columns]] +id = { table = 10, index = 3 } +name = "metadata" +ty = "String" +storage_ty = "Json" + +[[schema.tables.indices]] +id = { table = 10, index = 0 } +name = "index_profiles_by_loader_and_minecraft_version_and_loader__cb6a" +on = 10 +columns = [{ column = { table = 10, index = 0 }, op = "Eq", scope = "Partition" }, { column = { table = 10, index = 1 }, op = "Eq", scope = "Local" }, { column = { table = 10, index = 2 }, op = "Eq", scope = "Local" }] +unique = true +primary_key = true + +[[schema.tables.indices]] +id = { table = 10, index = 1 } +name = "index_profiles_by_minecraft_version" +on = 10 +columns = [{ column = { table = 10, index = 1 }, op = "Eq", scope = "Partition" }] diff --git a/apps/meta-generator/toasty/snapshots/20261009_104246_snapshot.toml b/apps/meta-generator/toasty/snapshots/20261009_104246_snapshot.toml new file mode 100644 index 00000000000..1dbf3566dc3 --- /dev/null +++ b/apps/meta-generator/toasty/snapshots/20261009_104246_snapshot.toml @@ -0,0 +1,499 @@ +version = 1 + +[schema] +[[schema.tables]] +id = 0 +name = "neoforge_catalogs" +primary_key = { columns = [{ table = 0, index = 0 }], index = { table = 0, index = 0 } } + +[[schema.tables.columns]] +id = { table = 0, index = 0 } +name = "id" +ty = "Uuid" +storage_ty = "Uuid" + +[[schema.tables.columns]] +id = { table = 0, index = 1 } +name = "download_run_id" +ty = "Uuid" +storage_ty = "Uuid" + +[[schema.tables.columns]] +id = { table = 0, index = 2 } +name = "forge_sha256" +ty = "String" +storage_ty = "Text" + +[[schema.tables.columns]] +id = { table = 0, index = 3 } +name = "neoforge_sha256" +ty = "String" +storage_ty = "Text" + +[[schema.tables.indices]] +id = { table = 0, index = 0 } +name = "index_neoforge_catalogs_by_id" +on = 0 +columns = [{ column = { table = 0, index = 0 }, op = "Eq", scope = "Partition" }] +unique = true +primary_key = true + +[[schema.tables.indices]] +id = { table = 0, index = 1 } +name = "index_neoforge_catalogs_by_download_run_id" +on = 0 +columns = [{ column = { table = 0, index = 1 }, op = "Eq", scope = "Partition" }] +unique = true + +[[schema.tables]] +id = 1 +name = "download_runs" +primary_key = { columns = [{ table = 1, index = 0 }], index = { table = 1, index = 0 } } + +[[schema.tables.columns]] +id = { table = 1, index = 0 } +name = "id" +ty = "Uuid" +storage_ty = "Uuid" + +[[schema.tables.columns]] +id = { table = 1, index = 1 } +name = "started_at" +ty = "Timestamp" +storage_ty = { Timestamp = 6 } + +[[schema.tables.columns]] +id = { table = 1, index = 2 } +name = "completed_at" +ty = "Timestamp" +storage_ty = { Timestamp = 6 } +nullable = true + +[[schema.tables.columns]] +id = { table = 1, index = 3 } +name = "errors" +ty = "String" +storage_ty = "Text" +nullable = true + +[[schema.tables.indices]] +id = { table = 1, index = 0 } +name = "index_download_runs_by_id" +on = 1 +columns = [{ column = { table = 1, index = 0 }, op = "Eq", scope = "Partition" }] +unique = true +primary_key = true + +[[schema.tables]] +id = 2 +name = "mojang_catalogs" +primary_key = { columns = [{ table = 2, index = 0 }], index = { table = 2, index = 0 } } + +[[schema.tables.columns]] +id = { table = 2, index = 0 } +name = "id" +ty = "Uuid" +storage_ty = "Uuid" + +[[schema.tables.columns]] +id = { table = 2, index = 1 } +name = "download_run_id" +ty = "Uuid" +storage_ty = "Uuid" + +[[schema.tables.columns]] +id = { table = 2, index = 2 } +name = "sha256" +ty = "String" +storage_ty = "Text" + +[[schema.tables.indices]] +id = { table = 2, index = 0 } +name = "index_mojang_catalogs_by_id" +on = 2 +columns = [{ column = { table = 2, index = 0 }, op = "Eq", scope = "Partition" }] +unique = true +primary_key = true + +[[schema.tables.indices]] +id = { table = 2, index = 1 } +name = "index_mojang_catalogs_by_download_run_id" +on = 2 +columns = [{ column = { table = 2, index = 1 }, op = "Eq", scope = "Partition" }] +unique = true + +[[schema.tables]] +id = 3 +name = "profiles" +primary_key = { columns = [{ table = 3, index = 0 }, { table = 3, index = 1 }, { table = 3, index = 2 }], index = { table = 3, index = 0 } } + +[[schema.tables.columns]] +id = { table = 3, index = 0 } +name = "loader" +ty = "String" +storage_ty = { Enum = { name = "minecraft_loader", variants = [{ name = "fabric" }, { name = "forge" }, { name = "neoforge" }, { name = "quilt" }] } } + +[[schema.tables.columns]] +id = { table = 3, index = 1 } +name = "minecraft_version" +ty = "String" +storage_ty = "Text" + +[[schema.tables.columns]] +id = { table = 3, index = 2 } +name = "loader_version" +ty = "String" +storage_ty = "Text" + +[[schema.tables.columns]] +id = { table = 3, index = 3 } +name = "metadata" +ty = "String" +storage_ty = "Json" + +[[schema.tables.indices]] +id = { table = 3, index = 0 } +name = "index_profiles_by_loader_and_minecraft_version_and_loader__cb6a" +on = 3 +columns = [{ column = { table = 3, index = 0 }, op = "Eq", scope = "Partition" }, { column = { table = 3, index = 1 }, op = "Eq", scope = "Local" }, { column = { table = 3, index = 2 }, op = "Eq", scope = "Local" }] +unique = true +primary_key = true + +[[schema.tables.indices]] +id = { table = 3, index = 1 } +name = "index_profiles_by_minecraft_version" +on = 3 +columns = [{ column = { table = 3, index = 1 }, op = "Eq", scope = "Partition" }] + +[[schema.tables]] +id = 4 +name = "blob_hashes" +primary_key = { columns = [{ table = 4, index = 0 }], index = { table = 4, index = 0 } } + +[[schema.tables.columns]] +id = { table = 4, index = 0 } +name = "sha256" +ty = "String" +storage_ty = "Text" + +[[schema.tables.columns]] +id = { table = 4, index = 1 } +name = "sha1" +ty = "String" +storage_ty = "Text" + +[[schema.tables.indices]] +id = { table = 4, index = 0 } +name = "index_blob_hashes_by_sha256" +on = 4 +columns = [{ column = { table = 4, index = 0 }, op = "Eq", scope = "Partition" }] +unique = true +primary_key = true + +[[schema.tables.indices]] +id = { table = 4, index = 1 } +name = "index_blob_hashes_by_sha1" +on = 4 +columns = [{ column = { table = 4, index = 1 }, op = "Eq", scope = "Partition" }] + +[[schema.tables]] +id = 5 +name = "blob_downloads" +primary_key = { columns = [{ table = 5, index = 0 }, { table = 5, index = 1 }], index = { table = 5, index = 0 } } + +[[schema.tables.columns]] +id = { table = 5, index = 0 } +name = "download_run_id" +ty = "Uuid" +storage_ty = "Uuid" + +[[schema.tables.columns]] +id = { table = 5, index = 1 } +name = "url" +ty = "String" +storage_ty = "Text" + +[[schema.tables.columns]] +id = { table = 5, index = 2 } +name = "sha256" +ty = "String" +storage_ty = "Text" + +[[schema.tables.indices]] +id = { table = 5, index = 0 } +name = "index_blob_downloads_by_download_run_id_and_url" +on = 5 +columns = [{ column = { table = 5, index = 0 }, op = "Eq", scope = "Partition" }, { column = { table = 5, index = 1 }, op = "Eq", scope = "Local" }] +unique = true +primary_key = true + +[[schema.tables.indices]] +id = { table = 5, index = 1 } +name = "index_blob_downloads_by_sha256" +on = 5 +columns = [{ column = { table = 5, index = 2 }, op = "Eq", scope = "Partition" }] + +[[schema.tables]] +id = 6 +name = "fabric_catalogs" +primary_key = { columns = [{ table = 6, index = 0 }], index = { table = 6, index = 0 } } + +[[schema.tables.columns]] +id = { table = 6, index = 0 } +name = "id" +ty = "Uuid" +storage_ty = "Uuid" + +[[schema.tables.columns]] +id = { table = 6, index = 1 } +name = "download_run_id" +ty = "Uuid" +storage_ty = "Uuid" + +[[schema.tables.columns]] +id = { table = 6, index = 2 } +name = "sha256" +ty = "String" +storage_ty = "Text" + +[[schema.tables.indices]] +id = { table = 6, index = 0 } +name = "index_fabric_catalogs_by_id" +on = 6 +columns = [{ column = { table = 6, index = 0 }, op = "Eq", scope = "Partition" }] +unique = true +primary_key = true + +[[schema.tables.indices]] +id = { table = 6, index = 1 } +name = "index_fabric_catalogs_by_download_run_id" +on = 6 +columns = [{ column = { table = 6, index = 1 }, op = "Eq", scope = "Partition" }] +unique = true + +[[schema.tables]] +id = 7 +name = "quilt_catalogs" +primary_key = { columns = [{ table = 7, index = 0 }], index = { table = 7, index = 0 } } + +[[schema.tables.columns]] +id = { table = 7, index = 0 } +name = "id" +ty = "Uuid" +storage_ty = "Uuid" + +[[schema.tables.columns]] +id = { table = 7, index = 1 } +name = "download_run_id" +ty = "Uuid" +storage_ty = "Uuid" + +[[schema.tables.columns]] +id = { table = 7, index = 2 } +name = "sha256" +ty = "String" +storage_ty = "Text" + +[[schema.tables.indices]] +id = { table = 7, index = 0 } +name = "index_quilt_catalogs_by_id" +on = 7 +columns = [{ column = { table = 7, index = 0 }, op = "Eq", scope = "Partition" }] +unique = true +primary_key = true + +[[schema.tables.indices]] +id = { table = 7, index = 1 } +name = "index_quilt_catalogs_by_download_run_id" +on = 7 +columns = [{ column = { table = 7, index = 1 }, op = "Eq", scope = "Partition" }] +unique = true + +[[schema.tables]] +id = 8 +name = "forgelike_installers" +primary_key = { columns = [{ table = 8, index = 0 }, { table = 8, index = 1 }], index = { table = 8, index = 0 } } + +[[schema.tables.columns]] +id = { table = 8, index = 0 } +name = "loader" +ty = "String" +storage_ty = { Enum = { name = "forgelike_loader", variants = [{ name = "forge" }, { name = "neoforge" }] } } + +[[schema.tables.columns]] +id = { table = 8, index = 1 } +name = "name" +ty = "String" +storage_ty = "Text" + +[[schema.tables.columns]] +id = { table = 8, index = 2 } +name = "download_run_id" +ty = "Uuid" +storage_ty = "Uuid" + +[[schema.tables.columns]] +id = { table = 8, index = 3 } +name = "sha256" +ty = "String" +storage_ty = "Text" + +[[schema.tables.indices]] +id = { table = 8, index = 0 } +name = "index_forgelike_installers_by_loader_and_name" +on = 8 +columns = [{ column = { table = 8, index = 0 }, op = "Eq", scope = "Partition" }, { column = { table = 8, index = 1 }, op = "Eq", scope = "Local" }] +unique = true +primary_key = true + +[[schema.tables.indices]] +id = { table = 8, index = 1 } +name = "index_forgelike_installers_by_sha256" +on = 8 +columns = [{ column = { table = 8, index = 3 }, op = "Eq", scope = "Partition" }] + +[[schema.tables]] +id = 9 +name = "forgelike_extracts" +primary_key = { columns = [{ table = 9, index = 0 }], index = { table = 9, index = 0 } } + +[[schema.tables.columns]] +id = { table = 9, index = 0 } +name = "installer_sha256" +ty = "String" +storage_ty = "Text" + +[[schema.tables.columns]] +id = { table = 9, index = 1 } +name = "created_at" +ty = "Timestamp" +storage_ty = { Timestamp = 6 } + +[[schema.tables.columns]] +id = { table = 9, index = 2 } +name = "metadata" +ty = "String" +storage_ty = "Json" + +[[schema.tables.columns]] +id = { table = 9, index = 3 } +name = "embedded_maven_artifacts" +ty = "String" +storage_ty = "Json" + +[[schema.tables.indices]] +id = { table = 9, index = 0 } +name = "index_forgelike_extracts_by_installer_sha256" +on = 9 +columns = [{ column = { table = 9, index = 0 }, op = "Eq", scope = "Partition" }] +unique = true +primary_key = true + +[[schema.tables]] +id = 10 +name = "forge_catalogs" +primary_key = { columns = [{ table = 10, index = 0 }], index = { table = 10, index = 0 } } + +[[schema.tables.columns]] +id = { table = 10, index = 0 } +name = "id" +ty = "Uuid" +storage_ty = "Uuid" + +[[schema.tables.columns]] +id = { table = 10, index = 1 } +name = "download_run_id" +ty = "Uuid" +storage_ty = "Uuid" + +[[schema.tables.columns]] +id = { table = 10, index = 2 } +name = "sha256" +ty = "String" +storage_ty = "Text" + +[[schema.tables.indices]] +id = { table = 10, index = 0 } +name = "index_forge_catalogs_by_id" +on = 10 +columns = [{ column = { table = 10, index = 0 }, op = "Eq", scope = "Partition" }] +unique = true +primary_key = true + +[[schema.tables.indices]] +id = { table = 10, index = 1 } +name = "index_forge_catalogs_by_download_run_id" +on = 10 +columns = [{ column = { table = 10, index = 1 }, op = "Eq", scope = "Partition" }] +unique = true + +[[schema.tables]] +id = 11 +name = "minecraft_versions" +primary_key = { columns = [{ table = 11, index = 0 }], index = { table = 11, index = 0 } } + +[[schema.tables.columns]] +id = { table = 11, index = 0 } +name = "name" +ty = "String" +storage_ty = "Text" + +[[schema.tables.columns]] +id = { table = 11, index = 1 } +name = "catalog_entry" +ty = "String" +storage_ty = "Json" + +[[schema.tables.columns]] +id = { table = 11, index = 2 } +name = "manifest" +ty = "String" +storage_ty = "Json" + +[[schema.tables.columns]] +id = { table = 11, index = 3 } +name = "source_catalog_sha256" +ty = "String" +storage_ty = "Text" + +[[schema.tables.columns]] +id = { table = 11, index = 4 } +name = "source_manifest_sha256" +ty = "String" +storage_ty = "Text" + +[[schema.tables.columns]] +id = { table = 11, index = 5 } +name = "is_latest_release" +ty = "Bool" +storage_ty = "Boolean" +nullable = true + +[[schema.tables.columns]] +id = { table = 11, index = 6 } +name = "is_latest_snapshot" +ty = "Bool" +storage_ty = "Boolean" +nullable = true + +[[schema.tables.indices]] +id = { table = 11, index = 0 } +name = "index_minecraft_versions_by_name" +on = 11 +columns = [{ column = { table = 11, index = 0 }, op = "Eq", scope = "Partition" }] +unique = true +primary_key = true + +[[schema.tables.indices]] +id = { table = 11, index = 1 } +name = "index_minecraft_versions_by_is_latest_release" +on = 11 +columns = [{ column = { table = 11, index = 5 }, op = "Eq", scope = "Partition" }] +unique = true + +[[schema.tables.indices]] +id = { table = 11, index = 2 } +name = "index_minecraft_versions_by_is_latest_snapshot" +on = 11 +columns = [{ column = { table = 11, index = 6 }, op = "Eq", scope = "Partition" }] +unique = true diff --git a/apps/meta-generator/toasty/snapshots/20261009_155457_snapshot.toml b/apps/meta-generator/toasty/snapshots/20261009_155457_snapshot.toml new file mode 100644 index 00000000000..00ae0dd588a --- /dev/null +++ b/apps/meta-generator/toasty/snapshots/20261009_155457_snapshot.toml @@ -0,0 +1,603 @@ +version = 1 + +[schema] +[[schema.tables]] +id = 0 +name = "neoforge_catalogs" +primary_key = { columns = [{ table = 0, index = 0 }], index = { table = 0, index = 0 } } + +[[schema.tables.columns]] +id = { table = 0, index = 0 } +name = "id" +ty = "Uuid" +storage_ty = "Uuid" + +[[schema.tables.columns]] +id = { table = 0, index = 1 } +name = "download_run_id" +ty = "Uuid" +storage_ty = "Uuid" + +[[schema.tables.columns]] +id = { table = 0, index = 2 } +name = "forge_sha256" +ty = "String" +storage_ty = "Text" + +[[schema.tables.columns]] +id = { table = 0, index = 3 } +name = "neoforge_sha256" +ty = "String" +storage_ty = "Text" + +[[schema.tables.indices]] +id = { table = 0, index = 0 } +name = "index_neoforge_catalogs_by_id" +on = 0 +columns = [{ column = { table = 0, index = 0 }, op = "Eq", scope = "Partition" }] +unique = true +primary_key = true + +[[schema.tables.indices]] +id = { table = 0, index = 1 } +name = "index_neoforge_catalogs_by_download_run_id" +on = 0 +columns = [{ column = { table = 0, index = 1 }, op = "Eq", scope = "Partition" }] +unique = true + +[[schema.tables]] +id = 1 +name = "download_runs" +primary_key = { columns = [{ table = 1, index = 0 }], index = { table = 1, index = 0 } } + +[[schema.tables.columns]] +id = { table = 1, index = 0 } +name = "id" +ty = "Uuid" +storage_ty = "Uuid" + +[[schema.tables.columns]] +id = { table = 1, index = 1 } +name = "started_at" +ty = "Timestamp" +storage_ty = { Timestamp = 6 } + +[[schema.tables.columns]] +id = { table = 1, index = 2 } +name = "completed_at" +ty = "Timestamp" +storage_ty = { Timestamp = 6 } +nullable = true + +[[schema.tables.columns]] +id = { table = 1, index = 3 } +name = "errors" +ty = "String" +storage_ty = "Text" +nullable = true + +[[schema.tables.indices]] +id = { table = 1, index = 0 } +name = "index_download_runs_by_id" +on = 1 +columns = [{ column = { table = 1, index = 0 }, op = "Eq", scope = "Partition" }] +unique = true +primary_key = true + +[[schema.tables]] +id = 2 +name = "mojang_catalogs" +primary_key = { columns = [{ table = 2, index = 0 }], index = { table = 2, index = 0 } } + +[[schema.tables.columns]] +id = { table = 2, index = 0 } +name = "id" +ty = "Uuid" +storage_ty = "Uuid" + +[[schema.tables.columns]] +id = { table = 2, index = 1 } +name = "download_run_id" +ty = "Uuid" +storage_ty = "Uuid" + +[[schema.tables.columns]] +id = { table = 2, index = 2 } +name = "sha256" +ty = "String" +storage_ty = "Text" + +[[schema.tables.indices]] +id = { table = 2, index = 0 } +name = "index_mojang_catalogs_by_id" +on = 2 +columns = [{ column = { table = 2, index = 0 }, op = "Eq", scope = "Partition" }] +unique = true +primary_key = true + +[[schema.tables.indices]] +id = { table = 2, index = 1 } +name = "index_mojang_catalogs_by_download_run_id" +on = 2 +columns = [{ column = { table = 2, index = 1 }, op = "Eq", scope = "Partition" }] +unique = true + +[[schema.tables]] +id = 3 +name = "profiles" +primary_key = { columns = [{ table = 3, index = 0 }, { table = 3, index = 1 }, { table = 3, index = 2 }], index = { table = 3, index = 0 } } + +[[schema.tables.columns]] +id = { table = 3, index = 0 } +name = "loader" +ty = "String" +storage_ty = { Enum = { name = "minecraft_loader", variants = [{ name = "fabric" }, { name = "forge" }, { name = "neoforge" }, { name = "quilt" }] } } + +[[schema.tables.columns]] +id = { table = 3, index = 1 } +name = "minecraft_version" +ty = "String" +storage_ty = "Text" + +[[schema.tables.columns]] +id = { table = 3, index = 2 } +name = "loader_version" +ty = "String" +storage_ty = "Text" + +[[schema.tables.columns]] +id = { table = 3, index = 3 } +name = "metadata" +ty = "String" +storage_ty = "Json" + +[[schema.tables.indices]] +id = { table = 3, index = 0 } +name = "index_profiles_by_loader_and_minecraft_version_and_loader__cb6a" +on = 3 +columns = [{ column = { table = 3, index = 0 }, op = "Eq", scope = "Partition" }, { column = { table = 3, index = 1 }, op = "Eq", scope = "Local" }, { column = { table = 3, index = 2 }, op = "Eq", scope = "Local" }] +unique = true +primary_key = true + +[[schema.tables.indices]] +id = { table = 3, index = 1 } +name = "index_profiles_by_minecraft_version" +on = 3 +columns = [{ column = { table = 3, index = 1 }, op = "Eq", scope = "Partition" }] + +[[schema.tables]] +id = 4 +name = "blob_hashes" +primary_key = { columns = [{ table = 4, index = 0 }], index = { table = 4, index = 0 } } + +[[schema.tables.columns]] +id = { table = 4, index = 0 } +name = "sha256" +ty = "String" +storage_ty = "Text" + +[[schema.tables.columns]] +id = { table = 4, index = 1 } +name = "sha1" +ty = "String" +storage_ty = "Text" + +[[schema.tables.indices]] +id = { table = 4, index = 0 } +name = "index_blob_hashes_by_sha256" +on = 4 +columns = [{ column = { table = 4, index = 0 }, op = "Eq", scope = "Partition" }] +unique = true +primary_key = true + +[[schema.tables.indices]] +id = { table = 4, index = 1 } +name = "index_blob_hashes_by_sha1" +on = 4 +columns = [{ column = { table = 4, index = 1 }, op = "Eq", scope = "Partition" }] + +[[schema.tables]] +id = 5 +name = "blob_downloads" +primary_key = { columns = [{ table = 5, index = 0 }, { table = 5, index = 1 }], index = { table = 5, index = 0 } } + +[[schema.tables.columns]] +id = { table = 5, index = 0 } +name = "download_run_id" +ty = "Uuid" +storage_ty = "Uuid" + +[[schema.tables.columns]] +id = { table = 5, index = 1 } +name = "url" +ty = "String" +storage_ty = "Text" + +[[schema.tables.columns]] +id = { table = 5, index = 2 } +name = "sha256" +ty = "String" +storage_ty = "Text" + +[[schema.tables.indices]] +id = { table = 5, index = 0 } +name = "index_blob_downloads_by_download_run_id_and_url" +on = 5 +columns = [{ column = { table = 5, index = 0 }, op = "Eq", scope = "Partition" }, { column = { table = 5, index = 1 }, op = "Eq", scope = "Local" }] +unique = true +primary_key = true + +[[schema.tables.indices]] +id = { table = 5, index = 1 } +name = "index_blob_downloads_by_sha256" +on = 5 +columns = [{ column = { table = 5, index = 2 }, op = "Eq", scope = "Partition" }] + +[[schema.tables]] +id = 6 +name = "fabric_catalogs" +primary_key = { columns = [{ table = 6, index = 0 }], index = { table = 6, index = 0 } } + +[[schema.tables.columns]] +id = { table = 6, index = 0 } +name = "id" +ty = "Uuid" +storage_ty = "Uuid" + +[[schema.tables.columns]] +id = { table = 6, index = 1 } +name = "download_run_id" +ty = "Uuid" +storage_ty = "Uuid" + +[[schema.tables.columns]] +id = { table = 6, index = 2 } +name = "sha256" +ty = "String" +storage_ty = "Text" + +[[schema.tables.indices]] +id = { table = 6, index = 0 } +name = "index_fabric_catalogs_by_id" +on = 6 +columns = [{ column = { table = 6, index = 0 }, op = "Eq", scope = "Partition" }] +unique = true +primary_key = true + +[[schema.tables.indices]] +id = { table = 6, index = 1 } +name = "index_fabric_catalogs_by_download_run_id" +on = 6 +columns = [{ column = { table = 6, index = 1 }, op = "Eq", scope = "Partition" }] +unique = true + +[[schema.tables]] +id = 7 +name = "quilt_catalogs" +primary_key = { columns = [{ table = 7, index = 0 }], index = { table = 7, index = 0 } } + +[[schema.tables.columns]] +id = { table = 7, index = 0 } +name = "id" +ty = "Uuid" +storage_ty = "Uuid" + +[[schema.tables.columns]] +id = { table = 7, index = 1 } +name = "download_run_id" +ty = "Uuid" +storage_ty = "Uuid" + +[[schema.tables.columns]] +id = { table = 7, index = 2 } +name = "sha256" +ty = "String" +storage_ty = "Text" + +[[schema.tables.indices]] +id = { table = 7, index = 0 } +name = "index_quilt_catalogs_by_id" +on = 7 +columns = [{ column = { table = 7, index = 0 }, op = "Eq", scope = "Partition" }] +unique = true +primary_key = true + +[[schema.tables.indices]] +id = { table = 7, index = 1 } +name = "index_quilt_catalogs_by_download_run_id" +on = 7 +columns = [{ column = { table = 7, index = 1 }, op = "Eq", scope = "Partition" }] +unique = true + +[[schema.tables]] +id = 8 +name = "forgelike_installers" +primary_key = { columns = [{ table = 8, index = 0 }, { table = 8, index = 1 }], index = { table = 8, index = 0 } } + +[[schema.tables.columns]] +id = { table = 8, index = 0 } +name = "loader" +ty = "String" +storage_ty = { Enum = { name = "forgelike_loader", variants = [{ name = "forge" }, { name = "neoforge" }] } } + +[[schema.tables.columns]] +id = { table = 8, index = 1 } +name = "name" +ty = "String" +storage_ty = "Text" + +[[schema.tables.columns]] +id = { table = 8, index = 2 } +name = "download_run_id" +ty = "Uuid" +storage_ty = "Uuid" + +[[schema.tables.columns]] +id = { table = 8, index = 3 } +name = "sha256" +ty = "String" +storage_ty = "Text" + +[[schema.tables.indices]] +id = { table = 8, index = 0 } +name = "index_forgelike_installers_by_loader_and_name" +on = 8 +columns = [{ column = { table = 8, index = 0 }, op = "Eq", scope = "Partition" }, { column = { table = 8, index = 1 }, op = "Eq", scope = "Local" }] +unique = true +primary_key = true + +[[schema.tables.indices]] +id = { table = 8, index = 1 } +name = "index_forgelike_installers_by_sha256" +on = 8 +columns = [{ column = { table = 8, index = 3 }, op = "Eq", scope = "Partition" }] + +[[schema.tables]] +id = 9 +name = "forgelike_extracts" +primary_key = { columns = [{ table = 9, index = 0 }], index = { table = 9, index = 0 } } + +[[schema.tables.columns]] +id = { table = 9, index = 0 } +name = "installer_sha256" +ty = "String" +storage_ty = "Text" + +[[schema.tables.columns]] +id = { table = 9, index = 1 } +name = "created_at" +ty = "Timestamp" +storage_ty = { Timestamp = 6 } + +[[schema.tables.columns]] +id = { table = 9, index = 2 } +name = "metadata" +ty = "String" +storage_ty = "Json" + +[[schema.tables.columns]] +id = { table = 9, index = 3 } +name = "embedded_maven_artifacts" +ty = "String" +storage_ty = "Json" + +[[schema.tables.indices]] +id = { table = 9, index = 0 } +name = "index_forgelike_extracts_by_installer_sha256" +on = 9 +columns = [{ column = { table = 9, index = 0 }, op = "Eq", scope = "Partition" }] +unique = true +primary_key = true + +[[schema.tables]] +id = 10 +name = "forge_catalogs" +primary_key = { columns = [{ table = 10, index = 0 }], index = { table = 10, index = 0 } } + +[[schema.tables.columns]] +id = { table = 10, index = 0 } +name = "id" +ty = "Uuid" +storage_ty = "Uuid" + +[[schema.tables.columns]] +id = { table = 10, index = 1 } +name = "download_run_id" +ty = "Uuid" +storage_ty = "Uuid" + +[[schema.tables.columns]] +id = { table = 10, index = 2 } +name = "sha256" +ty = "String" +storage_ty = "Text" + +[[schema.tables.indices]] +id = { table = 10, index = 0 } +name = "index_forge_catalogs_by_id" +on = 10 +columns = [{ column = { table = 10, index = 0 }, op = "Eq", scope = "Partition" }] +unique = true +primary_key = true + +[[schema.tables.indices]] +id = { table = 10, index = 1 } +name = "index_forge_catalogs_by_download_run_id" +on = 10 +columns = [{ column = { table = 10, index = 1 }, op = "Eq", scope = "Partition" }] +unique = true + +[[schema.tables]] +id = 11 +name = "fabriclike_catalogs" +primary_key = { columns = [{ table = 11, index = 0 }], index = { table = 11, index = 0 } } + +[[schema.tables.columns]] +id = { table = 11, index = 0 } +name = "loader" +ty = "String" +storage_ty = { Enum = { name = "fabriclike_loader", variants = [{ name = "fabric" }, { name = "quilt" }] } } + +[[schema.tables.columns]] +id = { table = 11, index = 1 } +name = "games" +ty = "String" +storage_ty = "Json" + +[[schema.tables.columns]] +id = { table = 11, index = 2 } +name = "groups" +ty = "String" +storage_ty = "Json" + +[[schema.tables.columns]] +id = { table = 11, index = 3 } +name = "artifacts" +ty = "String" +storage_ty = "Json" + +[[schema.tables.indices]] +id = { table = 11, index = 0 } +name = "index_fabriclike_catalogs_by_loader" +on = 11 +columns = [{ column = { table = 11, index = 0 }, op = "Eq", scope = "Partition" }] +unique = true +primary_key = true + +[[schema.tables]] +id = 12 +name = "fabriclike_profiles" +primary_key = { columns = [{ table = 12, index = 0 }, { table = 12, index = 1 }, { table = 12, index = 2 }], index = { table = 12, index = 0 } } + +[[schema.tables.columns]] +id = { table = 12, index = 0 } +name = "loader" +ty = "String" +storage_ty = { Enum = { name = "fabriclike_loader", variants = [{ name = "fabric" }, { name = "quilt" }] } } + +[[schema.tables.columns]] +id = { table = 12, index = 1 } +name = "version_group" +ty = "String" +storage_ty = "Text" + +[[schema.tables.columns]] +id = { table = 12, index = 2 } +name = "loader_version" +ty = "String" +storage_ty = "Text" + +[[schema.tables.columns]] +id = { table = 12, index = 3 } +name = "stable" +ty = "Bool" +storage_ty = "Boolean" + +[[schema.tables.columns]] +id = { table = 12, index = 4 } +name = "position" +ty = "I32" +storage_ty = { Integer = 4 } + +[[schema.tables.columns]] +id = { table = 12, index = 5 } +name = "source_url" +ty = "String" +storage_ty = "Text" + +[[schema.tables.columns]] +id = { table = 12, index = 6 } +name = "source_sha256" +ty = "String" +storage_ty = "Text" + +[[schema.tables.columns]] +id = { table = 12, index = 7 } +name = "template" +ty = "String" +storage_ty = "Text" + +[[schema.tables.columns]] +id = { table = 12, index = 8 } +name = "metadata" +ty = "String" +storage_ty = "Json" + +[[schema.tables.indices]] +id = { table = 12, index = 0 } +name = "index_fabriclike_profiles_by_loader_and_version_group_and__8be5" +on = 12 +columns = [{ column = { table = 12, index = 0 }, op = "Eq", scope = "Partition" }, { column = { table = 12, index = 1 }, op = "Eq", scope = "Local" }, { column = { table = 12, index = 2 }, op = "Eq", scope = "Local" }] +unique = true +primary_key = true + +[[schema.tables]] +id = 13 +name = "minecraft_versions" +primary_key = { columns = [{ table = 13, index = 0 }], index = { table = 13, index = 0 } } + +[[schema.tables.columns]] +id = { table = 13, index = 0 } +name = "name" +ty = "String" +storage_ty = "Text" + +[[schema.tables.columns]] +id = { table = 13, index = 1 } +name = "catalog_entry" +ty = "String" +storage_ty = "Json" + +[[schema.tables.columns]] +id = { table = 13, index = 2 } +name = "manifest" +ty = "String" +storage_ty = "Json" + +[[schema.tables.columns]] +id = { table = 13, index = 3 } +name = "source_catalog_sha256" +ty = "String" +storage_ty = "Text" + +[[schema.tables.columns]] +id = { table = 13, index = 4 } +name = "source_manifest_sha256" +ty = "String" +storage_ty = "Text" + +[[schema.tables.columns]] +id = { table = 13, index = 5 } +name = "is_latest_release" +ty = "Bool" +storage_ty = "Boolean" +nullable = true + +[[schema.tables.columns]] +id = { table = 13, index = 6 } +name = "is_latest_snapshot" +ty = "Bool" +storage_ty = "Boolean" +nullable = true + +[[schema.tables.indices]] +id = { table = 13, index = 0 } +name = "index_minecraft_versions_by_name" +on = 13 +columns = [{ column = { table = 13, index = 0 }, op = "Eq", scope = "Partition" }] +unique = true +primary_key = true + +[[schema.tables.indices]] +id = { table = 13, index = 1 } +name = "index_minecraft_versions_by_is_latest_release" +on = 13 +columns = [{ column = { table = 13, index = 5 }, op = "Eq", scope = "Partition" }] +unique = true + +[[schema.tables.indices]] +id = { table = 13, index = 2 } +name = "index_minecraft_versions_by_is_latest_snapshot" +on = 13 +columns = [{ column = { table = 13, index = 6 }, op = "Eq", scope = "Partition" }] +unique = true diff --git a/clippy.toml b/clippy.toml index ddc612e9083..4296655a040 100644 --- a/clippy.toml +++ b/clippy.toml @@ -1,2 +1 @@ allow-dbg-in-tests = true -msrv = "1.90.0" diff --git a/docker-compose.yml b/docker-compose.yml index 5909927ba98..643f51baeec 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,6 +18,61 @@ services: interval: 3s timeout: 5s retries: 3 + postgres_launcher_meta: + image: postgres:15-alpine + container_name: labrinth-postgres-launcher-meta + volumes: + - db-launcher-meta-data:/var/lib/postgresql/data + ports: + - '127.0.0.1:5433:5432' + environment: + POSTGRES_USER: launchermeta + POSTGRES_PASSWORD: launchermeta + POSTGRES_HOST_AUTH_METHOD: trust + PGUSER: launchermeta + healthcheck: + test: ['CMD', 'pg_isready', '-U', 'launchermeta'] + interval: 3s + timeout: 5s + retries: 3 + minecraft-meta-s3: + image: docker.io/rustfs/rustfs:1.0.1 + user: '0:0' + ports: + - '127.0.0.1:9000:9000' + - '127.0.0.1:9001:9001' + environment: + RUSTFS_VOLUMES: /data + RUSTFS_ADDRESS: 0.0.0.0:9000 + RUSTFS_CONSOLE_ADDRESS: 0.0.0.0:9001 + RUSTFS_CONSOLE_ENABLE: 'true' + RUSTFS_ACCESS_KEY: minecraftmeta + RUSTFS_SECRET_KEY: minecraftmeta-local + RUSTFS_OBS_LOG_DIRECTORY: /data/logs + volumes: + - minecraft-meta-s3-data:/data + healthcheck: + test: ['CMD', 'curl', '--fail', 'http://localhost:9000/health'] + interval: 3s + timeout: 5s + retries: 10 + minecraft-meta-s3-init: + image: docker.io/amazon/aws-cli:2.34.0 + environment: + AWS_ACCESS_KEY_ID: minecraftmeta + AWS_SECRET_ACCESS_KEY: minecraftmeta-local + AWS_DEFAULT_REGION: us-east-1 + AWS_ENDPOINT_URL: http://minecraft-meta-s3:9000 + AWS_PAGER: '' + depends_on: + minecraft-meta-s3: + condition: service_healthy + entrypoint: ['sh', '-ec'] + command: + - | + aws s3api head-bucket --bucket minecraft-meta-public || aws s3api create-bucket --bucket minecraft-meta-public + aws s3api head-bucket --bucket minecraft-meta-private || aws s3api create-bucket --bucket minecraft-meta-private + aws s3api put-bucket-policy --bucket minecraft-meta-public --policy '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::minecraft-meta-public/*"}]}' elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch:9.4.4 container_name: labrinth-elasticsearch @@ -429,6 +484,8 @@ networks: volumes: elasticsearch-data: db-data: + db-launcher-meta-data: + minecraft-meta-s3-data: redis-data: redis-insight-data: redpanda-data: diff --git a/packages/modrinth-minecraft-meta/Cargo.toml b/packages/modrinth-minecraft-meta/Cargo.toml new file mode 100644 index 00000000000..6165ed313bc --- /dev/null +++ b/packages/modrinth-minecraft-meta/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "modrinth-minecraft-meta" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true + +[dependencies] +anyhow = { workspace = true } +derive_more = { workspace = true, features = ["debug", "deref", "display"] } +hex = { workspace = true } +jiff = { workspace = true, features = ["serde"] } +regex = { workspace = true } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +serde_path_to_error = { workspace = true } +sha1 = { workspace = true } +sha2 = { workspace = true } +toasty = { workspace = true, features = ["jiff", "postgresql", "serde"] } +url = { workspace = true, features = ["serde"] } +uuid = { workspace = true, features = ["v7"] } diff --git a/packages/modrinth-minecraft-meta/README.md b/packages/modrinth-minecraft-meta/README.md new file mode 100644 index 00000000000..9f8e6a2acc7 --- /dev/null +++ b/packages/modrinth-minecraft-meta/README.md @@ -0,0 +1 @@ +Models and schema for game version and launcher version related metadata. diff --git a/packages/modrinth-minecraft-meta/src/lib.rs b/packages/modrinth-minecraft-meta/src/lib.rs new file mode 100644 index 00000000000..e4e5b246b59 --- /dev/null +++ b/packages/modrinth-minecraft-meta/src/lib.rs @@ -0,0 +1,3 @@ +pub mod model; +pub mod upstream; +pub mod util; diff --git a/packages/modrinth-minecraft-meta/src/model/download.rs b/packages/modrinth-minecraft-meta/src/model/download.rs new file mode 100644 index 00000000000..ac6f65b1619 --- /dev/null +++ b/packages/modrinth-minecraft-meta/src/model/download.rs @@ -0,0 +1,56 @@ +use jiff::Timestamp; +use toasty::{Embed, Model}; +use uuid::Uuid; + +use crate::util::{Sha1, Sha256}; + +/// Hashes for blobs retained in content-addressed storage. +#[derive(Debug, Clone, Model)] +#[table = "blob_hashes"] +pub struct BlobHash { + #[key] + pub sha256: Sha256, + #[index] + pub sha1: Sha1, +} + +/// ID for a [`DownloadRun`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Embed)] +pub struct DownloadRunId(pub Uuid); + +/// Single run of a task to download sources from our upstreams. +/// +/// During a [`DownloadRun`], we download a bunch of files and save them into +/// our database and file store. Any blobs saved within the same run can be +/// logically grouped together. +#[derive(Debug, Clone, Model)] +pub struct DownloadRun { + #[key] + #[auto] + pub id: DownloadRunId, + /// When this run started. + pub started_at: Timestamp, + /// When this run finished. + /// + /// If this run is still ongoing, this may be [`None`]. + pub completed_at: Option, + /// Errors accumulated during the run. + /// + /// If this run is still ongoing, this may be [`None`]. + #[column(type = text)] + pub errors: Option>>, +} + +/// During one [`DownloadRun`], what blobs were downloaded, and from where? +/// +/// Blobs are represented as [`Sha256`]. +#[derive(Debug, Clone, Model)] +#[key(download_run_id, url)] +pub struct BlobDownload { + pub download_run_id: DownloadRunId, + #[belongs_to] + pub download_run: toasty::Deferred, + pub url: String, + #[index] + pub sha256: Sha256, +} diff --git a/packages/modrinth-minecraft-meta/src/model/fabric.rs b/packages/modrinth-minecraft-meta/src/model/fabric.rs new file mode 100644 index 00000000000..f4e8f224619 --- /dev/null +++ b/packages/modrinth-minecraft-meta/src/model/fabric.rs @@ -0,0 +1,24 @@ +use toasty::{Embed, Model}; +use uuid::Uuid; + +use crate::{ + model::{DownloadRun, DownloadRunId}, + util::Sha256, +}; + +/// ID of a [`FabricCatalog`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Embed)] +pub struct FabricCatalogId(pub Uuid); + +/// Fabric version catalog downloaded during some [`DownloadRun`]. +#[derive(Debug, Clone, Model)] +pub struct FabricCatalog { + #[key] + #[auto] + pub id: FabricCatalogId, + #[unique] + pub download_run_id: DownloadRunId, + #[belongs_to] + pub download_run: toasty::Deferred, + pub sha256: Sha256, +} diff --git a/packages/modrinth-minecraft-meta/src/model/fabriclike.rs b/packages/modrinth-minecraft-meta/src/model/fabriclike.rs new file mode 100644 index 00000000000..1da69e32613 --- /dev/null +++ b/packages/modrinth-minecraft-meta/src/model/fabriclike.rs @@ -0,0 +1,132 @@ +use std::collections::HashMap; + +use jiff::Timestamp; +use serde::{Deserialize, Serialize}; +use toasty::{Embed, Model}; +use url::Url; + +use crate::{ + model::{Processor, SidedDataEntry}, + upstream::mojang::{ + Argument, ArgumentType, LibraryDownloads, LibraryExtract, + OperatingSystem, Rule, VersionType, + }, + util::{MavenCoordinate, Sha256}, +}; + +pub const GAME_VERSION_PLACEHOLDER: &str = "${modrinth.gameVersion}"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Embed)] +pub enum FabriclikeLoader { + Fabric, + Quilt, +} + +/// Selected games, profile groups, and concrete dependencies for one loader. +#[derive(Debug, Clone, Model)] +pub struct FabriclikeCatalog { + #[key] + pub loader: FabriclikeLoader, + #[column(type = json)] + pub games: toasty::Json>, + #[column(type = json)] + pub groups: toasty::Json>, + #[column(type = json)] + pub artifacts: toasty::Json>, +} + +/// A normalized loader profile, independent of its public store URL. +#[derive(Debug, Clone, Model)] +#[key(loader, version_group, loader_version)] +pub struct FabriclikeProfile { + pub loader: FabriclikeLoader, + /// Empty for Fabric's universal group; Quilt uses named groups. + pub version_group: String, + pub loader_version: String, + pub stable: bool, + pub position: i32, + pub source_url: String, + pub source_sha256: Sha256, + pub template: String, + #[column(type = json)] + pub metadata: toasty::Json, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FabriclikeGameVersion { + pub id: String, + pub stable: bool, +} + +/// Game versions sharing a profile, including its upstream template provenance. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FabriclikeProfileGroup { + pub id: Option, + pub template: String, + pub fallback_templates: Vec, + pub games: Vec, + pub profile_base_url: Url, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FabriclikeArtifact { + pub coordinate: MavenCoordinate, + pub repository: Url, + pub source: Option, + pub repository_source: String, +} + +/// A concrete dependency whose bytes are stored in content-addressed storage. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FabriclikeStoredArtifact { + #[serde(flatten)] + pub artifact: FabriclikeArtifact, + pub sha256: Sha256, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FabriclikeProfileMetadata { + pub id: String, + pub inherits_from: String, + pub release_time: Timestamp, + pub time: Timestamp, + #[serde(skip_serializing_if = "Option::is_none")] + pub main_class: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub minecraft_arguments: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub arguments: Option>>, + pub libraries: Vec, + #[serde(rename = "type")] + pub ty: VersionType, + #[serde(skip_serializing_if = "Option::is_none")] + pub data: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub processors: Option>, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FabriclikeLibrary { + #[serde(skip_serializing_if = "Option::is_none")] + pub downloads: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub extract: Option, + pub name: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub url: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub natives: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub rules: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub checksums: Option>, + #[serde(default = "default_true")] + pub include_in_classpath: bool, + #[serde(default = "default_true")] + pub downloadable: bool, +} + +fn default_true() -> bool { + true +} diff --git a/packages/modrinth-minecraft-meta/src/model/forge.rs b/packages/modrinth-minecraft-meta/src/model/forge.rs new file mode 100644 index 00000000000..96728a186ab --- /dev/null +++ b/packages/modrinth-minecraft-meta/src/model/forge.rs @@ -0,0 +1,22 @@ +use toasty::{Embed, Model}; +use uuid::Uuid; + +use crate::{ + model::{DownloadRun, DownloadRunId}, + util::Sha256, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Embed)] +pub struct ForgeCatalogId(pub Uuid); + +#[derive(Debug, Clone, Model)] +pub struct ForgeCatalog { + #[key] + #[auto] + pub id: ForgeCatalogId, + #[unique] + pub download_run_id: DownloadRunId, + #[belongs_to] + pub download_run: toasty::Deferred, + pub sha256: Sha256, +} diff --git a/packages/modrinth-minecraft-meta/src/model/forgelike/mod.rs b/packages/modrinth-minecraft-meta/src/model/forgelike/mod.rs new file mode 100644 index 00000000000..0a72a581251 --- /dev/null +++ b/packages/modrinth-minecraft-meta/src/model/forgelike/mod.rs @@ -0,0 +1,91 @@ +mod name; +pub use name::ForgelikeInstallerName; + +use jiff::Timestamp; +use serde::{Deserialize, Serialize}; +use toasty::{Embed, Model}; +use crate::model::NEOFORGE_BLACKLIST; + +use crate::{ + model::{DownloadRun, DownloadRunId, ProfileMetadata}, + util::{MavenCoordinate, Sha256}, +}; + + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Embed)] +pub enum ForgelikeLoader { + Forge, + Neoforge, +} + +#[derive(Debug, Clone, Model)] +#[key(loader, name)] +pub struct ForgelikeInstaller { + pub loader: ForgelikeLoader, + pub name: ForgelikeInstallerName, + pub download_run_id: DownloadRunId, + #[belongs_to] + pub download_run: toasty::Deferred, + #[index] + pub sha256: Sha256, +} + +impl ForgelikeInstaller { + pub fn is_blacklisted(&self) -> bool { + match self.loader { + ForgelikeLoader::Forge => false, + ForgelikeLoader::Neoforge => NEOFORGE_BLACKLIST.contains(&self.name.as_str()), + } + } +} + +/// An embedded or downloaded Maven artifact retained in content-addressed storage. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ForgelikeStoredArtifact { + pub coordinate: MavenCoordinate, + pub sha256: Sha256, +} + +/// Parsed metadata and stored dependencies from a Forge or NeoForge installer. +#[derive(Debug, Clone, Model)] +pub struct ForgelikeExtract { + /// [`Sha256`] hash of the installer JAR file which we extracted this + /// information from. + /// + /// We key on a file's hash since that's the source of truth for this info; + /// not a specific installer record. + #[key] + pub installer_sha256: Sha256, + /// When this installer was extracted, and the extraction record was + /// created. + #[auto] + pub created_at: Timestamp, + #[column(type = json)] + pub metadata: toasty::Json, + /// Referenced embedded and downloaded artifacts, with coordinates and CAS hashes. + /// The field name is retained for compatibility with existing extraction records. + #[column(type = json)] + pub embedded_maven_artifacts: toasty::Json>, +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn stored_artifact_round_trips_and_requires_hash() { + let value = json!({ + "coordinate": "net.example:library:1", + "sha256": "0000000000000000000000000000000000000000000000000000000000000000" + }); + let stored: ForgelikeStoredArtifact = serde_json::from_value(value.clone()).unwrap(); + assert_eq!(serde_json::to_value(stored).unwrap(), value); + assert!(serde_json::from_value::(json!({ + "coordinate": "net.example:library:1" + })).is_err()); + assert!(serde_json::from_value::>(json!([ + "maven/net/example/library/1/library-1.jar" + ])).is_err()); + } +} diff --git a/packages/modrinth-minecraft-meta/src/model/forgelike/name.rs b/packages/modrinth-minecraft-meta/src/model/forgelike/name.rs new file mode 100644 index 00000000000..635003edeb7 --- /dev/null +++ b/packages/modrinth-minecraft-meta/src/model/forgelike/name.rs @@ -0,0 +1,166 @@ +use std::{str::FromStr, sync::LazyLock}; + +use anyhow::ensure; +use derive_more::Display; +use regex::Regex; +use serde::{Deserialize, Serialize}; +use toasty::{ + schema::{Field, Load}, + stmt::{Assign, Assignment, Expr, IntoExpr, List, Path, Type, Value}, +}; + +/// Full upstream version name identifying a Forge or NeoForge installer. +/// +/// Forge uses `-`; modern NeoForge uses +/// its loader version alone. +#[derive( + Debug, Display, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, +)] +pub struct ForgelikeInstallerName(String); + +impl ForgelikeInstallerName { + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl FromStr for ForgelikeInstallerName { + type Err = anyhow::Error; + + fn from_str(value: &str) -> Result { + static PATTERN: LazyLock = LazyLock::new(|| { + Regex::new(r"\A[A-Za-z0-9_+\-]+(?:\.[A-Za-z0-9_+\-]+)*\z").unwrap() + }); + ensure!( + PATTERN.is_match(value), + "invalid Forge-like installer name: {value:?}" + ); + Ok(Self(value.to_owned())) + } +} + +impl<'de> Deserialize<'de> for ForgelikeInstallerName { + fn deserialize>( + deserializer: D, + ) -> Result { + String::deserialize(deserializer)? + .parse() + .map_err(serde::de::Error::custom) + } +} + +impl Load for ForgelikeInstallerName { + type Output = Self; + fn ty() -> Type { + Type::String + } + fn load(value: Value) -> toasty::Result { + match value { + Value::String(value) => value.parse().map_err(|error| { + toasty::Error::from_args(format_args!("{error}")) + }), + _ => Err(toasty::Error::type_conversion( + value, + "ForgelikeInstallerName", + )), + } + } + fn reload(target: &mut Self, value: Value) -> toasty::Result<()> { + *target = Self::load(value)?; + Ok(()) + } +} +impl Field for ForgelikeInstallerName { + type ExprTarget = Self; + type Path = Path; + type ListPath = Path>; + type Update<'a> = (); + type Inner = Self; + + fn new_path(path: Path) -> Self::Path { + path + } + + fn new_list_path( + path: Path>, + ) -> Self::ListPath { + path + } + + fn new_update<'a>( + _assignments: &'a mut toasty::codegen_support::core::stmt::Assignments, + _projection: toasty::stmt::Projection, + ) -> Self::Update<'a> { + } + + fn key_constraint( + &self, + target: Path, + ) -> Expr { + target.eq(self) + } +} + +impl IntoExpr for ForgelikeInstallerName { + fn into_expr(self) -> Expr { + Expr::from_untyped(Value::String(self.0.clone())) + } + + fn by_ref(&self) -> Expr { + Expr::from_untyped(Value::String(self.0.clone())) + } +} + +impl Assign for ForgelikeInstallerName { + fn into_assignment(self) -> Assignment { + toasty::stmt::set(self.into_expr()) + } +} + +impl toasty::codegen_support::index::IndexableField for ForgelikeInstallerName {} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn installer_names_validate_parsing_json_and_database_loading() { + for value in [ + "1.20.1-47.1.7", + "21.4.31-beta", + "26.1.0.0-alpha.1+snapshot-1", + ] { + let name: ForgelikeInstallerName = value.parse().unwrap(); + assert_eq!(name.as_str(), value); + assert_eq!(serde_json::to_value(&name).unwrap(), json!(value)); + assert_eq!( + serde_json::from_value::(json!(value)) + .unwrap(), + name + ); + assert_eq!( + ForgelikeInstallerName::load(Value::String(value.into())) + .unwrap(), + name + ); + } + for value in [ + "", ".", "..", "a/b", "a\\b", "a?b", "a b", "a\n", ".foo", "foo.", + "foo..bar", + ] { + assert!( + value.parse::().is_err(), + "{value:?}" + ); + assert!( + serde_json::from_value::(json!(value)) + .is_err() + ); + assert!( + ForgelikeInstallerName::load(Value::String(value.into())) + .is_err() + ); + } + } +} diff --git a/packages/modrinth-minecraft-meta/src/model/minecraft.rs b/packages/modrinth-minecraft-meta/src/model/minecraft.rs new file mode 100644 index 00000000000..7c55f804db9 --- /dev/null +++ b/packages/modrinth-minecraft-meta/src/model/minecraft.rs @@ -0,0 +1,47 @@ +use derive_more::Display; +use serde::{Deserialize, Serialize}; +use toasty::{Embed, Model}; + +use crate::{upstream::mojang, util::Sha256}; + +#[derive( + Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, Embed, +)] +pub enum MinecraftLoader { + Fabric, + Forge, + Neoforge, + Quilt, +} + +#[derive( + Debug, + Display, + Clone, + PartialEq, + Eq, + PartialOrd, + Ord, + Hash, + Serialize, + Deserialize, + Embed, +)] +pub struct MinecraftVersionName(pub String); + +/// Selected Minecraft metadata, with library patches applied for publication. +#[derive(Debug, Clone, Model)] +pub struct MinecraftVersion { + #[key] + pub name: MinecraftVersionName, + #[column(type = json)] + pub catalog_entry: toasty::Json, + #[column(type = json)] + pub manifest: toasty::Json, + pub source_catalog_sha256: Sha256, + pub source_manifest_sha256: Sha256, + #[unique] + pub is_latest_release: Option, + #[unique] + pub is_latest_snapshot: Option, +} diff --git a/packages/modrinth-minecraft-meta/src/model/mod.rs b/packages/modrinth-minecraft-meta/src/model/mod.rs new file mode 100644 index 00000000000..2689c7d3c85 --- /dev/null +++ b/packages/modrinth-minecraft-meta/src/model/mod.rs @@ -0,0 +1,22 @@ +//! Models for tables stored in the database. + +mod download; +mod fabric; +mod fabriclike; +mod forge; +mod forgelike; +mod minecraft; +mod mojang; +mod neoforge; +mod profile; +mod quilt; +pub use download::*; +pub use fabric::*; +pub use fabriclike::*; +pub use forge::*; +pub use forgelike::*; +pub use minecraft::*; +pub use mojang::*; +pub use neoforge::*; +pub use profile::*; +pub use quilt::*; diff --git a/packages/modrinth-minecraft-meta/src/model/mojang.rs b/packages/modrinth-minecraft-meta/src/model/mojang.rs new file mode 100644 index 00000000000..432eb38b73e --- /dev/null +++ b/packages/modrinth-minecraft-meta/src/model/mojang.rs @@ -0,0 +1,22 @@ +use toasty::{Embed, Model}; +use uuid::Uuid; + +use crate::{ + model::{DownloadRun, DownloadRunId}, + util::Sha256, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Embed)] +pub struct MojangCatalogId(pub Uuid); + +#[derive(Debug, Clone, Model)] +pub struct MojangCatalog { + #[key] + #[auto] + pub id: MojangCatalogId, + #[unique] + pub download_run_id: DownloadRunId, + #[belongs_to] + pub download_run: toasty::Deferred, + pub sha256: Sha256, +} diff --git a/packages/modrinth-minecraft-meta/src/model/neoforge.rs b/packages/modrinth-minecraft-meta/src/model/neoforge.rs new file mode 100644 index 00000000000..6e2654c22ad --- /dev/null +++ b/packages/modrinth-minecraft-meta/src/model/neoforge.rs @@ -0,0 +1,33 @@ +use toasty::{Embed, Model}; +use uuid::Uuid; + +use crate::{ + model::{DownloadRun, DownloadRunId}, + util::Sha256, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Embed)] +pub struct NeoforgeCatalogId(pub Uuid); + +#[derive(Debug, Clone, Model)] +pub struct NeoforgeCatalog { + #[key] + #[auto] + pub id: NeoforgeCatalogId, + #[unique] + pub download_run_id: DownloadRunId, + #[belongs_to] + pub download_run: toasty::Deferred, + pub forge_sha256: Sha256, + pub neoforge_sha256: Sha256, +} + +/// Specific NeoForge installer versions which we can't download or process. +pub const NEOFORGE_BLACKLIST: &[&str] = &[ + // listed upstream, but installer JARs and POMs return 404 + "1.20.1-47.1.7", + "47.1.82", + // the installer references a missing universal JAR; upstream and launcher-meta + // return 404, although production publishes a manifest referencing that JAR + "21.4.31-beta", +]; diff --git a/packages/modrinth-minecraft-meta/src/model/profile.rs b/packages/modrinth-minecraft-meta/src/model/profile.rs new file mode 100644 index 00000000000..c54446ef134 --- /dev/null +++ b/packages/modrinth-minecraft-meta/src/model/profile.rs @@ -0,0 +1,65 @@ +use std::collections::HashMap; + +use jiff::Timestamp; +use serde::{Deserialize, Serialize}; +use toasty::Model; + +use crate::{ + model::{MinecraftLoader, MinecraftVersionName}, + upstream::mojang::{Argument, ArgumentType, Library, VersionType}, +}; + +/// Normalized loader profile to be published as a version JSON artifact. +#[derive(Debug, Clone, Model)] +#[key(loader, minecraft_version, loader_version)] +pub struct Profile { + pub loader: MinecraftLoader, + #[index] + pub minecraft_version: MinecraftVersionName, + pub loader_version: String, + #[column(type = json)] + pub metadata: toasty::Json, +} + +/// Launch metadata combined with optional Forge-style installation metadata. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ProfileMetadata { + pub id: String, + pub inherits_from: String, + #[serde(deserialize_with = "crate::util::deserialize_timestamp")] + pub release_time: Timestamp, + #[serde(deserialize_with = "crate::util::deserialize_timestamp")] + pub time: Timestamp, + #[serde(skip_serializing_if = "Option::is_none")] + pub main_class: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub minecraft_arguments: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub arguments: Option>>, + pub libraries: Vec, + #[serde(rename = "type")] + pub ty: VersionType, + #[serde(skip_serializing_if = "Option::is_none")] + pub data: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub processors: Option>, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SidedDataEntry { + pub client: String, + pub server: String, +} + +/// Installation step executed by the launcher after downloading libraries. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Processor { + pub jar: String, + pub classpath: Vec, + pub args: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub outputs: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub sides: Option>, +} diff --git a/packages/modrinth-minecraft-meta/src/model/quilt.rs b/packages/modrinth-minecraft-meta/src/model/quilt.rs new file mode 100644 index 00000000000..00bd14636e4 --- /dev/null +++ b/packages/modrinth-minecraft-meta/src/model/quilt.rs @@ -0,0 +1,22 @@ +use toasty::{Embed, Model}; +use uuid::Uuid; + +use crate::{ + model::{DownloadRun, DownloadRunId}, + util::Sha256, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Embed)] +pub struct QuiltCatalogId(pub Uuid); + +#[derive(Debug, Clone, Model)] +pub struct QuiltCatalog { + #[key] + #[auto] + pub id: QuiltCatalogId, + #[unique] + pub download_run_id: DownloadRunId, + #[belongs_to] + pub download_run: toasty::Deferred, + pub sha256: Sha256, +} diff --git a/packages/modrinth-minecraft-meta/src/upstream/fabric.rs b/packages/modrinth-minecraft-meta/src/upstream/fabric.rs new file mode 100644 index 00000000000..43bccbb40ab --- /dev/null +++ b/packages/modrinth-minecraft-meta/src/upstream/fabric.rs @@ -0,0 +1,68 @@ +use derive_more::Display; +use serde::{Deserialize, Serialize}; +use url::Url; + +use super::fabriclike::{GameVersion, GameVersionName, LoaderVersionName}; + +use crate::util::MavenCoordinate; + +pub const CATALOG_URL: &str = "https://meta.fabricmc.net/v2/versions"; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Catalog { + pub game: Vec, + pub mappings: Vec, + pub intermediary: Vec, + pub loader: Vec, + pub installer: Vec, +} + +#[derive( + Debug, Display, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, +)] +pub struct MappingVersionName(pub String); + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MappingVersion { + pub game_version: GameVersionName, + pub separator: String, + pub build: u32, + pub maven: MavenCoordinate, + pub version: MappingVersionName, + pub stable: bool, +} + +#[derive( + Debug, Display, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, +)] +pub struct IntermediaryVersionName(pub String); + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct IntermediaryVersion { + pub maven: MavenCoordinate, + pub version: IntermediaryVersionName, + pub stable: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LoaderVersion { + pub separator: String, + pub build: u32, + pub maven: MavenCoordinate, + pub version: LoaderVersionName, + pub stable: bool, +} + +#[derive( + Debug, Display, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, +)] +pub struct InstallerVersionName(pub String); + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct InstallerVersion { + pub url: Url, + pub maven: MavenCoordinate, + pub version: InstallerVersionName, + pub stable: bool, +} diff --git a/packages/modrinth-minecraft-meta/src/upstream/fabriclike.rs b/packages/modrinth-minecraft-meta/src/upstream/fabriclike.rs new file mode 100644 index 00000000000..f43b78bf205 --- /dev/null +++ b/packages/modrinth-minecraft-meta/src/upstream/fabriclike.rs @@ -0,0 +1,45 @@ +use derive_more::Display; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Display, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct GameVersionName(pub String); + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GameVersion { + pub version: GameVersionName, + pub stable: bool, +} + +#[derive(Debug, Display, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct LoaderVersionName(pub String); + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GameLoaderProfile { + pub id: String, +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn game_versions_preserve_upstream_shape() { + let value = json!({"version": "1.21", "stable": true}); + let game: GameVersion = serde_json::from_value(value.clone()).unwrap(); + assert_eq!(game.version, GameVersionName("1.21".into())); + assert_eq!(serde_json::to_value(game).unwrap(), value); + assert!(serde_json::from_value::(json!({"version": "1.21"})).is_err()); + } + + + #[test] + fn profiles_require_id_and_accept_extra_fields() { + let profile: GameLoaderProfile = serde_json::from_value(json!({ + "id": "fabric-loader-0.19.5-1.21", + "libraries": [] + })).unwrap(); + assert_eq!(profile.id, "fabric-loader-0.19.5-1.21"); + assert!(serde_json::from_value::(json!({})).is_err()); + } +} diff --git a/packages/modrinth-minecraft-meta/src/upstream/forge.rs b/packages/modrinth-minecraft-meta/src/upstream/forge.rs new file mode 100644 index 00000000000..c132184a919 --- /dev/null +++ b/packages/modrinth-minecraft-meta/src/upstream/forge.rs @@ -0,0 +1,23 @@ +use std::collections::HashMap; + +use derive_more::Display; +use serde::{Deserialize, Serialize}; + +use crate::model; + +pub const CATALOG_URL: &str = "https://files.minecraftforge.net/net/minecraftforge/forge/maven-metadata.json"; + +/// Forge loader manifest for all game versions, mapped to all loader versions +/// available for that game version. +/// +/// Available at [`CATALOG_URL`]. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Catalog( + pub HashMap>, +); + +/// Name of a Minecraft game version. +#[derive( + Debug, Display, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, +)] +pub struct GameVersionName(pub String); diff --git a/packages/modrinth-minecraft-meta/src/upstream/mod.rs b/packages/modrinth-minecraft-meta/src/upstream/mod.rs new file mode 100644 index 00000000000..082a9367a53 --- /dev/null +++ b/packages/modrinth-minecraft-meta/src/upstream/mod.rs @@ -0,0 +1,30 @@ +//! Schemas for files we download from our upstreams (e.g. Minecraft version +//! catalog manifest). + +pub mod fabric; +pub mod fabriclike; +pub mod forge; +pub mod mojang; +pub mod neoforge; +pub mod quilt; + +#[derive(Debug, Clone, Copy)] +pub struct Upstreams { + pub mojang: bool, + pub fabric: bool, + pub forge: bool, + pub neoforge: bool, + pub quilt: bool, +} + +impl Default for Upstreams { + fn default() -> Self { + Self { + mojang: true, + fabric: true, + forge: true, + neoforge: true, + quilt: true, + } + } +} diff --git a/packages/modrinth-minecraft-meta/src/upstream/mojang.rs b/packages/modrinth-minecraft-meta/src/upstream/mojang.rs new file mode 100644 index 00000000000..4dda5ec56a0 --- /dev/null +++ b/packages/modrinth-minecraft-meta/src/upstream/mojang.rs @@ -0,0 +1,286 @@ +use std::collections::HashMap; + +use anyhow::Result; +use jiff::Timestamp; +use serde::{Deserialize, Serialize}; +use url::Url; + +use crate::{ + model, + util::{Sha1, de_error}, +}; + +pub const CATALOG_URL: &str = + "https://piston-meta.mojang.com/mc/game/version_manifest_v2.json"; + +/// Catalog of all game versions. +/// +/// Available at [`CATALOG_URL`]. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Catalog { + pub latest: Latest, + pub versions: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Latest { + pub release: model::MinecraftVersionName, + pub snapshot: model::MinecraftVersionName, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Version { + pub id: model::MinecraftVersionName, + #[serde(rename = "type")] + pub ty: VersionType, + pub url: Url, + pub time: Timestamp, + pub release_time: Timestamp, + pub sha1: Sha1, + pub compliance_level: u32, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum VersionType { + Release, + Snapshot, + OldAlpha, + OldBeta, +} + +/// Manifest for a single game version. +/// +/// Available at [`Version::url`]. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct VersionManifest { + #[serde(skip_serializing_if = "Option::is_none")] + pub arguments: Option>>, + pub asset_index: AssetIndex, + pub assets: String, + pub downloads: HashMap, + pub id: model::MinecraftVersionName, + pub java_version: Option, + pub libraries: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub logging: Option>, + pub main_class: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub minecraft_arguments: Option, + pub minimum_launcher_version: u32, + pub release_time: Timestamp, + pub time: Timestamp, + #[serde(rename = "type")] + pub ty: VersionType, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AssetIndex { + pub id: String, + pub sha1: Sha1, + pub size: u32, + pub total_size: u32, + pub url: Url, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum DownloadType { + Client, + ClientMappings, + Server, + ServerMappings, + WindowsServer, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Download { + pub sha1: Sha1, + pub size: u32, + pub url: Url, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct JavaVersion { + pub component: String, + pub major_version: u32, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum ArgumentType { + Game, + Jvm, + DefaultUserJvm, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(untagged)] +pub enum Argument { + Normal(String), + Ruled { + #[serde(default)] + rules: Vec, + value: ArgumentValue, + }, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(untagged)] +pub enum ArgumentValue { + Single(String), + Many(Vec), +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Rule { + pub action: RuleAction, + #[serde(skip_serializing_if = "Option::is_none")] + pub os: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub features: Option, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RuleAction { + Allow, + Disallow, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct OperatingSystemRule { + #[serde(skip_serializing_if = "Option::is_none")] + pub name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub version: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub arch: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum OperatingSystem { + Osx, + OsxArm64, + Windows, + WindowsArm64, + Linux, + LinuxArm64, + LinuxArm32, + Unknown, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FeatureRule { + #[serde(skip_serializing_if = "Option::is_none")] + pub is_demo_user: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub has_custom_resolution: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub has_quick_plays_support: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub is_quick_play_singleplayer: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub is_quick_play_multiplayer: Option, + pub is_quick_play_realms: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Library { + #[serde(skip_serializing_if = "Option::is_none")] + pub downloads: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub extract: Option, + pub name: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub url: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub natives: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub rules: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub checksums: Option>, + #[serde(default = "default_true")] + pub include_in_classpath: bool, + #[serde(default = "default_true")] + pub downloadable: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LibraryDownloads { + #[serde(skip_serializing_if = "Option::is_none")] + pub artifact: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub classifiers: Option>, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LibraryDownload { + #[serde(skip_serializing_if = "Option::is_none")] + pub path: Option, + pub sha1: Sha1, + pub size: u32, + #[serde( + default, + deserialize_with = "deserialize_optional_url", + skip_serializing_if = "Option::is_none" + )] + pub url: Option, +} + +fn deserialize_optional_url<'de, D>( + deserializer: D, +) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + let value = Option::::deserialize(deserializer)?; + match value.as_deref() { + None | Some("") => Ok(None), + Some(value) => Url::parse(value).map(Some).map_err(de_error::), + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LibraryExtract { + #[serde(skip_serializing_if = "Option::is_none")] + pub exclude: Option>, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LoggingSide { + Client, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LogConfigDownload { + pub id: String, + pub sha1: Sha1, + pub size: u32, + pub url: Url, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde( + tag = "type", + rename_all = "kebab-case", + rename_all_fields = "camelCase" +)] +pub enum LoggingConfiguration { + Log4j2Xml { + argument: String, + file: LogConfigDownload, + }, +} + +fn default_true() -> bool { + true +} diff --git a/packages/modrinth-minecraft-meta/src/upstream/neoforge.rs b/packages/modrinth-minecraft-meta/src/upstream/neoforge.rs new file mode 100644 index 00000000000..7895868b5e9 --- /dev/null +++ b/packages/modrinth-minecraft-meta/src/upstream/neoforge.rs @@ -0,0 +1,17 @@ +use derive_more::Display; +use serde::{Deserialize, Serialize}; + +pub const FORGE_CATALOG_URL: &str = "https://maven.neoforged.net/api/maven/versions/releases/net/neoforged/forge"; +pub const NEOFORGE_CATALOG_URL: &str = "https://maven.neoforged.net/api/maven/versions/releases/net/neoforged/neoforge"; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Catalog { + pub is_snapshot: bool, + pub versions: Vec, +} + +#[derive( + Debug, Display, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, +)] +pub struct VersionName(pub String); diff --git a/packages/modrinth-minecraft-meta/src/upstream/quilt.rs b/packages/modrinth-minecraft-meta/src/upstream/quilt.rs new file mode 100644 index 00000000000..fa764a6b83f --- /dev/null +++ b/packages/modrinth-minecraft-meta/src/upstream/quilt.rs @@ -0,0 +1,95 @@ +use derive_more::Display; +use serde::{Deserialize, Serialize}; +use url::Url; + +use super::fabriclike::{GameVersion, GameVersionName, LoaderVersionName}; + +use crate::util::{MavenCoordinate, Sha1, Sha256}; + +pub const CATALOG_URL: &str = "https://meta.quiltmc.org/v3/versions"; +pub const PROFILE_BASE_URL: &str = + "https://meta.quiltmc.org/v3/versions/loader/"; +pub const MAVEN_URL: &str = "https://maven.quiltmc.org/repository/release/"; + +pub struct MetadataGroup { + pub id: String, + pub template: GameVersionName, + pub games: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Catalog { + pub game: Vec, + pub mappings: Vec, + pub hashed: Vec, + pub loader: Vec, + pub installer: Vec, +} + +#[derive( + Debug, Display, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, +)] +pub struct MappingVersionName(pub String); + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MappingVersion { + pub maven: MavenCoordinate, + pub version: MappingVersionName, + #[serde(rename = "gameVersion")] + pub game_version: GameVersionName, + pub build: u32, + pub separator: String, + pub hashed: HashedVersionName, + #[serde(rename = "file_size")] + pub file_size: u64, + pub hashes: Hashes, +} + +#[derive( + Debug, Display, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, +)] +pub struct HashedVersionName(pub String); + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct HashedVersion { + pub maven: MavenCoordinate, + pub version: HashedVersionName, + pub file_size: u64, + pub hashes: Hashes, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct LoaderVersion { + #[serde(default)] + pub stable: bool, + pub maven: MavenCoordinate, + pub version: LoaderVersionName, + pub build: u32, + pub separator: String, + pub file_size: u64, + pub hashes: Hashes, +} + +#[derive( + Debug, Display, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, +)] +pub struct InstallerVersionName(pub String); + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct InstallerVersion { + pub maven: MavenCoordinate, + pub version: InstallerVersionName, + pub url: Url, + pub file_size: u64, + pub hashes: Hashes, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Hashes { + pub sha1: Sha1, + pub sha256: Sha256, + pub sha512: String, +} diff --git a/packages/modrinth-minecraft-meta/src/util/json.rs b/packages/modrinth-minecraft-meta/src/util/json.rs new file mode 100644 index 00000000000..f4d6f890fae --- /dev/null +++ b/packages/modrinth-minecraft-meta/src/util/json.rs @@ -0,0 +1,37 @@ +use anyhow::{Context, anyhow}; +use serde::{Serialize, de::DeserializeOwned}; + +pub fn from_json_value( + value: &serde_json::Value, +) -> anyhow::Result { + serde_path_to_error::deserialize::<_, T>(value).map_err(|err| { + let path = err.path().to_string(); + let err = anyhow!(err.into_inner()); + anyhow!("at `{path}`: {err:#}") + }) +} + +pub fn from_json_slice(data: &[u8]) -> anyhow::Result { + let value = serde_json::from_slice::(data) + .with_context(|| { + anyhow!("invalid JSON\n\n{}", String::from_utf8_lossy(data)) + })?; + from_json_value(&value) +} + +pub fn from_json_str(str: &str) -> anyhow::Result { + let value = serde_json::from_str::(str) + .with_context(|| anyhow!("invalid JSON\n\n{str}"))?; + from_json_value(&value) +} + +pub fn to_json_string(value: &T) -> String { + serde_json::to_string(value).expect("serialization should never fail") +} + +pub fn de_error<'de, D: serde::Deserializer<'de>>( + err: impl Into, +) -> D::Error { + let err = err.into(); + serde::de::Error::custom(format!("{err:#}")) +} diff --git a/packages/modrinth-minecraft-meta/src/util/maven.rs b/packages/modrinth-minecraft-meta/src/util/maven.rs new file mode 100644 index 00000000000..09c1526368a --- /dev/null +++ b/packages/modrinth-minecraft-meta/src/util/maven.rs @@ -0,0 +1,310 @@ +use std::{str::FromStr, sync::LazyLock}; + +use anyhow::{Context, ensure}; +use derive_more::Display; +use regex::Regex; +use serde::{Deserialize, Serialize}; + +use crate::util::de_error; + +#[derive(Debug, Clone)] +pub struct MavenCoordinate { + pub group_id: CoordinatePart, + pub artifact_id: CoordinatePart, + pub version: CoordinatePart, + pub classifier: Option, + pub extension: Option, +} + +impl MavenCoordinate { + pub fn to_maven_path(&self) -> String { + let group_id = self.group_id.as_str().replace('.', "/"); + let artifact_id = self.artifact_id.as_str(); + let version = self.version.as_str(); + let classifier = self + .classifier + .as_ref() + .map(|classifier| format!("-{classifier}")) + .unwrap_or_default(); + let extension = self + .extension + .as_ref() + .map(CoordinatePart::as_str) + .unwrap_or("jar"); + format!( + "maven/{group_id}/{artifact_id}/{version}/{artifact_id}-{version}{classifier}.{extension}" + ) + } + + pub fn from_maven_path(path: &str) -> anyhow::Result { + let relative = path + .strip_prefix("maven/") + .context("missing maven/ prefix")?; + let (directories, filename) = relative + .rsplit_once('/') + .context("missing artifact filename")?; + let (directories, version) = directories + .rsplit_once('/') + .context("missing version directory")?; + let (group, artifact_id) = directories + .rsplit_once('/') + .context("missing artifact directory")?; + for segment in group.split('/') { + segment + .parse::() + .context("invalid group directory")?; + ensure!( + !segment.contains('.'), + "group directory contains a dot: {segment:?}" + ); + } + let group_id = group + .replace('/', ".") + .parse() + .context("invalid group ID")?; + let artifact_id: CoordinatePart = + artifact_id.parse().context("invalid artifact ID")?; + let version: CoordinatePart = + version.parse().context("invalid version")?; + let prefix = format!("{artifact_id}-{version}"); + let suffix = filename + .strip_prefix(&prefix) + .context("filename does not match artifact and version")?; + let (classifier, extension) = + if let Some(extension) = suffix.strip_prefix('.') { + (None, extension) + } else { + let suffix = suffix + .strip_prefix('-') + .context("invalid artifact filename suffix")?; + let (classifier, extension) = suffix + .rsplit_once('.') + .context("missing artifact extension")?; + ( + Some(classifier.parse().context("invalid classifier")?), + extension, + ) + }; + Ok(Self { + group_id, + artifact_id, + version, + classifier, + extension: Some(extension.parse().context("invalid extension")?), + }) + } +} + +impl std::fmt::Display for MavenCoordinate { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}:{}:{}", self.group_id, self.artifact_id, self.version)?; + if let Some(classifier) = &self.classifier { + write!(f, ":{classifier}")?; + } + if let Some(extension) = &self.extension { + write!(f, "@{extension}")?; + } + Ok(()) + } +} + +#[derive(Debug, Display, Clone, PartialEq, Eq, Hash)] +pub struct CoordinatePart(String); + +impl CoordinatePart { + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl FromStr for CoordinatePart { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + static PATTERN: LazyLock = LazyLock::new(|| { + Regex::new(r"\A[A-Za-z0-9_+ -]+(?:\.[A-Za-z0-9_+ -]+)*\z") + .expect("valid coordinate part regex") + }); + ensure!(PATTERN.is_match(s), "invalid Maven coordinate part: {s:?}"); + + Ok(Self(s.to_owned())) + } +} + +impl FromStr for MavenCoordinate { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + let (coordinate, extension) = match s.split_once('@') { + Some((coordinate, extension)) => ( + coordinate, + Some(extension.parse().context("invalid extension")?), + ), + None => (s, None), + }; + let (group_id, remaining) = coordinate + .split_once(':') + .context("missing group ID separator")?; + let (artifact_id, remaining) = remaining + .split_once(':') + .context("missing artifact ID separator")?; + let (version, classifier) = match remaining.split_once(':') { + Some((version, classifier)) => ( + version, + Some(classifier.parse().context("invalid classifier")?), + ), + None => (remaining, None), + }; + Ok(Self { + group_id: group_id.parse().context("invalid group ID")?, + artifact_id: artifact_id.parse().context("invalid artifact ID")?, + version: version.parse().context("invalid version")?, + classifier, + extension, + }) + } +} + +impl Serialize for MavenCoordinate { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + self.to_string().serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for MavenCoordinate { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + s.parse::().map_err(de_error::) + } +} + +#[cfg(test)] +mod tests { + use super::{CoordinatePart, MavenCoordinate}; + + #[test] + fn maven_paths_round_trip() { + for (input, path) in [ + ( + "net.fabricmc:intermediary:1.14.2 Pre-Release 4", + "maven/net/fabricmc/intermediary/1.14.2 Pre-Release 4/intermediary-1.14.2 Pre-Release 4.jar", + ), + ( + "net.fabricmc:intermediary:3D Shareware v1.34", + "maven/net/fabricmc/intermediary/3D Shareware v1.34/intermediary-3D Shareware v1.34.jar", + ), + ( + "net.fabricmc:intermediary:1.21.1+build.3", + "maven/net/fabricmc/intermediary/1.21.1+build.3/intermediary-1.21.1+build.3.jar", + ), + ( + "net.minecraftforge:forge:1.21.8-58.1.22", + "maven/net/minecraftforge/forge/1.21.8-58.1.22/forge-1.21.8-58.1.22.jar", + ), + ( + "com.example:tool:1:client@lzma", + "maven/com/example/tool/1/tool-1-client.lzma", + ), + ( + "com.example:tool:1:client.extra@jar", + "maven/com/example/tool/1/tool-1-client.extra.jar", + ), + ( + "com.example:tool:1@zip", + "maven/com/example/tool/1/tool-1.zip", + ), + ] { + let coordinate: MavenCoordinate = input.parse().unwrap(); + assert_eq!(coordinate.to_maven_path(), path); + assert_eq!( + MavenCoordinate::from_maven_path(path) + .unwrap() + .to_maven_path(), + path + ); + } + } + + #[test] + fn invalid_maven_paths_are_rejected() { + for path in [ + "maven/../tool/1/tool-1.jar", + "maven/com//example/tool/1/tool-1.jar", + "maven/com.example/tool/1/tool-1.jar", + "maven/com/example/tool/1/other-1.jar", + "maven/com/example/tool/1/tool-1-.jar", + "maven/com/example/tool/1/tool-1", + "/maven/com/example/tool/1/tool-1.jar", + ] { + assert!( + MavenCoordinate::from_maven_path(path).is_err(), + "accepted {path:?}" + ); + } + } + + #[test] + fn coordinate_parts_are_validated() { + for value in [ + "net.minecraftforge", + "fabric-loader", + "1.21.8-58.1.22", + "a_b", + "1+2", + "1.21.1+build.3", + "1.14.2 Pre-Release 4", + "3D Shareware v1.34", + ] { + assert_eq!( + value.parse::().unwrap().as_str(), + value + ); + } + for value in [ + "", ".", "..", ".foo", "foo.", "foo..bar", "a/b", "a\\b", "a:b", + "a\t", "a\n", + ] { + assert!( + value.parse::().is_err(), + "accepted {value:?}" + ); + } + } + + #[test] + fn coordinates_round_trip_and_validate_each_field() { + for input in [ + "net.minecraftforge:forge:1.21.8-58.1.22", + "net.minecraftforge:forge:1.21.8-58.1.22:universal", + "net.minecraftforge:forge:1.21.8-58.1.22@zip", + "net.minecraftforge:forge:1.21.8-58.1.22:client@lzma", + "net.fabricmc:intermediary:1.21.1+build.3", + "net.fabricmc:intermediary:1.14.2 Pre-Release 4", + "net.fabricmc:intermediary:3D Shareware v1.34", + ] { + let coordinate: MavenCoordinate = + serde_json::from_value(serde_json::json!(input)).unwrap(); + assert_eq!(coordinate.to_string(), input); + assert_eq!(serde_json::to_value(coordinate).unwrap(), input); + } + for value in [ + "bad/group:forge:1", + "net.minecraftforge:..:1", + "net.minecraftforge:forge:", + "g:a:1:", + "g:a:1@", + "g:a:1:c:extra", + "g:a:1@jar@zip", + "g:a:1:../client", + "g:a:1@../jar", + ] { + assert!(value.parse::().is_err()); + } + } +} diff --git a/packages/modrinth-minecraft-meta/src/util/mod.rs b/packages/modrinth-minecraft-meta/src/util/mod.rs new file mode 100644 index 00000000000..726c9e764b5 --- /dev/null +++ b/packages/modrinth-minecraft-meta/src/util/mod.rs @@ -0,0 +1,11 @@ +mod json; +mod maven; +mod sha1; +mod sha256; +mod time; + +pub use json::*; +pub use maven::*; +pub use sha1::*; +pub use sha256::*; +pub use time::*; diff --git a/packages/modrinth-minecraft-meta/src/util/sha1.rs b/packages/modrinth-minecraft-meta/src/util/sha1.rs new file mode 100644 index 00000000000..9b7ad3eca1e --- /dev/null +++ b/packages/modrinth-minecraft-meta/src/util/sha1.rs @@ -0,0 +1,132 @@ +use std::fmt; + +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use sha2::Digest; +use toasty::{ + schema::{Field, Load}, + stmt::{Assign, Assignment, Expr, IntoExpr, List, Path, Type, Value}, +}; + +use crate::util::de_error; + +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Sha1(pub [u8; 20]); + +impl Sha1 { + #[must_use] + pub fn from_digest(data: impl AsRef<[u8]>) -> Self { + let sha1 = sha1::Sha1::digest(data); + let sha1 = sha1.as_array::<20>().expect("should be 20 bytes long"); + Self(*sha1) + } +} + +impl fmt::Debug for Sha1 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&hex::encode(self.0)) + } +} + +impl fmt::Display for Sha1 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&hex::encode(self.0)) + } +} + +impl Serialize for Sha1 { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(&hex::encode(self.0)) + } +} + +impl<'de> Deserialize<'de> for Sha1 { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let value = String::deserialize(deserializer)?; + let mut bytes = [0; 20]; + hex::decode_to_slice(value, &mut bytes).map_err(de_error::)?; + Ok(Self(bytes)) + } +} + +impl Load for Sha1 { + type Output = Self; + + fn ty() -> Type { + Type::String + } + + fn load(value: Value) -> toasty::Result { + match value { + Value::String(value) => { + let mut bytes = [0; 20]; + hex::decode_to_slice(value, &mut bytes).map_err(|error| { + toasty::Error::from_args(format_args!( + "invalid SHA-1 digest: {error}" + )) + })?; + Ok(Self(bytes)) + } + _ => Err(toasty::Error::type_conversion(value, "Sha1")), + } + } + + fn reload(target: &mut Self, value: Value) -> toasty::Result<()> { + *target = Self::load(value)?; + Ok(()) + } +} + +impl Field for Sha1 { + type ExprTarget = Self; + type Path = Path; + type ListPath = Path>; + type Update<'a> = (); + type Inner = Self; + + fn new_path(path: Path) -> Self::Path { + path + } + + fn new_list_path( + path: Path>, + ) -> Self::ListPath { + path + } + + fn new_update<'a>( + _assignments: &'a mut toasty::codegen_support::core::stmt::Assignments, + _projection: toasty::stmt::Projection, + ) -> Self::Update<'a> { + } + + fn key_constraint( + &self, + target: Path, + ) -> Expr { + target.eq(self) + } +} + +impl IntoExpr for Sha1 { + fn into_expr(self) -> Expr { + Expr::from_untyped(Value::String(hex::encode(self.0))) + } + + fn by_ref(&self) -> Expr { + Expr::from_untyped(Value::String(hex::encode(self.0))) + } +} + +impl Assign for Sha1 { + fn into_assignment(self) -> Assignment { + toasty::stmt::set(self.into_expr()) + } +} + +impl toasty::codegen_support::index::IndexableField for Sha1 {} diff --git a/packages/modrinth-minecraft-meta/src/util/sha256.rs b/packages/modrinth-minecraft-meta/src/util/sha256.rs new file mode 100644 index 00000000000..ce901ac039b --- /dev/null +++ b/packages/modrinth-minecraft-meta/src/util/sha256.rs @@ -0,0 +1,132 @@ +use std::fmt; + +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use sha2::Digest; +use toasty::{ + schema::{Field, Load}, + stmt::{Assign, Assignment, Expr, IntoExpr, List, Path, Type, Value}, +}; + +use crate::util::de_error; + +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Sha256(pub [u8; 32]); + +impl Sha256 { + #[must_use] + pub fn from_digest(data: impl AsRef<[u8]>) -> Self { + let sha256 = sha2::Sha256::digest(data); + let sha256 = sha256.as_array::<32>().expect("should be 32 bytes long"); + Self(*sha256) + } +} + +impl fmt::Debug for Sha256 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&hex::encode(self.0)) + } +} + +impl fmt::Display for Sha256 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&hex::encode(self.0)) + } +} + +impl Serialize for Sha256 { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(&hex::encode(self.0)) + } +} + +impl<'de> Deserialize<'de> for Sha256 { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let value = String::deserialize(deserializer)?; + let mut bytes = [0; 32]; + hex::decode_to_slice(value, &mut bytes).map_err(de_error::)?; + Ok(Self(bytes)) + } +} + +impl Load for Sha256 { + type Output = Self; + + fn ty() -> Type { + Type::String + } + + fn load(value: Value) -> toasty::Result { + match value { + Value::String(value) => { + let mut bytes = [0; 32]; + hex::decode_to_slice(value, &mut bytes).map_err(|error| { + toasty::Error::from_args(format_args!( + "invalid SHA-256 digest: {error}" + )) + })?; + Ok(Self(bytes)) + } + _ => Err(toasty::Error::type_conversion(value, "Sha256")), + } + } + + fn reload(target: &mut Self, value: Value) -> toasty::Result<()> { + *target = Self::load(value)?; + Ok(()) + } +} + +impl Field for Sha256 { + type ExprTarget = Self; + type Path = Path; + type ListPath = Path>; + type Update<'a> = (); + type Inner = Self; + + fn new_path(path: Path) -> Self::Path { + path + } + + fn new_list_path( + path: Path>, + ) -> Self::ListPath { + path + } + + fn new_update<'a>( + _assignments: &'a mut toasty::codegen_support::core::stmt::Assignments, + _projection: toasty::stmt::Projection, + ) -> Self::Update<'a> { + } + + fn key_constraint( + &self, + target: Path, + ) -> Expr { + target.eq(self) + } +} + +impl IntoExpr for Sha256 { + fn into_expr(self) -> Expr { + Expr::from_untyped(Value::String(hex::encode(self.0))) + } + + fn by_ref(&self) -> Expr { + Expr::from_untyped(Value::String(hex::encode(self.0))) + } +} + +impl Assign for Sha256 { + fn into_assignment(self) -> Assignment { + toasty::stmt::set(self.into_expr()) + } +} + +impl toasty::codegen_support::index::IndexableField for Sha256 {} diff --git a/packages/modrinth-minecraft-meta/src/util/time.rs b/packages/modrinth-minecraft-meta/src/util/time.rs new file mode 100644 index 00000000000..6db865f40d4 --- /dev/null +++ b/packages/modrinth-minecraft-meta/src/util/time.rs @@ -0,0 +1,67 @@ +use jiff::{Timestamp, civil::DateTime, tz::TimeZone}; +use serde::{Deserialize, Deserializer}; + +/// Parses timestamps with an explicit offset, or assumes UTC for offsetless datetimes. +pub fn deserialize_timestamp<'de, D>( + deserializer: D, +) -> Result +where + D: Deserializer<'de>, +{ + let value = String::deserialize(deserializer)?; + value + .parse::() + .or_else(|_| { + if !value.split_once('T').is_some_and(|(_, time)| { + time.contains(':') + && time.bytes().all(|byte| { + byte.is_ascii_digit() || matches!(byte, b':' | b'.') + }) + }) { + return value.parse::(); + } + value + .parse::()? + .to_zoned(TimeZone::UTC) + .map(|datetime| datetime.timestamp()) + }) + .map_err(serde::de::Error::custom) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[derive(Deserialize)] + struct Example { + #[serde(deserialize_with = "deserialize_timestamp")] + time: Timestamp, + } + + #[test] + fn explicit_offsets_are_preserved() { + for value in ["2026-10-11T19:45:54Z", "2026-10-11T19:45:54+02:00"] { + let example: Example = + serde_json::from_value(serde_json::json!({"time": value})) + .unwrap(); + assert_eq!(example.time, value.parse::().unwrap()); + } + } + + #[test] + fn invalid_timestamps_are_rejected() { + for value in [ + "invalid", + "2026-10-11", + "2026-99-11T19:45:54", + "2026-10-11T19:45:54+99:00", + ] { + assert!( + serde_json::from_value::( + serde_json::json!({"time": value}) + ) + .is_err() + ); + } + } +} diff --git a/packages/sqlx-tracing/Cargo.toml b/packages/sqlx-tracing/Cargo.toml index 3ce51c659a8..e741e3c60a9 100644 --- a/packages/sqlx-tracing/Cargo.toml +++ b/packages/sqlx-tracing/Cargo.toml @@ -14,9 +14,9 @@ categories = ["asynchronous", "database", "development-tools::debugging", "devel [dependencies] derive_more = { workspace = true, features = ["deref", "deref_mut"] } -futures = { version = "0.3" } -sqlx = { version = "0.8", default-features = false, features = ["derive"] } -tracing = { version = "0.1" } +futures = { workspace = true } +sqlx = { workspace = true, default-features = false, features = ["derive"] } +tracing = { workspace = true } [dev-dependencies] opentelemetry = "0.30" diff --git a/packages/toasty/.cargo_vcs_info.json b/packages/toasty/.cargo_vcs_info.json new file mode 100644 index 00000000000..acc838cf478 --- /dev/null +++ b/packages/toasty/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "d71114e4101ce33ee5272ef2bd69ac6d5a6d749a" + }, + "path_in_vcs": "crates/toasty" +} \ No newline at end of file diff --git a/packages/toasty/CHANGELOG.md b/packages/toasty/CHANGELOG.md new file mode 100644 index 00000000000..b48a0d63433 --- /dev/null +++ b/packages/toasty/CHANGELOG.md @@ -0,0 +1,654 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +## [0.11.0](https://github.com/tokio-rs/toasty/compare/toasty-v0.10.0...toasty-v0.11.0) - 2026-09-24 + +### Added + +- Preload relations inside embedded types ([#1228]) +- Explicit support for MariaDB ([#1227]) +- Accept borrowed values in JSON setters ([#1213]) +- [**breaking**] Turso serverless deployments now connect over HTTP ([#1211]) +- Reference embedded relations with model values ([#1172]) +- Support `jiff::tz::TimeZone` as a model field type ([#1221]) +- PostgreSQL support for unique Vec constraints ([#1199]) +- Conditional execution in query expressions ([#1182]) + +### Fixed + +- `Deferred` now derives `PartialEq`, `Eq`, and `Hash` ([#1238]) +- Variant-rooted includes now work on update queries ([#1237]) +- Enum fields now resolve by variant-local index correctly ([#1218]) +- `Not` operator now works in eval verify and type inference ([#1214]) +- `Connect` now reads `max_connection` from driver configuration ([#1195]) +- Fields named `path` and `from_path` now work in macros ([#1196]) +- [**breaking**] Batch insert no longer infers IDs on MySQL ([#1194]) + +### Changed + +- [**breaking**] Generated model fields now preserve target types ([#1249]) + +[#1172]: https://github.com/tokio-rs/toasty/pull/1172 +[#1182]: https://github.com/tokio-rs/toasty/pull/1182 +[#1194]: https://github.com/tokio-rs/toasty/pull/1194 +[#1195]: https://github.com/tokio-rs/toasty/pull/1195 +[#1196]: https://github.com/tokio-rs/toasty/pull/1196 +[#1199]: https://github.com/tokio-rs/toasty/pull/1199 +[#1211]: https://github.com/tokio-rs/toasty/pull/1211 +[#1213]: https://github.com/tokio-rs/toasty/pull/1213 +[#1214]: https://github.com/tokio-rs/toasty/pull/1214 +[#1218]: https://github.com/tokio-rs/toasty/pull/1218 +[#1221]: https://github.com/tokio-rs/toasty/pull/1221 +[#1227]: https://github.com/tokio-rs/toasty/pull/1227 +[#1228]: https://github.com/tokio-rs/toasty/pull/1228 +[#1237]: https://github.com/tokio-rs/toasty/pull/1237 +[#1238]: https://github.com/tokio-rs/toasty/pull/1238 +[#1249]: https://github.com/tokio-rs/toasty/pull/1249 + +## [0.10.0](https://github.com/tokio-rs/toasty/compare/toasty-v0.9.0...toasty-v0.10.0) - 2026-08-11 + +### Added + +- Re-export external statement value types ([#1181]) +- *(macros)* Generate asc/desc on newtype embed fields ([#1180]) +- Add network address types ([#1178]) +- Accept #[belongs_to] fields in embedded types ([#1170]) +- Embed migrations in application binaries ([#1095]) + +### Fixed + +- Add wasm32 build support ([#1173]) +- *(mysql)* [**breaking**] Make TLS features additive with SQLx ([#1147]) +- *(engine)* Omit previous cursor on first page ([#1153]) +- *(engine)* Fix IN-subquery statement optimization ([#1138]) +- *(engine)* Preserve pagination cursors through includes ([#1152]) +- Preserve query offset when selecting one row ([#1151]) +- *(engine)* Exclude null foreign keys from relation subqueries ([#1148]) +- *(engine)* Make cursor pagination deterministic ([#1142]) +- Resolve authority-form sqlite and turso URLs to a file path ([#1127]) +- *(engine)* Fix newtype foreign keys in included relations ([#1137]) +- Support pagination with multiple order by keys ([#1124]) + +### Changed + +- *(core)* [**breaking**] Name the SQL dialect on `Capability::sql` ([#1155]) +- [**breaking**] Remove unused schema and statement APIs ([#1149]) + +[#1095]: https://github.com/tokio-rs/toasty/pull/1095 +[#1124]: https://github.com/tokio-rs/toasty/pull/1124 +[#1127]: https://github.com/tokio-rs/toasty/pull/1127 +[#1137]: https://github.com/tokio-rs/toasty/pull/1137 +[#1138]: https://github.com/tokio-rs/toasty/pull/1138 +[#1142]: https://github.com/tokio-rs/toasty/pull/1142 +[#1147]: https://github.com/tokio-rs/toasty/pull/1147 +[#1148]: https://github.com/tokio-rs/toasty/pull/1148 +[#1149]: https://github.com/tokio-rs/toasty/pull/1149 +[#1151]: https://github.com/tokio-rs/toasty/pull/1151 +[#1152]: https://github.com/tokio-rs/toasty/pull/1152 +[#1153]: https://github.com/tokio-rs/toasty/pull/1153 +[#1155]: https://github.com/tokio-rs/toasty/pull/1155 +[#1170]: https://github.com/tokio-rs/toasty/pull/1170 +[#1173]: https://github.com/tokio-rs/toasty/pull/1173 +[#1178]: https://github.com/tokio-rs/toasty/pull/1178 +[#1180]: https://github.com/tokio-rs/toasty/pull/1180 +[#1181]: https://github.com/tokio-rs/toasty/pull/1181 + +## [0.9.0](https://github.com/tokio-rs/toasty/compare/toasty-v0.8.0...toasty-v0.9.0) - 2026-07-23 + +### Added + +- support order_by in includes ([#1109]) +- relation link/unlink return a builder instead of executing eagerly ([#1118]) +- support serde_json::Value fields ([#1116]) +- support native JSON and JSONB column storage ([#1114]) +- [**breaking**] require explicit column types for JSON fields ([#1106]) +- support temporal Vec fields ([#1105]) +- filter associations in include ([#1089]) +- introduce Expr::Static for inline SQL literals, hook up LIMIT/OFFSET ([#1001]) +- support integer storage for enum discriminants ([#1101]) +- add upsert support ([#1091]) +- add #[shared] variant fields and enum-level #[index]/#[unique] ([#1078]) +- implement Scalar for unit enum embeds ([#1082]) +- add #[document] storage for embedded types with nested-path filtering ([#1028]) + +### Fixed + +- type indexed key discovery as primary keys on DynamoDB ([#1113]) +- serialize unconstrained numeric migration snapshots ([#1115]) +- roll back transactions when finalization fails ([#1102]) +- support any() on many-to-many relations ([#1097]) +- store Vec as a native enum array on Postgres ([#1092]) +- compare composite-FK include filter against target fields ([#1086]) +- return None for optional belongs_to with NULL foreign key ([#1090]) +- lower IN-list over an embedded-field projection ([#1084]) + +[#1001]: https://github.com/tokio-rs/toasty/pull/1001 +[#1028]: https://github.com/tokio-rs/toasty/pull/1028 +[#1078]: https://github.com/tokio-rs/toasty/pull/1078 +[#1082]: https://github.com/tokio-rs/toasty/pull/1082 +[#1084]: https://github.com/tokio-rs/toasty/pull/1084 +[#1086]: https://github.com/tokio-rs/toasty/pull/1086 +[#1089]: https://github.com/tokio-rs/toasty/pull/1089 +[#1090]: https://github.com/tokio-rs/toasty/pull/1090 +[#1091]: https://github.com/tokio-rs/toasty/pull/1091 +[#1092]: https://github.com/tokio-rs/toasty/pull/1092 +[#1097]: https://github.com/tokio-rs/toasty/pull/1097 +[#1101]: https://github.com/tokio-rs/toasty/pull/1101 +[#1102]: https://github.com/tokio-rs/toasty/pull/1102 +[#1105]: https://github.com/tokio-rs/toasty/pull/1105 +[#1106]: https://github.com/tokio-rs/toasty/pull/1106 +[#1109]: https://github.com/tokio-rs/toasty/pull/1109 +[#1113]: https://github.com/tokio-rs/toasty/pull/1113 +[#1114]: https://github.com/tokio-rs/toasty/pull/1114 +[#1115]: https://github.com/tokio-rs/toasty/pull/1115 +[#1116]: https://github.com/tokio-rs/toasty/pull/1116 +[#1118]: https://github.com/tokio-rs/toasty/pull/1118 + +## [0.8.0](https://github.com/tokio-rs/toasty/compare/toasty-v0.7.0...toasty-v0.8.0) - 2026-07-06 + +### Added + +- Emit one toasty::query event per statement and propagate caller spans ([#1071]) +- Support #[version] optimistic concurrency on SQL drivers ([#1065]) +- Infer `key` and `references` in `#[belongs_to]` ([#1063]) +- Share columns across enum variants via #[column("name")] ([#1064]) +- Add escape support for LIKE expressions ([#1039]) +- Add set_* replace-variants to Query builder ([#1037]) +- Implement serde serialization and deserialization for toasty::Json ([#1035]) +- Allow index on unit enum ([#1027]) +- Add between operator to query DSL ([#1029]) +- Support Option model fields ([#1021]) +- Support scalar terminal fields in has_many relations ([#1012]) + +### Fixed + +- Avoid panic when updating a mixed enum to a unit variant ([#1069]) +- Fix enum decoding for OR'd variant filters ([#1067]) +- Make multi-key delete and update consistent ([#1053]) +- Increment #[version] field on query-based updates ([#1022]) + +### Changed + +- [**breaking**] Make UpdateByKey returning columns explicit ([#1024]) +- [**breaking**] Rename RelationManyField/RelationOneField assoc type to Target ([#1015]) +- [**breaking**] Align stmt::Query with per-model Query ([#1011]) +- [**breaking**] Unify per-model query structs into Query ([#995]) +- [**breaking**] Remove the Register trait ([#1006]) +- Remove compile-time field validation from create! macro ([#997]) + +[#995]: https://github.com/tokio-rs/toasty/pull/995 +[#997]: https://github.com/tokio-rs/toasty/pull/997 +[#1006]: https://github.com/tokio-rs/toasty/pull/1006 +[#1011]: https://github.com/tokio-rs/toasty/pull/1011 +[#1012]: https://github.com/tokio-rs/toasty/pull/1012 +[#1015]: https://github.com/tokio-rs/toasty/pull/1015 +[#1021]: https://github.com/tokio-rs/toasty/pull/1021 +[#1022]: https://github.com/tokio-rs/toasty/pull/1022 +[#1024]: https://github.com/tokio-rs/toasty/pull/1024 +[#1027]: https://github.com/tokio-rs/toasty/pull/1027 +[#1029]: https://github.com/tokio-rs/toasty/pull/1029 +[#1035]: https://github.com/tokio-rs/toasty/pull/1035 +[#1037]: https://github.com/tokio-rs/toasty/pull/1037 +[#1039]: https://github.com/tokio-rs/toasty/pull/1039 +[#1053]: https://github.com/tokio-rs/toasty/pull/1053 +[#1063]: https://github.com/tokio-rs/toasty/pull/1063 +[#1064]: https://github.com/tokio-rs/toasty/pull/1064 +[#1065]: https://github.com/tokio-rs/toasty/pull/1065 +[#1067]: https://github.com/tokio-rs/toasty/pull/1067 +[#1069]: https://github.com/tokio-rs/toasty/pull/1069 +[#1071]: https://github.com/tokio-rs/toasty/pull/1071 + +## [0.7.0](https://github.com/tokio-rs/toasty/compare/toasty-v0.6.1...toasty-v0.7.0) - 2026-05-29 + +### Added + +- derive Clone and add Deserialize for Deferred ([#994](https://github.com/tokio-rs/toasty/pull/994)) +- *(macros)* generate field projection methods on Query/Many/One ([#987](https://github.com/tokio-rs/toasty/pull/987)) +- [**breaking**] add increment, decrement, add, subtract update ops ([#979](https://github.com/tokio-rs/toasty/pull/979)) +- *(macros)* add update! macro for concise field updates ([#980](https://github.com/tokio-rs/toasty/pull/980)) +- reject create() on multi-step relation scopes at compile time ([#978](https://github.com/tokio-rs/toasty/pull/978)) +- add raw SQL execution API ([#965](https://github.com/tokio-rs/toasty/pull/965)) +- remove the `#[deferred]` field attribute in favor of `Deferred` ([#961](https://github.com/tokio-rs/toasty/pull/961)) +- support eager relation fields ([#958](https://github.com/tokio-rs/toasty/pull/958)) +- *(engine)* support `.include()` of multi-step `via` relations ([#946](https://github.com/tokio-rs/toasty/pull/946)) +- expose migration core from toasty ([#944](https://github.com/tokio-rs/toasty/pull/944)) +- *(turso)* add Turso driver with TransactionMode-aware concurrent writes ([#938](https://github.com/tokio-rs/toasty/pull/938)) +- *(engine)* dispatch has-many `stmt::apply` batches per entry ([#932](https://github.com/tokio-rs/toasty/pull/932)) +- add TransactionMode for SQLite lock-acquisition control ([#931](https://github.com/tokio-rs/toasty/pull/931)) +- allow #[version] on tuple-newtype embeds of u64 ([#930](https://github.com/tokio-rs/toasty/pull/930)) +- *(sql)* serialize `SELECT DISTINCT` ([#934](https://github.com/tokio-rs/toasty/pull/934)) +- [**breaking**] replace `#[serialize(json)]` with `toasty::Json` wrapper ([#926](https://github.com/tokio-rs/toasty/pull/926)) +- expose primary-key type via Model::PrimaryKey ([#921](https://github.com/tokio-rs/toasty/pull/921)) +- *(engine)* fold simple Batch assignments in update lowering ([#917](https://github.com/tokio-rs/toasty/pull/917)) +- add multi-step (via) has_many and has_one relations ([#890](https://github.com/tokio-rs/toasty/pull/890)) +- add non-panicking `try_get` to relation types ([#918](https://github.com/tokio-rs/toasty/pull/918)) + +### Fixed + +- deserialize a present Deferred value as loaded ([#999](https://github.com/tokio-rs/toasty/pull/999)) +- *(engine)* lift relation-path LIKE into a foreign-key subquery ([#992](https://github.com/tokio-rs/toasty/pull/992)) +- *(engine)* lift relation-path IN-subquery through BelongsTo chains ([#990](https://github.com/tokio-rs/toasty/pull/990)) +- make starts_with case-sensitive on SQLite and MySQL ([#983](https://github.com/tokio-rs/toasty/pull/983)) +- *(engine)* handle ExprOr in eval verify_expr ([#959](https://github.com/tokio-rs/toasty/pull/959)) +- [**breaking**] scope `.ilike()` to PostgreSQL and document operator pass-through ([#937](https://github.com/tokio-rs/toasty/pull/937)) + +### Other + +- field projection methods on Query/Many/One ([#993](https://github.com/tokio-rs/toasty/pull/993)) +- *(engine)* move UpdateTarget::Query rewrite into lower ([#975](https://github.com/tokio-rs/toasty/pull/975)) +- simplify `Field` bounds now that `Load` is required ([#976](https://github.com/tokio-rs/toasty/pull/976)) +- [**breaking**] merge one relation field traits ([#971](https://github.com/tokio-rs/toasty/pull/971)) +- [**breaking**] delete Relation trait, tighten relation field shapes ([#967](https://github.com/tokio-rs/toasty/pull/967)) +- *(core)* split via relations into field variant ([#966](https://github.com/tokio-rs/toasty/pull/966)) +- *(core)* [**breaking**] merge has relation field variants ([#964](https://github.com/tokio-rs/toasty/pull/964)) +- [**breaking**] require Deferred relation fields ([#954](https://github.com/tokio-rs/toasty/pull/954)) +- gate sqlite connect doctest ([#953](https://github.com/tokio-rs/toasty/pull/953)) +- split relation field traits from targets ([#950](https://github.com/tokio-rs/toasty/pull/950)) +- unify lazy-slot relation encoding ([#949](https://github.com/tokio-rs/toasty/pull/949)) +- *(core)* [**breaking**] move schema diff types to `schema::diff` ([#929](https://github.com/tokio-rs/toasty/pull/929)) +- consolidate migration types in toasty crate and reorganize db::diff API ([#928](https://github.com/tokio-rs/toasty/pull/928)) +## [0.6.1](https://github.com/tokio-rs/toasty/compare/toasty-v0.6.0...toasty-v0.6.1) - 2026-05-16 + +### Added + +- Chain relation methods through multiple steps ([#903]) +- Order by multiple fields ([#901]) + +### Fixed + +- Composite foreign keys in relation traversal ([#915]) +- Belongs-to relations with embed-typed primary keys ([#912]) +- Queries with composite keys in equality conditions ([#906]) +- Multiple order_by expressions now stack correctly ([#899]) + +[#899]: https://github.com/tokio-rs/toasty/pull/899 +[#901]: https://github.com/tokio-rs/toasty/pull/901 +[#903]: https://github.com/tokio-rs/toasty/pull/903 +[#906]: https://github.com/tokio-rs/toasty/pull/906 +[#912]: https://github.com/tokio-rs/toasty/pull/912 +[#915]: https://github.com/tokio-rs/toasty/pull/915 + +## [0.6.0](https://github.com/tokio-rs/toasty/compare/toasty-v0.5.0...toasty-v0.6.0) - 2026-05-14 + +### Added + +- Vec model fields with push, extend, clear, pop, remove_at, and remove operations on PostgreSQL, MySQL, SQLite, and DynamoDB ([#866], [#872], [#880], [#887]) +- Connection pooling improvements: lifetime capping, idle time limits, and automatic broken connection detection and eviction ([#879], [#882], [#874], [#867]) +- Query capabilities: .select() projection through BelongsTo relations, per-call column projection, ilike() filter method, filtering by associated model fields, all-condition filters for associations, and latest_by queries ([#827], [#820], [#801], [#781], [#784], [#707]) +- Compile-time validation for column storage types and create! macro field sets ([#832], [#648]) +- Auto proxying through tuple-newtype Embed types ([#836]) +- #[deferred] field attribute and Deferred wrapper with embedded type support ([#793], [#799]) +- Backward pagination driver capability ([#757]) +- Full-table scan support on DynamoDB ([#821]) +- IN-list parameter optimization for PostgreSQL ([#818]) + +### Fixed + +- Compile-time validation for explicit auto strategies ([#851]) +- Record equality comparisons now properly apply type casting rules ([#838]) + +[#648]: https://github.com/tokio-rs/toasty/pull/648 +[#707]: https://github.com/tokio-rs/toasty/pull/707 +[#757]: https://github.com/tokio-rs/toasty/pull/757 +[#781]: https://github.com/tokio-rs/toasty/pull/781 +[#784]: https://github.com/tokio-rs/toasty/pull/784 +[#793]: https://github.com/tokio-rs/toasty/pull/793 +[#799]: https://github.com/tokio-rs/toasty/pull/799 +[#801]: https://github.com/tokio-rs/toasty/pull/801 +[#818]: https://github.com/tokio-rs/toasty/pull/818 +[#820]: https://github.com/tokio-rs/toasty/pull/820 +[#821]: https://github.com/tokio-rs/toasty/pull/821 +[#827]: https://github.com/tokio-rs/toasty/pull/827 +[#832]: https://github.com/tokio-rs/toasty/pull/832 +[#836]: https://github.com/tokio-rs/toasty/pull/836 +[#838]: https://github.com/tokio-rs/toasty/pull/838 +[#851]: https://github.com/tokio-rs/toasty/pull/851 +[#866]: https://github.com/tokio-rs/toasty/pull/866 +[#867]: https://github.com/tokio-rs/toasty/pull/867 +[#872]: https://github.com/tokio-rs/toasty/pull/872 +[#874]: https://github.com/tokio-rs/toasty/pull/874 +[#879]: https://github.com/tokio-rs/toasty/pull/879 +[#880]: https://github.com/tokio-rs/toasty/pull/880 +[#882]: https://github.com/tokio-rs/toasty/pull/882 +[#887]: https://github.com/tokio-rs/toasty/pull/887 + +## [0.5.0](https://github.com/tokio-rs/toasty/compare/toasty-v0.4.0...toasty-v0.5.0) - 2026-04-27 + +### Added + +- String prefix filter operators (`starts_with` and `LIKE`) ([#745]) +- Connection pool configuration ([#759]) +- Optimistic concurrency control for DynamoDB via `#[version]` attribute ([#694]) +- Pair attributes to disambiguate has_many/has_one relationships ([#746]) +- Limit::Offset pagination for DynamoDB ([#674]) +- Float field type support ([#687]) +- Native database enums for embedded enums ([#665]) +- Multiple external crate glob imports in models! macro ([#685]) + +### Fixed + +- Preserve non-reference constraints when lifting BelongsTo IN-subquery ([#777]) +- Deduplicate GetByKey input keys and strengthen HashIndex invariant ([#750]) +- Gate simplification rules on expression stability ([#703]) +- Nested includes sharing a prefix being overwritten ([#699]) +- [**breaking**] `.first()` returns the first row instead of panicking on multiple matches ([#693]) + +[#665]: https://github.com/tokio-rs/toasty/pull/665 +[#674]: https://github.com/tokio-rs/toasty/pull/674 +[#685]: https://github.com/tokio-rs/toasty/pull/685 +[#687]: https://github.com/tokio-rs/toasty/pull/687 +[#693]: https://github.com/tokio-rs/toasty/pull/693 +[#694]: https://github.com/tokio-rs/toasty/pull/694 +[#699]: https://github.com/tokio-rs/toasty/pull/699 +[#703]: https://github.com/tokio-rs/toasty/pull/703 +[#745]: https://github.com/tokio-rs/toasty/pull/745 +[#746]: https://github.com/tokio-rs/toasty/pull/746 +[#750]: https://github.com/tokio-rs/toasty/pull/750 +[#759]: https://github.com/tokio-rs/toasty/pull/759 +[#777]: https://github.com/tokio-rs/toasty/pull/777 + +## [0.4.0](https://github.com/tokio-rs/toasty/compare/toasty-v0.3.0...toasty-v0.4.0) - 2026-04-11 + +### Added + +- add support for newtype embedded structs ([#634](https://github.com/tokio-rs/toasty/pull/634)) +- auto-discover related models through fields ([#635](https://github.com/tokio-rs/toasty/pull/635)) +- support boxed and smart pointer foreign keys in has_many relations ([#630](https://github.com/tokio-rs/toasty/pull/630)) + +### Other + +- make FieldName::app_name optional to support unnamed fields ([#633](https://github.com/tokio-rs/toasty/pull/633)) + +## [0.3.0](https://github.com/tokio-rs/toasty/compare/toasty-v0.2.0...toasty-v0.3.0) - 2026-04-03 + +### Added + +- [**breaking**] bring back `db.transaction_builder()` API, add non-trait methods for `executor.transaction()` ([#625](https://github.com/tokio-rs/toasty/pull/625)) +- add IN list support to query macro filter expressions ([#605](https://github.com/tokio-rs/toasty/pull/605)) +- automatic global model discovery with `models!(crate::*)` using the `inventory` crate ([#614](https://github.com/tokio-rs/toasty/pull/614)) +- [**breaking**] add `ModelSet` and `models!` macro to replace `.register::()` ([#615](https://github.com/tokio-rs/toasty/pull/615)) +- add Assign trait and stmt combinators for unified update mutations ([#607](https://github.com/tokio-rs/toasty/pull/607)) +- eliminate redundant casts when field type matches target ([#612](https://github.com/tokio-rs/toasty/pull/612)) + +### Fixed + +- make Assignment Send + Sync by removing boxed closures ([#627](https://github.com/tokio-rs/toasty/pull/627)) +- remove bogus `impl> IntoExpr> for &T` ([#621](https://github.com/tokio-rs/toasty/pull/621)) +- replace broken BinaryOp::reverse with commute in index matching ([#611](https://github.com/tokio-rs/toasty/pull/611)) + +### Other + +- remove transaction_builder() from Db and Connection ([#622](https://github.com/tokio-rs/toasty/pull/622)) +- replace IntoExpr for &Option with Field::key_constraint ([#619](https://github.com/tokio-rs/toasty/pull/619)) +- push pagination handling into engine ([#610](https://github.com/tokio-rs/toasty/pull/610)) +- add badges to README ([#606](https://github.com/tokio-rs/toasty/pull/606)) +- update README examples to use create! macro syntax ([#603](https://github.com/tokio-rs/toasty/pull/603)) + +## [0.2.0](https://github.com/tokio-rs/toasty/compare/toasty-v0.1.1...toasty-v0.2.0) - 2026-03-30 + +### Added + +- implement string discriminants for embedded enums ([#580](https://github.com/tokio-rs/toasty/pull/580)) +- add tracing-based logging across the ORM ([#586](https://github.com/tokio-rs/toasty/pull/586)) +- add IntoAssignment trait and has-many update combinators ([#576](https://github.com/tokio-rs/toasty/pull/576)) +- add Path associated type and new_path/new_root_path methods to Model trait ([#574](https://github.com/tokio-rs/toasty/pull/574)) +- add Scope trait and implement it for HasMany ([#570](https://github.com/tokio-rs/toasty/pull/570)) +- implement basic query! macro with filter support ([#533](https://github.com/tokio-rs/toasty/pull/533)) +- add count() method to Query ([#534](https://github.com/tokio-rs/toasty/pull/534)) +- implement IntoExpr trait for Batch type ([#512](https://github.com/tokio-rs/toasty/pull/512)) +- support has-one conditional updates with existence checks on NoSQL drivers ([#506](https://github.com/tokio-rs/toasty/pull/506)) +- add pagination support for composite-key queries on NoSQL drivers ([#484](https://github.com/tokio-rs/toasty/pull/484)) +- add Bijection type for field-to-column mappings ([#433](https://github.com/tokio-rs/toasty/pull/433)) +- support update and delete statements in toasty::batch ([#428](https://github.com/tokio-rs/toasty/pull/428)) +- support create statements in toasty::batch ([#417](https://github.com/tokio-rs/toasty/pull/417)) +- implement batch queries for sending multiple independent queries in a single round-trip ([#411](https://github.com/tokio-rs/toasty/pull/411)) +- implement runtime serialization codegen for #[serialize(json)] fields ([#404](https://github.com/tokio-rs/toasty/pull/404)) +- add #[serialize] attribute bookkeeping and design doc ([#400](https://github.com/tokio-rs/toasty/pull/400)) +- create macro ([#398](https://github.com/tokio-rs/toasty/pull/398)) +- filter on embedded enum variants ([#389](https://github.com/tokio-rs/toasty/pull/389)) +- embedded enums with fields ([#381](https://github.com/tokio-rs/toasty/pull/381)) +- add support for limit(n) queries. ([#368](https://github.com/tokio-rs/toasty/pull/368)) +- embedded unit enums ([#355](https://github.com/tokio-rs/toasty/pull/355)) +- support embedded structs as field types ([#299](https://github.com/tokio-rs/toasty/pull/299)) +- better #[auto] handling for different types ([#262](https://github.com/tokio-rs/toasty/pull/262)) +- support auto incrementing IDs ([#192](https://github.com/tokio-rs/toasty/pull/192)) +- adds in list simplifications ([#251](https://github.com/tokio-rs/toasty/pull/251)) +- adds range to equality simplification ([#247](https://github.com/tokio-rs/toasty/pull/247)) +- adds canonicalization simplification ([#245](https://github.com/tokio-rs/toasty/pull/245)) +- adds boolean constant comparison simplifications ([#244](https://github.com/tokio-rs/toasty/pull/244)) +- adds complement law simplifications ([#243](https://github.com/tokio-rs/toasty/pull/243)) +- adds factoring simplifications ([#219](https://github.com/tokio-rs/toasty/pull/219)) +- adds the absorption law simplifications ([#218](https://github.com/tokio-rs/toasty/pull/218)) +- support smartpointers ([#233](https://github.com/tokio-rs/toasty/pull/233)) +- simplify is_null on non nullable fields ([#229](https://github.com/tokio-rs/toasty/pull/229)) +- adds idempotent law simplifications ([#217](https://github.com/tokio-rs/toasty/pull/217)) +- adds DeMorgan's law simplifications ([#216](https://github.com/tokio-rs/toasty/pull/216)) +- adds `ExprNot` ([#214](https://github.com/tokio-rs/toasty/pull/214)) +- introduces self-comparison simplifications ([#213](https://github.com/tokio-rs/toasty/pull/213)) +- introduces constant folding simplifications ([#212](https://github.com/tokio-rs/toasty/pull/212)) +- adds simplification for `ExprOr` statements ([#209](https://github.com/tokio-rs/toasty/pull/209)) +- adds `false` short-circuiting to the AND simplifications ([#206](https://github.com/tokio-rs/toasty/pull/206)) +- Implement fully dynamic model ID generation ([#130](https://github.com/tokio-rs/toasty/pull/130)) + +### Fixed + +- use weak dependency features for postgresql and mysql drivers ([#590](https://github.com/tokio-rs/toasty/pull/590)) +- strip ExprCast from IsNull/IsNotNull to fix is_none() panic on Option columns ([#584](https://github.com/tokio-rs/toasty/pull/584)) +- update error messages to reference ModelField instead of Field ([#565](https://github.com/tokio-rs/toasty/pull/565)) +- bring back associated type for `Model::Create` ([#555](https://github.com/tokio-rs/toasty/pull/555)) +- make create! macro syntax to be consistent with tuple and array ([#525](https://github.com/tokio-rs/toasty/pull/525)) +- reflect correct type in statement generics ([#517](https://github.com/tokio-rs/toasty/pull/517)) +- update IntoExpr comment to use List syntax ([#467](https://github.com/tokio-rs/toasty/pull/467)) +- error when creating a db with a bad url ([#452](https://github.com/tokio-rs/toasty/pull/452)) +- split composite KV filters in engine and enable batch update test ([#442](https://github.com/tokio-rs/toasty/pull/442)) +- enable ignored nested preload tests and DRY up set_returning_field ([#441](https://github.com/tokio-rs/toasty/pull/441)) +- correct batch_load_index for nested HasMany→HasOne inserts with auto-increment IDs ([#440](https://github.com/tokio-rs/toasty/pull/440)) +- lower ORDER BY expressions ([#439](https://github.com/tokio-rs/toasty/pull/439)) +- fix subtle cancellation safety issues ([#432](https://github.com/tokio-rs/toasty/pull/432)) +- preload HasOne> and refactor ExprLet to Vec bindings ([#402](https://github.com/tokio-rs/toasty/pull/402)) +- bound Relation by Sized ([#394](https://github.com/tokio-rs/toasty/pull/394)) +- DDB N+1 Select Fix ([#380](https://github.com/tokio-rs/toasty/pull/380)) +- Optimize O(N×M) association algorithm to O(N+M) for all relationship types ([#146](https://github.com/tokio-rs/toasty/pull/146)) + +### Other + +- upgrade dependencies ([#598](https://github.com/tokio-rs/toasty/pull/598)) +- configure readme field in workspace package metadata ([#597](https://github.com/tokio-rs/toasty/pull/597)) +- organize imports and format code for consistency ([#595](https://github.com/tokio-rs/toasty/pull/595)) +- update README status from Incubating to Preview ([#594](https://github.com/tokio-rs/toasty/pull/594)) +- replace CreateMany in create! macro with path-based nested builders ([#575](https://github.com/tokio-rs/toasty/pull/575)) +- add warn(missing_docs) to toasty crate and document all public items ([#573](https://github.com/tokio-rs/toasty/pull/573)) +- switch Assignments to BTreeMap with multi-value support ([#566](https://github.com/tokio-rs/toasty/pull/566)) +- replace ManyField/OneField with per-model fields structs ([#571](https://github.com/tokio-rs/toasty/pull/571)) +- make Connection public and Db stateless ([#569](https://github.com/tokio-rs/toasty/pull/569)) +- delete unused Field trait and rename ModelField to Field ([#568](https://github.com/tokio-rs/toasty/pull/568)) +- inline Create trait into Model and Relation traits ([#567](https://github.com/tokio-rs/toasty/pull/567)) +- relax Auto trait bound from Field to ModelField ([#563](https://github.com/tokio-rs/toasty/pull/563)) +- move numeric type impls from schema/field.rs to schema/num.rs ([#562](https://github.com/tokio-rs/toasty/pull/562)) +- rename PoolConnection to Connection and move to db/connection.rs ([#561](https://github.com/tokio-rs/toasty/pull/561)) +- remove Load and ModelField supertraits from Field trait ([#559](https://github.com/tokio-rs/toasty/pull/559)) +- use minimal trait bounds on wrapper-type blanket impls ([#558](https://github.com/tokio-rs/toasty/pull/558)) +- move reload from Field trait to Load trait ([#556](https://github.com/tokio-rs/toasty/pull/556)) +- extract RegisterField trait for schema registration concerns ([#553](https://github.com/tokio-rs/toasty/pull/553)) +- remove driver type re-exports from toasty::db except Driver and Capability ([#552](https://github.com/tokio-rs/toasty/pull/552)) +- extract Create trait from Model and Relation ([#549](https://github.com/tokio-rs/toasty/pull/549)) +- move ty() method from Field trait to Load trait ([#545](https://github.com/tokio-rs/toasty/pull/545)) +- remove `fn load` from Field trait, use Load supertrait ([#544](https://github.com/tokio-rs/toasty/pull/544)) +- reorganize db module exports and move executor/transaction ([#542](https://github.com/tokio-rs/toasty/pull/542)) +- remove async_trait reexport from toasty-core ([#539](https://github.com/tokio-rs/toasty/pull/539)) +- consolidate toasty-codegen into toasty-macros ([#536](https://github.com/tokio-rs/toasty/pull/536)) +- bump dependencies ([#530](https://github.com/tokio-rs/toasty/pull/530)) +- rm Path::new ([#528](https://github.com/tokio-rs/toasty/pull/528)) +- move batch, page, and create_many into stmt module ([#526](https://github.com/tokio-rs/toasty/pull/526)) +- rename unwrap methods to expect for consistency and clarity ([#518](https://github.com/tokio-rs/toasty/pull/518)) +- separate assignments from statement in update builder ([#516](https://github.com/tokio-rs/toasty/pull/516)) +- add API documentation to stmt module types ([#513](https://github.com/tokio-rs/toasty/pull/513)) +- remove unused union query functionality ([#515](https://github.com/tokio-rs/toasty/pull/515)) +- distinguish between single-row and multi-row updates with type system ([#514](https://github.com/tokio-rs/toasty/pull/514)) +- reorganize model and relation types under schema module ([#505](https://github.com/tokio-rs/toasty/pull/505)) +- replace wildcard import with explicit imports in schema module ([#493](https://github.com/tokio-rs/toasty/pull/493)) +- move driver types from toasty to toasty_core ([#492](https://github.com/tokio-rs/toasty/pull/492)) +- update code generation to use toasty::core namespace paths ([#491](https://github.com/tokio-rs/toasty/pull/491)) +- add comprehensive library documentation to toasty crate ([#489](https://github.com/tokio-rs/toasty/pull/489)) +- make Path generic over origin and target types ([#482](https://github.com/tokio-rs/toasty/pull/482)) +- simplify batch filtering with in_list helper and remove tuple IntoExpr impl ([#477](https://github.com/tokio-rs/toasty/pull/477)) +- remove unsafe ([#480](https://github.com/tokio-rs/toasty/pull/480)) +- fix API docs link and redesign crate index page ([#472](https://github.com/tokio-rs/toasty/pull/472)) +- rename terminal query method .all() to .exec() ([#471](https://github.com/tokio-rs/toasty/pull/471)) +- rename `in_set` to `in_list` ([#470](https://github.com/tokio-rs/toasty/pull/470)) +- add nightly API docs link to README ([#469](https://github.com/tokio-rs/toasty/pull/469)) +- add developer guide link to README ([#466](https://github.com/tokio-rs/toasty/pull/466)) +- replace mod.rs files with named module files ([#463](https://github.com/tokio-rs/toasty/pull/463)) +- rename toasty::stmt::Select to toasty::stmt::Query ([#460](https://github.com/tokio-rs/toasty/pull/460)) +- rename IntoStatement::Output to IntoStatement::Returning ([#459](https://github.com/tokio-rs/toasty/pull/459)) +- remove Cursor type and return Vec directly from query execution ([#448](https://github.com/tokio-rs/toasty/pull/448)) +- Add support for filtering parents by has_many associations ([#447](https://github.com/tokio-rs/toasty/pull/447)) +- add compile testing for documentation code snippets ([#446](https://github.com/tokio-rs/toasty/pull/446)) +- Allow tuples to be used as batch expressions for nested creates ([#443](https://github.com/tokio-rs/toasty/pull/443)) +- Add offset ([#438](https://github.com/tokio-rs/toasty/pull/438)) +- reuse connection bg-tasks ([#434](https://github.com/tokio-rs/toasty/pull/434)) +- DDB Client Creation Refactor ([#391](https://github.com/tokio-rs/toasty/pull/391)) +- Revert "feat: add Bijection type for field-to-column mappings ([#433](https://github.com/tokio-rs/toasty/pull/433))" ([#436](https://github.com/tokio-rs/toasty/pull/436)) +- Rename stmt::Primitive trait to model::Field and move Auto to model::Auto ([#416](https://github.com/tokio-rs/toasty/pull/416)) +- Dynamic batch support ([#415](https://github.com/tokio-rs/toasty/pull/415)) +- Add scope-depth-aware expression walker and refactor callers ([#414](https://github.com/tokio-rs/toasty/pull/414)) +- Use associated type on IntoStatement for output cardinality ([#412](https://github.com/tokio-rs/toasty/pull/412)) +- loosen M: Model bounds to M: Load or unconstrained ([#410](https://github.com/tokio-rs/toasty/pull/410)) +- give each trait its own file in the toasty crate ([#407](https://github.com/tokio-rs/toasty/pull/407)) +- extract Load trait from Model and simplify Cursor ([#406](https://github.com/tokio-rs/toasty/pull/406)) +- Support deeply nested association preloading ([#311](https://github.com/tokio-rs/toasty/pull/311)) +- remove unused ToStatement trait ([#405](https://github.com/tokio-rs/toasty/pull/405)) +- add missing single-level preload permutations ([#403](https://github.com/tokio-rs/toasty/pull/403)) +- remove Option impls from model.rs ([#393](https://github.com/tokio-rs/toasty/pull/393)) +- OR tautology elimination for enum variants ([#395](https://github.com/tokio-rs/toasty/pull/395)) +- Interactive transactions ([#376](https://github.com/tokio-rs/toasty/pull/376)) +- rm Arc from Schema.db field. ([#387](https://github.com/tokio-rs/toasty/pull/387)) +- add Expr::Error ([#385](https://github.com/tokio-rs/toasty/pull/385)) +- ExprMatch simplification ([#383](https://github.com/tokio-rs/toasty/pull/383)) +- Connection per Db ([#379](https://github.com/tokio-rs/toasty/pull/379)) +- Add transaction isolation levels and read-only mode to Operation::Transaction ([#375](https://github.com/tokio-rs/toasty/pull/375)) +- wrap multi-op ExecPlan in BEGIN...COMMIT for atomicity ([#370](https://github.com/tokio-rs/toasty/pull/370)) +- proptest fuzz testing for simplification pipeline ([#372](https://github.com/tokio-rs/toasty/pull/372)) +- mv simplification tests ([#369](https://github.com/tokio-rs/toasty/pull/369))s +- cleanup llm context files ([#365](https://github.com/tokio-rs/toasty/pull/365)) +- rm Expr::Key, ExprReference::Model already covers the case ([#359](https://github.com/tokio-rs/toasty/pull/359)) +- expression eval tests & improvements ([#358](https://github.com/tokio-rs/toasty/pull/358)) +- support OR queries over primary key ([#350](https://github.com/tokio-rs/toasty/pull/350)) +- index values in memory and use it for nested merge ([#352](https://github.com/tokio-rs/toasty/pull/352)) +- move Model.fields -> ModelRoot ([#347](https://github.com/tokio-rs/toasty/pull/347)) +- Fix #317 ([#342](https://github.com/tokio-rs/toasty/pull/342)) +- Add Primitive impl for Vec (Bytes) ([#344](https://github.com/tokio-rs/toasty/pull/344)) +- Add is_none() and is_some() filter methods for Option fields ([#337](https://github.com/tokio-rs/toasty/pull/337)) +- Remove dead enum code ([#341](https://github.com/tokio-rs/toasty/pull/341)) +- Migrate last tests to using errors. ([#339](https://github.com/tokio-rs/toasty/pull/339)) +- Remove Id type ([#334](https://github.com/tokio-rs/toasty/pull/334)) +- support partial updates for embedded structs ([#325](https://github.com/tokio-rs/toasty/pull/325)) +- add .not() method on Expr for NOT queries ([#315](https://github.com/tokio-rs/toasty/pull/315)) +- return errors from driver tests ([#319](https://github.com/tokio-rs/toasty/pull/319)) +- Add database reset functionality (+ serial tests) ([#322](https://github.com/tokio-rs/toasty/pull/322)) +- Support "postgres" URL scheme for PostgreSQL ([#320](https://github.com/tokio-rs/toasty/pull/320)) +- consolidate generated update builders ([#316](https://github.com/tokio-rs/toasty/pull/316)) +- switch assignments to be a map of Projection and not usize. ([#312](https://github.com/tokio-rs/toasty/pull/312)) +- Add database migration CLI tool ([#271](https://github.com/tokio-rs/toasty/pull/271)) +- support or queries ([#305](https://github.com/tokio-rs/toasty/pull/305)) +- allow embedded struct fields in queries ([#303](https://github.com/tokio-rs/toasty/pull/303)) +- move Capability to Driver ([#300](https://github.com/tokio-rs/toasty/pull/300)) +- remove anyhow dependency ([#297](https://github.com/tokio-rs/toasty/pull/297)) +- remove bail! ([#296](https://github.com/tokio-rs/toasty/pull/296)) +- misc tweaks ([#295](https://github.com/tokio-rs/toasty/pull/295)) +- rename some error types ([#292](https://github.com/tokio-rs/toasty/pull/292)) +- InvalidResultError ([#290](https://github.com/tokio-rs/toasty/pull/290)) +- add "too many records" error ([#289](https://github.com/tokio-rs/toasty/pull/289)) +- add condition failed error ([#286](https://github.com/tokio-rs/toasty/pull/286)) +- RecordNotFoundError ([#283](https://github.com/tokio-rs/toasty/pull/283)) +- introduce type conversion errors ([#282](https://github.com/tokio-rs/toasty/pull/282)) +- add a custom error type ([#279](https://github.com/tokio-rs/toasty/pull/279)) +- expose generated types when useful ([#278](https://github.com/tokio-rs/toasty/pull/278)) +- move more tests to the integration suite ([#274](https://github.com/tokio-rs/toasty/pull/274)) +- move more tests to the test suite ([#273](https://github.com/tokio-rs/toasty/pull/273)) +- move more tests to the integration suite ([#270](https://github.com/tokio-rs/toasty/pull/270)) +- Move more tests to the integration suite ([#268](https://github.com/tokio-rs/toasty/pull/268)) +- move more tests to integration suite. ([#265](https://github.com/tokio-rs/toasty/pull/265)) +- extract integration tests to a reusable crate ([#263](https://github.com/tokio-rs/toasty/pull/263)) +- Add database connection pooling ([#260](https://github.com/tokio-rs/toasty/pull/260)) +- Add bool primitive implementation ([#242](https://github.com/tokio-rs/toasty/pull/242)) +- Add native support for Postgres NUMERIC and MySQL DECIMAL with rust_decimal ([#248](https://github.com/tokio-rs/toasty/pull/248)) +- Update test function ([#241](https://github.com/tokio-rs/toasty/pull/241)) +- Add basic support for bigdecimal::BigDecimal ([#238](https://github.com/tokio-rs/toasty/pull/238)) +- add Primitive impl for Cow ([#232](https://github.com/tokio-rs/toasty/pull/232)) +- Implement backwards navigation for paginated queries ([#234](https://github.com/tokio-rs/toasty/pull/234)) +- Extract cursor from last item in paginated result set ([#231](https://github.com/tokio-rs/toasty/pull/231)) +- remove auto-mapping many models to one table ([#225](https://github.com/tokio-rs/toasty/pull/225)) +- Update readme ([#230](https://github.com/tokio-rs/toasty/pull/230)) +- Add Page as part of improved paginatation API ([#223](https://github.com/tokio-rs/toasty/pull/223)) +- uses Toasty macros for test model generation ([#211](https://github.com/tokio-rs/toasty/pull/211)) +- Adds high level docs to core engine components ([#205](https://github.com/tokio-rs/toasty/pull/205)) +- Adds docs for expression statements, tests for simplifications ([#204](https://github.com/tokio-rs/toasty/pull/204)) +- Add date/time times using `jiff` ([#201](https://github.com/tokio-rs/toasty/pull/201)) +- Use `Expr::Default` for model fields that have a `#[auto]` annotation ([#199](https://github.com/tokio-rs/toasty/pull/199)) +- Add `Expr::Default` ([#198](https://github.com/tokio-rs/toasty/pull/198)) +- rm normalize_insertion_values in favor of starting out Insert with the correct amount of fields ([#197](https://github.com/tokio-rs/toasty/pull/197)) +- rm impl Default for stmt::Expr ([#196](https://github.com/tokio-rs/toasty/pull/196)) +- more post-refactor cleanup ([#194](https://github.com/tokio-rs/toasty/pull/194)) +- Allow specifying more storage types, e.g. `TEXT` for UUIDs ([#181](https://github.com/tokio-rs/toasty/pull/181)) +- more post refactor cleanup ([#184](https://github.com/tokio-rs/toasty/pull/184)) +- more post-refactor cleanup ([#182](https://github.com/tokio-rs/toasty/pull/182)) +- Add support for UUIDs using the uuid crate ([#178](https://github.com/tokio-rs/toasty/pull/178)) +- Add support for specifying a different database name for fields ([#174](https://github.com/tokio-rs/toasty/pull/174)) +- combine plan and exec mods ([#173](https://github.com/tokio-rs/toasty/pull/173)) +- remove 2 from types and fns ([#172](https://github.com/tokio-rs/toasty/pull/172)) +- Add fixed-size Rust primitive type support ([#170](https://github.com/tokio-rs/toasty/pull/170)) +- remove `ng` module ([#171](https://github.com/tokio-rs/toasty/pull/171)) +- handle empty tables when preloading ([#168](https://github.com/tokio-rs/toasty/pull/168)) +- move rest of planner to new engine. ([#167](https://github.com/tokio-rs/toasty/pull/167)) +- Combine lowering with new planner's "decompose" step ([#164](https://github.com/tokio-rs/toasty/pull/164)) +- Add stmt::Filter + some refactors. ([#163](https://github.com/tokio-rs/toasty/pull/163)) +- Integrate KV select path with the new planner ([#162](https://github.com/tokio-rs/toasty/pull/162)) +- Include association eager-loading when lowering statement ([#159](https://github.com/tokio-rs/toasty/pull/159)) +- refactor lowering to use expr context ([#160](https://github.com/tokio-rs/toasty/pull/160)) +- unify ExprReference and ExprColumn ([#158](https://github.com/tokio-rs/toasty/pull/158)) +- update readme to align with the working code ([#112](https://github.com/tokio-rs/toasty/pull/112)) +- Add `serde::Serialize` support ([#143](https://github.com/tokio-rs/toasty/pull/143)) +- refactor ExprColumn to remove direct ColumnId references. ([#156](https://github.com/tokio-rs/toasty/pull/156)) +- (toasty): Derive on relation types ([#157](https://github.com/tokio-rs/toasty/pull/157)) +- stop hardcoding FieldId in expressions. ([#155](https://github.com/tokio-rs/toasty/pull/155)) +- don't hardcode ModelId in ExprReference ([#154](https://github.com/tokio-rs/toasty/pull/154)) +- Include all table refs at the top of a SourceTable ([#152](https://github.com/tokio-rs/toasty/pull/152)) +- track association includes on returning clause ([#150](https://github.com/tokio-rs/toasty/pull/150)) +- Update HasMany Debug impl ([#142](https://github.com/tokio-rs/toasty/pull/142)) +- implement HasOne preload support ([#144](https://github.com/tokio-rs/toasty/pull/144)) +- quick fix to support multiple includes with queries ([#140](https://github.com/tokio-rs/toasty/pull/140)) +- Enhance testing infrastructure and refactor tuple Like implementations ([#139](https://github.com/tokio-rs/toasty/pull/139)) +- add context documentation and reorganize project docs ([#137](https://github.com/tokio-rs/toasty/pull/137)) +- reduce glob imports in rest of crates ([#133](https://github.com/tokio-rs/toasty/pull/133)) +- use macro to generate repetitive Primitive implementations ([#131](https://github.com/tokio-rs/toasty/pull/131)) +- Switch Model::ID and Primitive::TYPE to methods to avoid const requirement ([#129](https://github.com/tokio-rs/toasty/pull/129)) +- Add support for unsigned types ([#122](https://github.com/tokio-rs/toasty/pull/122)) +- Unify ExprField and ExprReference ([#120](https://github.com/tokio-rs/toasty/pull/120)) +- Remove `Value::to_$ty() -> Result`. ([#117](https://github.com/tokio-rs/toasty/pull/117)) +- Add support for i16 ([#116](https://github.com/tokio-rs/toasty/pull/116)) +- Support i8 ([#115](https://github.com/tokio-rs/toasty/pull/115)) +- add support for i32 types ([#113](https://github.com/tokio-rs/toasty/pull/113)) +- Initial pagination implementation ([#111](https://github.com/tokio-rs/toasty/pull/111)) +- add support for "order by" ([#110](https://github.com/tokio-rs/toasty/pull/110)) +- rm Query from schema ([#109](https://github.com/tokio-rs/toasty/pull/109)) +- rm PartialEq from stmt and schema ([#108](https://github.com/tokio-rs/toasty/pull/108)) +- rename Relation2 -> Relation, codegen2 -> codegen ([#107](https://github.com/tokio-rs/toasty/pull/107)) +- Switch proc macro to `#[derive(Model)]` ([#105](https://github.com/tokio-rs/toasty/pull/105)) +- Add annotation to specify DB column type ([#104](https://github.com/tokio-rs/toasty/pull/104)) +- ran cargo `clippy --fix -- -Wclippy::use_self` ([#103](https://github.com/tokio-rs/toasty/pull/103)) +- Flatten capability struct and DRY db definitions ([#102](https://github.com/tokio-rs/toasty/pull/102)) +- switch `driver::Rows::Count` type to u64 ([#98](https://github.com/tokio-rs/toasty/pull/98)) +- complete driver, include in CI ([#97](https://github.com/tokio-rs/toasty/pull/97)) +- first pass at MySQL driver ([#96](https://github.com/tokio-rs/toasty/pull/96)) +- Refactor sql serializer ([#95](https://github.com/tokio-rs/toasty/pull/95)) +- move crates to flatter structure ([#91](https://github.com/tokio-rs/toasty/pull/91)) +- Switch Toasty to use proc macros for schema declaration ([#76](https://github.com/tokio-rs/toasty/pull/76)) +- Fix typo in README.md ([#4](https://github.com/tokio-rs/toasty/pull/4)) +- Initial commit diff --git a/packages/toasty/Cargo.toml b/packages/toasty/Cargo.toml new file mode 100644 index 00000000000..03b0f6ea404 --- /dev/null +++ b/packages/toasty/Cargo.toml @@ -0,0 +1,254 @@ +[package] +name = "toasty" +version = "0.11.0" +edition = "2024" +rust-version = "1.95" +description = "An async ORM for Rust supporting SQL and NoSQL databases" +documentation = "https://docs.rs/toasty" +readme = "README.md" +homepage = "https://github.com/tokio-rs/toasty" +repository = "https://github.com/tokio-rs/toasty" +license = "MIT" +keywords = [ + "async", + "database", + "nosql", + "orm", + "sql", +] +categories = ["database"] +# authors = ["Carl Lerche "] # deprecated field, causes tombi warnings +build = "build.rs" +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +resolver = "2" + +[lib] +name = "toasty" +path = "src/lib.rs" + +[[test]] +name = "embedded_migrations" +path = "tests/embedded_migrations.rs" + +[[test]] +name = "generate_unique_id" +path = "tests/generate_unique_id.rs" + +[[test]] +name = "migration_generate" +path = "tests/migration_generate.rs" + +[[test]] +name = "pool_sweep" +path = "tests/pool_sweep.rs" + +[[test]] +name = "query_macro" +path = "tests/query_macro.rs" + +[[test]] +name = "relation_load" +path = "tests/relation_load.rs" + +[[test]] +name = "stmt_into_expr" +path = "tests/stmt_into_expr.rs" + +[[test]] +name = "stmt_into_scope" +path = "tests/stmt_into_scope.rs" + +[dependencies.async-trait] +version = "0.1.89" + +[dependencies.bigdecimal] +version = "0.4.10" +optional = true + +[dependencies.bit-set] +version = "0.11.0" + +[dependencies.by_address] +version = "1.2.1" + +[dependencies.deadpool] +version = "0.13" +features = ["rt_tokio_1"] + +[dependencies.hashbrown] +version = "0.17" + +[dependencies.index_vec] +version = "0.1.4" + +[dependencies.indexmap] +version = "2.13.0" + +[dependencies.inventory] +version = "0.3.24" + +[dependencies.jiff] +version = "0.2.23" +optional = true + +[dependencies.rust_decimal] +version = "1.41.0" +optional = true + +[dependencies.serde] +version = "1.0.228" +features = ["derive"] +optional = true + +[dependencies.serde_core] +version = "1.0.228" +optional = true + +[dependencies.serde_json] +version = "1.0.149" +optional = true + +[dependencies.toasty-core] +version = "0.11.0" + +[dependencies.toasty-driver-dynamodb] +version = "0.11.0" +optional = true + +[dependencies.toasty-driver-mysql] +version = "0.11.0" +default-features = false +optional = true + +[dependencies.toasty-driver-postgresql] +version = "0.11.0" +optional = true + +[dependencies.toasty-driver-turso] +version = "0.11.0" +optional = true + +[dependencies.toasty-macros] +version = "0.11.0" + +[dependencies.tokio] +version = "1.50" +default-features = false +features = [ + "macros", + "rt", + "sync", + "time", +] + +[dependencies.tokio-stream] +version = "0.1.18" +default-features = false + +[dependencies.toml] +version = "1.1.0" +optional = true + +[dependencies.toml_edit] +version = "0.25.8" +features = ["serde"] +optional = true + +[dependencies.tracing] +version = "0.1" + +[dependencies.uuid] +version = "1.23.0" +features = [ + "fast-rng", + "js", + "v4", + "v7", +] + +[dev-dependencies.assert-struct] +version = "0.5.0" + +[dev-dependencies.proptest] +version = "1" + +[dev-dependencies.tempfile] +version = "3.27" + +[dev-dependencies.tokio] +version = "1.50" +default-features = false +features = [ + "macros", + "rt", + "sync", + "test-util", + "time", +] + +[features] +default = ["rustls"] +bigdecimal = [ + "dep:bigdecimal", + "toasty-core/bigdecimal", + "toasty-driver-mysql?/bigdecimal", + "toasty-driver-postgresql?/bigdecimal", + + "toasty-driver-turso?/bigdecimal", +] +dynamodb = ["dep:toasty-driver-dynamodb"] +jiff = [ + "dep:jiff", + "toasty-core/jiff", + "toasty-driver-mysql?/jiff", + "toasty-driver-postgresql?/jiff", + + "toasty-driver-turso?/jiff", +] +mariadb = ["dep:toasty-driver-mysql"] +migration = [ + "dep:serde", + "dep:toml", + "dep:toml_edit", + "toasty-core/migration", +] +mysql = ["dep:toasty-driver-mysql"] +native-tls = ["toasty-driver-mysql?/native-tls"] +net = [ + "toasty-core/net", + "toasty-driver-mysql?/net", + "toasty-driver-postgresql?/net", + + "toasty-driver-turso?/net", +] +postgresql = ["dep:toasty-driver-postgresql"] +rust_decimal = [ + "dep:rust_decimal", + "toasty-core/rust_decimal", + "toasty-driver-mysql?/rust_decimal", + "toasty-driver-postgresql?/rust_decimal", + + "toasty-driver-turso?/rust_decimal", +] +rustls = ["toasty-driver-mysql?/rustls"] +serde = [ + "dep:serde_core", + "dep:serde_json", +] + +turso = ["dep:toasty-driver-turso"] +turso-serverless = [ + "toasty-driver-turso?/serverless", + "turso", +] +turso-sync = [ + "toasty-driver-turso?/sync", + "turso", +] + +[lints.rust] +unsafe_code = "deny" diff --git a/packages/toasty/LICENSE b/packages/toasty/LICENSE new file mode 100644 index 00000000000..a86b7b0a4d8 --- /dev/null +++ b/packages/toasty/LICENSE @@ -0,0 +1,18 @@ +Copyright (c) 2026 Carl Lerche + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the “Software”), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/packages/toasty/README.md b/packages/toasty/README.md new file mode 100644 index 00000000000..292262beec2 --- /dev/null +++ b/packages/toasty/README.md @@ -0,0 +1,144 @@ +
+

Toasty

+
+ +
+

The cozy, easy ORM for Rust

+ Guide • + API Docs • + Crates.io • + Discord +
+ +
+ +
+ +[![Crates.io][crates-badge]][crates-url] +[![MIT licensed][mit-badge]][mit-url] +[![Build Status][actions-badge]][actions-url] +[![Discord chat][discord-badge]][discord-url] + +
+ +[crates-badge]: https://img.shields.io/crates/v/toasty.svg +[crates-url]: https://crates.io/crates/toasty +[mit-badge]: https://img.shields.io/badge/license-MIT-blue.svg +[mit-url]: https://github.com/tokio-rs/toasty/blob/main/LICENSE +[actions-badge]: https://github.com/tokio-rs/toasty/workflows/Cargo%20Build%20%26%20Test/badge.svg +[actions-url]: https://github.com/tokio-rs/toasty/actions?query=workflow%3A%22Cargo+Build+%26+Test%22+branch%3Amain +[discord-badge]: https://img.shields.io/discord/500028886025895936.svg?logo=discord&style=flat-square +[discord-url]: https://discord.gg/tokio + +Toasty supports SQL databases (SQLite/Turso, PostgreSQL, MySQL) and DynamoDB. +It does not hide database capabilities — it exposes features based on the +target database. + +## Using Toasty + +You will define your data model using Rust structs annotated with the +`#[derive(toasty::Model)]` derive macro. Runnable examples live in the +[`examples/`](examples/) directory; the snippets below illustrate the basics. + +```rust +#[derive(Debug, toasty::Model)] +struct User { + #[key] + #[auto] + id: u64, + + name: String, + + #[unique] + email: String, + + #[has_many] + todos: toasty::Deferred>, +} + +#[derive(Debug, toasty::Model)] +struct Todo { + #[key] + #[auto] + id: u64, + + #[index] + user_id: u64, + + #[belongs_to] + user: toasty::Deferred, + + title: String, +} +``` + +Then, you can easily work with the data model: + +```rust +// Create a new user and give them some todos. +let user = toasty::create!(User { + name: "John Doe", + email: "john@example.com", + todos: [ + { title: "Make pizza" }, + { title: "Finish Toasty" }, + { title: "Sleep" }, + ], +}).exec(&mut db).await?; + +// Load the user from the database +let user = User::get_by_id(&mut db, &user.id).await?; + +// Load and iterate the user's todos +let todos = user.todos().exec(&mut db).await?; + +for todo in &todos { + println!("{:#?}", todo); +} +``` + +## SQL and NoSQL + +Toasty supports both SQL and NoSQL databases. Current drivers are SQLite, +Turso, PostgreSQL, MySQL, and DynamoDB. However, it does not aim to abstract +the database. Instead, Toasty leans into the target database's capabilities +and aims to help the user avoid issuing inefficient queries for that database. + +When targeting both SQL and NoSQL databases, Toasty generates query methods +(e.g. `get_by_id` only for access patterns that are indexed). When targeting a +SQL database, Toasty might allow arbitrary additional query constraints. When +targeting a NoSQL database, Toasty will only allow constraints that the +specific target database can execute. For example, with DynamoDB, query methods +might be generated based on the table's primary key, and additional constraints +may be set for the sort key. + +## Application data model vs. database schema + +Toasty decouples the application data model from the database's schema. By +default, a toasty application schema will map 1-1 with a database schema. +However, additional annotations may be specified to customize how the +application data model maps to the database schema. + +## Roadmap + +Development priorities are based on feedback and contributions. If you run into +missing features or rough edges, please open an issue or submit a pull request. +Planned work lives under [`docs/dev/roadmap.md`](docs/dev/roadmap.md). + +## Contributing + +See [`CONTRIBUTING.md`](CONTRIBUTING.md). Small fixes can go straight to a PR; +larger changes follow a lightweight propose → roadmap + design doc → implement +flow. + +## License + +This project is licensed under the [MIT license]. + +[MIT license]: LICENSE + +### Contribution + +Unless you explicitly state otherwise, any contribution intentionally submitted +for inclusion in Toasty by you, shall be licensed as MIT, without any additional +terms or conditions. diff --git a/packages/toasty/build.rs b/packages/toasty/build.rs new file mode 100644 index 00000000000..128c1f15bff --- /dev/null +++ b/packages/toasty/build.rs @@ -0,0 +1,15 @@ +use std::{env, fs, path::PathBuf}; + +fn main() { + println!("cargo::rerun-if-env-changed=TOASTY_GUIDE_URL"); + + let url = env::var("TOASTY_GUIDE_URL") + .unwrap_or_else(|_| "https://github.com/tokio-rs/toasty/tree/main/docs/guide".to_string()); + + let out = PathBuf::from(env::var("OUT_DIR").unwrap()); + fs::write( + out.join("guide_link.md"), + format!("[Toasty guide]: {url}\n"), + ) + .unwrap(); +} diff --git a/packages/toasty/proptest-regressions/engine/fold/tests/prop_idempotence.txt b/packages/toasty/proptest-regressions/engine/fold/tests/prop_idempotence.txt new file mode 100644 index 00000000000..f47318f10a0 --- /dev/null +++ b/packages/toasty/proptest-regressions/engine/fold/tests/prop_idempotence.txt @@ -0,0 +1,7 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc f8c9e99c9ce4a9ad6638043e3e0bec128ff94eff680061d5a5abf81f31e93f52 # shrinks to expr = ExprNot { expr: ExprOr { operands: [Bool(false), ExprInList { expr: I64(0), list: List([I64(0)]) }] } } diff --git a/packages/toasty/src/codegen_support.rs b/packages/toasty/src/codegen_support.rs new file mode 100644 index 00000000000..91842d1ae9d --- /dev/null +++ b/packages/toasty/src/codegen_support.rs @@ -0,0 +1,238 @@ +//! Internal types referenced by code generated by `#[derive(Model)]` and the +//! `create!` / `update!` / `query!` macros. Not part of the public API; the +//! contents may change between releases. + +#![doc(hidden)] + +pub mod auto; +pub mod index; +pub mod newtype; +pub mod shared_column; +pub mod storage; +pub mod version; +pub mod via; + +pub use crate::schema::inventory; +pub use crate::{ + Db, Error, Executor, Result, Statement, + schema::{ + Auto, Deferred, DiscoverItem, Document, Embed, EmbedCreate, Field, Load, Model, QueryMany, + QueryOne, QueryOptionOne, RelationManyField, RelationOneField, Scalar, Scope, ViaMany, + ViaManyField, ViaPath, ViaTarget, generate_unique_id, + }, + stmt::CreateMany, + stmt::{self, Assign, Expr, IntoExpr, IntoInsert, IntoStatement, List, Path}, + update_target::UpdateTarget, +}; +#[cfg(feature = "serde")] +pub use serde_json; +pub use std::{clone::Clone, convert::Into, default::Default, option::Option}; + +pub use self::version::Version; +pub use toasty_core as core; + +/// The expression-level type that a create/update setter accepts for a field +/// of type `F`. This is just [`Field::ExprTarget`], but naming it keeps the +/// long `::ExprTarget` projection out of generated setter +/// signatures (and out of the compiler errors they produce). +pub type FieldExprTarget = ::ExprTarget; + +/// The field accessor for a singular relation declared as `F`, rooted at +/// `Origin`. Resolves to the target model's fields with the path target from +/// [`RelationOneField::Expr`], preserving optionality for eager and deferred +/// relations. +pub type RelationOnePath = + <::Target as Model>::OneField::Expr>; + +/// Internal constructors used by generated model field accessors. +pub trait ModelCodegen: Model { + /// Construct the field accessor for a singular relation to this model. + fn new_one_field(path: Path) -> Self::OneField; + + /// Encode referenced fields so the engine can resolve a relation key. + fn to_relation_expr(&self, fields: &[&str]) -> core::stmt::Expr; +} + +/// Infer the [`Scope`] type from a scope expression and return its fields +/// path. +/// +/// The `create!` macro uses this in the scoped form (`in expr { ... }`) to +/// obtain the field struct for nested builders. Because the macro has no +/// type information, it cannot call `S::new_path_root()` directly — this function +/// lets Rust infer `S` from the scope argument. +pub fn scope_fields(_scope: &S) -> S::Path { + S::new_path_root() +} + +/// Create a record inside a scoped create target. +pub fn create_in_scope(scope: S) -> S::Create { + S::create_in_scope(scope) +} + +/// Convert a value into an untyped [`core::stmt::Expr`] via the typed +/// [`IntoExpr`] trait. +/// +/// Generated code (`#[derive(Model)]`, `#[derive(Embed)]`) splices this in +/// instead of inlining the `let expr: Expr = value.into_expr(); expr.into()` +/// pattern at every field site. The explicit `T` type parameter +/// disambiguates which `IntoExpr` impl to use. +pub fn into_untyped_expr>(value: V) -> core::stmt::Expr { + let expr: stmt::Expr = value.into_expr(); + expr.into() +} + +/// A value usable as the parent of an embedded relation in a write. +/// +/// The trait parameter `F` is the relation field's *declared* type +/// (`M`, `Option`, `Deferred`, or `Deferred>`); the accepted value is the target +/// model itself (by value or reference) or a value of the declared shape. +/// Keying by `F` rather than by the model keeps inference working when the +/// setter receives `Deferred::default()` — the setter signature pins `F`, so +/// the deferred wrapper's inner type resolves uniquely. +/// +/// `model_ref` resolves to the parent model when one is present: an unloaded +/// `Deferred` (or a loaded `Deferred>` holding `None`) resolves to +/// `None`, meaning the write has no parent value to take keys from and the +/// explicitly set key fields stand. +pub trait EmbeddedRelationValue { + type Model; + + fn model_ref(&self) -> Option<&Self::Model>; +} + +macro_rules! impl_embedded_relation_value { + ($field:ty) => { + impl EmbeddedRelationValue<$field> for M { + type Model = M; + + fn model_ref(&self) -> Option<&M> { + Some(self) + } + } + + impl EmbeddedRelationValue<$field> for &M { + type Model = M; + + fn model_ref(&self) -> Option<&M> { + Some(self) + } + } + + impl EmbeddedRelationValue<$field> for &$field { + type Model = M; + + fn model_ref(&self) -> Option<&M> { + (*self).model_ref() + } + } + }; +} + +impl_embedded_relation_value!(Deferred); +impl_embedded_relation_value!(Deferred>); +impl_embedded_relation_value!(Option); + +impl EmbeddedRelationValue for M { + type Model = M; + + fn model_ref(&self) -> Option<&M> { + Some(self) + } +} + +impl EmbeddedRelationValue for &M { + type Model = M; + + fn model_ref(&self) -> Option<&M> { + Some(self) + } +} + +impl EmbeddedRelationValue> for Option { + type Model = M; + + fn model_ref(&self) -> Option<&M> { + self.as_ref() + } +} + +impl EmbeddedRelationValue> for Deferred { + type Model = M; + + fn model_ref(&self) -> Option<&M> { + if self.is_unloaded() { + None + } else { + Some(self.get()) + } + } +} + +impl EmbeddedRelationValue>> for Deferred> { + type Model = M; + + fn model_ref(&self) -> Option<&M> { + if self.is_unloaded() { + None + } else { + self.get().as_ref() + } + } +} + +/// Encode a loaded relation's key, leaving unloaded relations unset. +pub fn embedded_relation_value_expr(value: &V, fields: &[&str]) -> core::stmt::Expr +where + V: EmbeddedRelationValue, + V::Model: ModelCodegen, +{ + value + .model_ref() + .map_or_else(core::stmt::Expr::null, |model| { + model.to_relation_expr(fields) + }) +} + +/// Continue a `has_many` traversal from `query` along `path`. +/// +/// If `query` was already scoped from a relation traversal, append +/// `field_offset` to that traversal's path so the result remains a single +/// chained association. Otherwise build a fresh `Association::many` rooted +/// at `query`. Used by generated `.field()` chain methods on the per-model +/// `Query>` for `has_many` relations. +pub fn chain_or_build_many( + mut query: stmt::Query>, + field_offset: usize, + path: stmt::Path>, +) -> stmt::Association> +where + Source: Model, + Target: Model, +{ + match query.take_via_assoc() { + Some(untyped) => stmt::Association::>::from_untyped(untyped) + .chain_field::(field_offset), + None => stmt::Association::many(query, path), + } +} + +/// Continue a `belongs_to` / `has_one` traversal from `query` along `path`. +/// +/// Mirrors [`chain_or_build_many`] for singular relation steps: when no +/// existing traversal is present the fresh association is built via +/// [`stmt::Association::many_via_one`]. +pub fn chain_or_build_many_via_one( + mut query: stmt::Query>, + field_offset: usize, + path: stmt::Path, +) -> stmt::Association> +where + Source: Model, + Target: Model, +{ + match query.take_via_assoc() { + Some(untyped) => stmt::Association::>::from_untyped(untyped) + .chain_field::(field_offset), + None => stmt::Association::many_via_one(query, path), + } +} diff --git a/packages/toasty/src/codegen_support/auto.rs b/packages/toasty/src/codegen_support/auto.rs new file mode 100644 index 00000000000..15c7765f236 --- /dev/null +++ b/packages/toasty/src/codegen_support/auto.rs @@ -0,0 +1,69 @@ +//! Compile-time validation for explicit `#[auto(...)]` strategies. +//! +//! When `#[derive(Model)]` sees an explicit strategy on a field +//! (`#[auto(uuid)]`, `#[auto(uuid(v4))]`, `#[auto(uuid(v7))]`, +//! `#[auto(increment)]`), the generated code emits a one-shot obligation that +//! the field's Rust type implements `AutoCompatible` for the matching +//! marker `Tag`. The bare `#[auto]` form already gets a `T: Auto` obligation +//! via the `STRATEGY` const lookup, so it does not need this trait. +//! +//! Mirrors the pattern in [`crate::codegen_support::storage`]: tag types are +//! zero-sized markers, the trait carries a `#[diagnostic::on_unimplemented]` +//! message, and the macro names tags by absolute path so user-side type +//! aliases or shadowing cannot defeat the check. + +/// Marker types for each `#[auto(...)]` strategy. +/// +/// These types are zero-sized and only ever appear as the `Strategy` +/// parameter to [`AutoCompatible`]. Macro-generated code names them by +/// absolute path. +#[allow(missing_docs)] +pub mod tag { + pub struct Uuid; + pub struct Increment; +} + +/// Asserts that a Rust field type can carry an auto-generated value of the +/// given strategy. +/// +/// Macro-generated code emits a `_check::()` call where `_check` +/// is bounded by `AutoCompatible`. The bound is checked against the +/// resolved field type, so type aliases and newtype embeds compose naturally. +#[diagnostic::on_unimplemented( + message = "field type `{Self}` is not compatible with the requested auto strategy `{Strategy}`", + label = "incompatible Rust type for `#[auto(...)]`", + note = "see `toasty::codegen_support::auto` for the compatibility table" +)] +pub trait AutoCompatible {} + +// `Option` is compatible with the same strategies as `T`. Auto-populated +// fields are typically non-nullable, but the schema tracks nullability +// separately so the strategy check is unaffected by the option wrapper. +impl AutoCompatible for Option where T: AutoCompatible {} + +impl AutoCompatible for i8 {} +impl AutoCompatible for i16 {} +impl AutoCompatible for i32 {} +impl AutoCompatible for i64 {} +impl AutoCompatible for u8 {} +impl AutoCompatible for u16 {} +impl AutoCompatible for u32 {} +impl AutoCompatible for u64 {} +impl AutoCompatible for isize {} +impl AutoCompatible for usize {} + +impl AutoCompatible for uuid::Uuid {} + +// A tuple-newtype embed is compatible with whatever its inner type is +// compatible with. Mirrors the `Auto` blanket in +// [`crate::codegen_support::newtype`]. +// +// `do_not_recommend` keeps the blanket out of suggestion lists so the +// `AutoCompatible` `#[diagnostic::on_unimplemented]` message wins. +#[diagnostic::do_not_recommend] +impl AutoCompatible for T +where + T: super::newtype::NewtypeOf, + ::Inner: AutoCompatible, +{ +} diff --git a/packages/toasty/src/codegen_support/index.rs b/packages/toasty/src/codegen_support/index.rs new file mode 100644 index 00000000000..6b40ee839bb --- /dev/null +++ b/packages/toasty/src/codegen_support/index.rs @@ -0,0 +1,109 @@ +//! Compile-time validation that a field used in an index maps to a single +//! index column. +//! +//! When `#[derive(Model)]` sees a field participating in a secondary index or +//! unique constraint (`#[index]`, `#[unique]`, or a model-level +//! `#[index(...)]` / `#[unique(...)]`), the generated code emits a one-shot +//! obligation that the field's Rust type implements [`IndexableField`]. +//! +//! Scalars and storable `Vec` fields implement it directly; tuple-newtype +//! embeds and unit (data-less) enums use impls emitted by `#[derive(Embed)]`. +//! Data-carrying enums and multi-field embedded structs map to more than one +//! column, so they do not implement it and naming one in an index is a compile +//! error rather than a runtime panic. Backend support for indexing a collection +//! is checked separately while building the database schema. +//! +//! Mirrors the pattern in [`crate::codegen_support::storage`] and +//! [`crate::codegen_support::auto`]: a trait carrying a +//! `#[diagnostic::on_unimplemented]` message and concrete impls for the +//! supported scalar types. Unlike those, newtype embeds get a per-type impl +//! from the derive rather than a `NewtypeOf` blanket — a blanket would conflict +//! with the `Box` forwarding impl below, since `Box` is `#[fundamental]`. + +/// Asserts that a Rust field type can serve as a single index column. +/// +/// Macro-generated code emits a `_check::()` call bounded by +/// `IndexableField`. The Rust type checker — not the macro — resolves the field +/// type, so type aliases and newtype embeds are handled correctly. +#[diagnostic::on_unimplemented( + message = "field type `{Self}` cannot be used in an index", + label = "not indexable", + note = "only scalar fields, storable Vec fields, newtype embeds, and unit (data-less) \ + enums can be index targets; data-carrying enums and multi-field embedded structs \ + span multiple columns and have no single index column" +)] +pub trait IndexableField {} + +// `Option` is indexable wherever `T` is. The schema tracks nullability +// separately, so the option wrapper does not affect indexability. +impl IndexableField for Option where T: IndexableField {} + +// Smart-pointer wrappers (used for boxed foreign keys) are transparent for +// storage, so they are indexable wherever the pointee is. +impl IndexableField for Box where T: IndexableField {} +impl IndexableField for std::rc::Rc where T: IndexableField {} +impl IndexableField for std::sync::Arc where T: IndexableField {} + +// A `Deferred` field is loaded lazily but stores the same single column as +// `T`, so it is indexable wherever `T` is. +impl IndexableField for crate::Deferred where T: IndexableField {} + +impl IndexableField for bool {} + +impl IndexableField for i8 {} +impl IndexableField for i16 {} +impl IndexableField for i32 {} +impl IndexableField for i64 {} +impl IndexableField for isize {} + +impl IndexableField for u8 {} +impl IndexableField for u16 {} +impl IndexableField for u32 {} +impl IndexableField for u64 {} +impl IndexableField for usize {} + +impl IndexableField for f32 {} +impl IndexableField for f64 {} + +impl IndexableField for String {} + +// A collection is a single logical index value wherever it is a valid field. +// Whether a specific backend can create the requested index is a separate +// schema-build capability check. This bound includes `Vec` without +// overlapping its special bytes `Field` implementation. +impl IndexableField for Vec where Vec: crate::schema::Field {} + +impl IndexableField for uuid::Uuid {} + +#[cfg(feature = "rust_decimal")] +impl IndexableField for rust_decimal::Decimal {} + +#[cfg(feature = "bigdecimal")] +impl IndexableField for bigdecimal::BigDecimal {} + +#[cfg(feature = "jiff")] +mod jiff_impls { + use super::IndexableField; + + impl IndexableField for jiff::Timestamp {} + impl IndexableField for jiff::civil::Date {} + impl IndexableField for jiff::civil::Time {} + impl IndexableField for jiff::civil::DateTime {} + impl IndexableField for jiff::tz::TimeZone {} +} + +#[cfg(feature = "net")] +mod net_impls { + use super::IndexableField; + use crate::stmt::{IpCidr, IpInet, MacAddr6, MacAddr8}; + + impl IndexableField for IpCidr {} + impl IndexableField for IpInet {} + impl IndexableField for MacAddr6 {} + impl IndexableField for MacAddr8 {} +} + +// Tuple-newtype embeds get a per-type `IndexableField` impl from +// `#[derive(Embed)]` (see `expand_embedded_indexable_impl` in toasty-macros), +// forwarding to their inner type. A `NewtypeOf` blanket would be more concise +// but conflicts with the `Box` impl above, since `Box` is `#[fundamental]`. diff --git a/packages/toasty/src/codegen_support/newtype.rs b/packages/toasty/src/codegen_support/newtype.rs new file mode 100644 index 00000000000..a9d5abbb7be --- /dev/null +++ b/packages/toasty/src/codegen_support/newtype.rs @@ -0,0 +1,50 @@ +//! Marker trait emitted by `#[derive(Embed)]` for tuple-newtype embeds. +//! +//! The companion blanket impl turns any newtype embed whose inner type +//! implements [`Auto`] into an [`Auto`] type itself, so a user does not need +//! to repeat the strategy on the wrapper: +//! +//! ```ignore +//! #[derive(toasty::Embed)] +//! struct UserId(uuid::Uuid); +//! +//! // Automatically: `UserId: Auto` because `Uuid: Auto`. +//! ``` +//! +//! Newtypes around non-`Auto` types stay non-`Auto` — the blanket's +//! `T::Inner: Auto` bound is on a generic associated type, so it is properly +//! conditional and doesn't error at impl declaration. +//! +//! `NewtypeOf` is local to toasty. Combined with the orphan rule, that keeps +//! the blanket from overlapping with the concrete `impl Auto for u64`, +//! `impl Auto for uuid::Uuid`, etc.: no downstream crate can ever add +//! `impl NewtypeOf for u64`, and toasty itself never will. + +use crate::schema::{Auto, Field}; +use toasty_core::schema::app::AutoStrategy; + +/// Marker for a tuple-newtype embedded type, carrying its single inner field +/// type as an associated type. Emitted by `#[derive(Embed)]` for tuple +/// structs with exactly one field. +pub trait NewtypeOf { + /// The inner field's type. + type Inner; + + /// Unwrap the newtype, returning the inner value. + fn into_inner(self) -> Self::Inner; + + /// Wrap an inner value in the newtype. + fn from_inner(inner: Self::Inner) -> Self; +} + +// `do_not_recommend` keeps the blanket out of error suggestions so the +// `Auto` trait's `#[diagnostic::on_unimplemented]` message wins for users +// who hit a missing-`Auto` error on an embed. +#[diagnostic::do_not_recommend] +impl Auto for T +where + T: NewtypeOf + Field, + ::Inner: Auto, +{ + const STRATEGY: AutoStrategy = <::Inner as Auto>::STRATEGY; +} diff --git a/packages/toasty/src/codegen_support/shared_column.rs b/packages/toasty/src/codegen_support/shared_column.rs new file mode 100644 index 00000000000..b0f1f883645 --- /dev/null +++ b/packages/toasty/src/codegen_support/shared_column.rs @@ -0,0 +1,28 @@ +//! Compile-time validation that enum variant fields sharing a column agree on +//! their type. +//! +//! When two `#[derive(Embed)]` enum variants declare the same shared logical +//! field via `#[shared()]`, the generated code emits a +//! [`SameColumnType`] obligation between the two field types. The trait is +//! implemented only reflexively, so the obligation holds exactly when the two +//! types are identical; otherwise the `#[diagnostic::on_unimplemented]` +//! message points the user at the mismatch. +//! +//! The schema builder enforces the same rule at runtime as a backstop for +//! schemas built without the derive macro. + +/// Asserts that two enum variant fields mapped to the same column have the same +/// Rust type. +/// +/// Implemented only reflexively (`impl SameColumnType for T`), so +/// `A: SameColumnType` holds iff `A` and `B` are the same type. Macro- +/// generated code emits a `_check::()` bounded by this trait for each pair +/// of fields that land on one shared column. +#[diagnostic::on_unimplemented( + message = "enum variant fields sharing a column must have the same type", + label = "`{Self}` does not match `{Other}`, the type another variant shares with this column", + note = "give both variants' fields the same type, or use distinct `#[shared(...)]` identifiers so they map to separate columns" +)] +pub trait SameColumnType {} + +impl SameColumnType for T {} diff --git a/packages/toasty/src/codegen_support/storage.rs b/packages/toasty/src/codegen_support/storage.rs new file mode 100644 index 00000000000..7065f04a071 --- /dev/null +++ b/packages/toasty/src/codegen_support/storage.rs @@ -0,0 +1,249 @@ +//! Compile-time validation for `#[column(type = ...)]`. +//! +//! When `#[derive(Model)]` sees an explicit storage type on a field, the +//! generated code emits a one-shot obligation that the field's Rust type +//! implements `CompatibleWith` for the matching marker `Tag`. The Rust +//! type checker — not the macro — resolves the field type, so type aliases +//! and generic parameters are handled correctly. + +/// Marker types corresponding to each `#[column(type = ...)]` storage tag. +/// +/// These types are zero-sized and only ever appear as the `S` parameter to +/// [`CompatibleWith`]. Macro-generated code names them by absolute path. +#[allow(missing_docs, non_camel_case_types)] +pub mod tag { + pub struct Boolean; + + /// Signed integer storage with a non-standard byte width. + pub struct Int; + /// Unsigned integer storage with a non-standard byte width. + pub struct UInt; + + pub struct I8; + pub struct I16; + pub struct I32; + pub struct I64; + + pub struct U8; + pub struct U16; + pub struct U32; + pub struct U64; + + pub struct F32; + pub struct F64; + + pub struct Text; + pub struct VarChar; + pub struct Json; + pub struct Jsonb; + + pub struct Binary; + pub struct Blob; + + pub struct Timestamp; + pub struct Date; + pub struct Time; + pub struct DateTime; + + pub struct Cidr; + pub struct Inet; + pub struct MacAddr; + pub struct MacAddr8; +} + +/// Asserts that a Rust field type can be stored as the given storage tag. +/// +/// Macro-generated code emits a `_check::()` call where `_check` +/// is bounded by `CompatibleWith`. Trait resolution happens after type +/// alias substitution, so `type String = u32;` cannot defeat the check the +/// way it would defeat any analysis of the raw `String` token in the macro. +#[diagnostic::on_unimplemented( + message = "field type `{Self}` is not compatible with the requested column storage `{Storage}`", + label = "incompatible Rust type for `#[column(type = ...)]`", + note = "see `toasty::codegen_support::storage` for the compatibility table" +)] +pub trait CompatibleWith {} + +/// Marker implemented by the integer storage tags accepted for embedded enum +/// discriminant columns. +pub trait IntegerStorage {} + +impl IntegerStorage for tag::I8 {} +impl IntegerStorage for tag::I16 {} +impl IntegerStorage for tag::I32 {} +impl IntegerStorage for tag::I64 {} +impl IntegerStorage for tag::U8 {} +impl IntegerStorage for tag::U16 {} +impl IntegerStorage for tag::U32 {} +impl IntegerStorage for tag::U64 {} +impl IntegerStorage for tag::Int {} +impl IntegerStorage for tag::UInt {} + +// `Option` is compatible with the same storage tags as `T`. The schema +// tracks nullability separately, so the storage type is unaffected by the +// option wrapper. +impl CompatibleWith for Option where T: CompatibleWith {} + +// Deferred fields and smart pointers are transparent storage wrappers. +impl CompatibleWith for crate::Deferred where T: CompatibleWith {} +impl CompatibleWith for Box where T: CompatibleWith {} +impl CompatibleWith for std::rc::Rc where T: CompatibleWith {} +impl CompatibleWith for std::sync::Arc where T: CompatibleWith {} + +// A unit enum is both `Embed` and `Scalar`. Integer storage selected on a +// `Vec` field describes each discriminant, not the collection as a +// whole. Keep these implementations limited to enum collections so `Vec` +// retains its binary-blob meaning. +macro_rules! impl_integer_enum_collection_compat { + ($($storage:ty),* $(,)?) => { + $( + impl CompatibleWith<$storage> for Vec + where + T: crate::schema::Embed + + crate::schema::Scalar + + CompatibleWith<$storage>, + {} + )* + }; +} + +impl_integer_enum_collection_compat!( + tag::I8, + tag::I16, + tag::I32, + tag::I64, + tag::U8, + tag::U16, + tag::U32, + tag::U64, +); + +impl CompatibleWith> for Vec where + T: crate::schema::Embed + crate::schema::Scalar + CompatibleWith> +{ +} + +impl CompatibleWith> for Vec where + T: crate::schema::Embed + crate::schema::Scalar + CompatibleWith> +{ +} + +impl CompatibleWith for bool {} + +impl CompatibleWith for i8 {} +impl CompatibleWith for i16 {} +impl CompatibleWith for i32 {} +impl CompatibleWith for i64 {} + +impl CompatibleWith for u8 {} +impl CompatibleWith for u16 {} +impl CompatibleWith for u32 {} +impl CompatibleWith for u64 {} + +// `int(N)` and `uint(N)` have no exact Rust counterpart when N is not a +// power of two. Preserve the integer-family check while the engine performs +// the value-level checked cast. +macro_rules! impl_custom_integer_compat { + ($storage:ident => $($ty:ty),* $(,)?) => { + $( + #[diagnostic::do_not_recommend] + impl CompatibleWith> for $ty {} + )* + }; +} + +impl_custom_integer_compat!(Int => i8, i16, i32, i64); +impl_custom_integer_compat!(UInt => u8, u16, u32, u64); + +impl CompatibleWith for f32 {} +impl CompatibleWith for f64 {} +// Both directions are allowed — drivers convert at the boundary, and the +// storage choice is the user's explicit decision. +impl CompatibleWith for f64 {} +impl CompatibleWith for f32 {} + +impl CompatibleWith for String {} +impl CompatibleWith for String {} + +// JSON fields serialize through Toasty's string expression type. They support +// text storage and the native JSON types recognized by the SQL drivers. +#[cfg(feature = "serde")] +impl CompatibleWith for crate::Json {} +#[cfg(feature = "serde")] +impl CompatibleWith for crate::Json {} +#[cfg(feature = "serde")] +impl CompatibleWith for crate::Json {} +#[cfg(feature = "serde")] +impl CompatibleWith for crate::Json {} +#[cfg(feature = "serde")] +impl CompatibleWith for serde_json::Value {} +#[cfg(feature = "serde")] +impl CompatibleWith for serde_json::Value {} +#[cfg(feature = "serde")] +impl CompatibleWith for serde_json::Value {} +#[cfg(feature = "serde")] +impl CompatibleWith for serde_json::Value {} + +impl CompatibleWith for Vec {} +impl CompatibleWith for Vec {} + +// UUIDs may be stored as their native form, as text, or as raw bytes. +impl CompatibleWith for uuid::Uuid {} +impl CompatibleWith for uuid::Uuid {} +impl CompatibleWith for uuid::Uuid {} +impl CompatibleWith for uuid::Uuid {} + +#[cfg(feature = "rust_decimal")] +impl CompatibleWith for rust_decimal::Decimal {} +#[cfg(feature = "rust_decimal")] +impl CompatibleWith for rust_decimal::Decimal {} + +#[cfg(feature = "bigdecimal")] +impl CompatibleWith for bigdecimal::BigDecimal {} +#[cfg(feature = "bigdecimal")] +impl CompatibleWith for bigdecimal::BigDecimal {} + +#[cfg(feature = "jiff")] +mod jiff_impls { + use super::{CompatibleWith, tag}; + + impl CompatibleWith for jiff::Timestamp {} + impl CompatibleWith for jiff::civil::Date {} + impl CompatibleWith for jiff::civil::Time {} + impl CompatibleWith for jiff::civil::DateTime {} + + // Temporal types may also be serialized as text (ISO 8601 form). The + // driver handles the conversion at the boundary. + impl CompatibleWith for jiff::Timestamp {} + impl CompatibleWith for jiff::Timestamp {} + impl CompatibleWith for jiff::civil::Date {} + impl CompatibleWith for jiff::civil::Date {} + impl CompatibleWith for jiff::civil::Time {} + impl CompatibleWith for jiff::civil::Time {} + impl CompatibleWith for jiff::civil::DateTime {} + impl CompatibleWith for jiff::civil::DateTime {} + + impl CompatibleWith for jiff::tz::TimeZone {} + impl CompatibleWith for jiff::tz::TimeZone {} +} + +#[cfg(feature = "net")] +mod net_impls { + use super::{CompatibleWith, tag}; + use crate::stmt::{IpCidr, IpInet, MacAddr6, MacAddr8}; + + impl CompatibleWith for IpCidr {} + impl CompatibleWith for IpInet {} + impl CompatibleWith for MacAddr6 {} + impl CompatibleWith for MacAddr8 {} + + // Backends without native network types store their canonical text forms. + impl CompatibleWith for IpCidr {} + impl CompatibleWith for IpCidr {} + impl CompatibleWith for IpInet {} + impl CompatibleWith for IpInet {} + impl CompatibleWith for MacAddr6 {} + impl CompatibleWith for MacAddr6 {} + impl CompatibleWith for MacAddr8 {} + impl CompatibleWith for MacAddr8 {} +} diff --git a/packages/toasty/src/codegen_support/version.rs b/packages/toasty/src/codegen_support/version.rs new file mode 100644 index 00000000000..e079fa968c0 --- /dev/null +++ b/packages/toasty/src/codegen_support/version.rs @@ -0,0 +1,61 @@ +//! Compile-time validation and runtime support for `#[version]` fields. +//! +//! `#[derive(Model)]` emits a `_check::()` obligation bounded by +//! [`Version`] for every `#[version]` field. The concrete impl covers +//! `u64` directly; the blanket propagates through tuple-newtype embeds of any +//! depth, mirroring the [`Auto`] blanket in +//! [`crate::codegen_support::newtype`]. +//! +//! At runtime, generated code calls [`Version::as_u64`] to extract the +//! raw counter for use in OCC condition expressions. + +use super::newtype::NewtypeOf; + +/// A field type that can be used as an OCC version counter. +/// +/// Only `u64` and tuple-newtype embeds that wrap a `Version` type (directly +/// or transitively) satisfy this bound. +#[diagnostic::on_unimplemented( + message = "`{Self}` cannot be used as a `#[version]` field", + label = "invalid type for `#[version]`", + note = "only `u64` and tuple-newtype embeds of `u64` are supported" +)] +pub trait Version: Copy { + /// Return the raw `u64` counter value stored in this field. + fn as_u64(self) -> u64; + + /// Construct a field value from a raw `u64` counter, wrapping through any + /// newtype layers. Used by generated code to build the next-version + /// expression in the correct shape for the update assignment. + fn from_u64(v: u64) -> Self; +} + +impl Version for u64 { + fn as_u64(self) -> u64 { + self + } + + fn from_u64(v: u64) -> Self { + v + } +} + +// Propagate through tuple-newtype embeds of any depth. +// +// `do_not_recommend` keeps the blanket out of error suggestions so the +// `Version` `#[diagnostic::on_unimplemented]` message wins for users +// who hit a missing-`Version` error on an embed. +#[diagnostic::do_not_recommend] +impl Version for T +where + T: NewtypeOf + Copy, + ::Inner: Version, +{ + fn as_u64(self) -> u64 { + self.into_inner().as_u64() + } + + fn from_u64(v: u64) -> Self { + T::from_inner(::from_u64(v)) + } +} diff --git a/packages/toasty/src/codegen_support/via.rs b/packages/toasty/src/codegen_support/via.rs new file mode 100644 index 00000000000..28c25bbae5e --- /dev/null +++ b/packages/toasty/src/codegen_support/via.rs @@ -0,0 +1,66 @@ +//! Helpers for the `#[has_one(via = …)]` and `#[has_many(via = …)]` code the +//! derive macro emits. + +use crate::schema::{Model, RelationOneField}; +use crate::stmt::Path; + +use toasty_core::schema::Name; +use toasty_core::schema::app::{self, FieldTy, Via}; +use toasty_core::stmt; + +/// A declared singular via's expression type that permits a step targeting +/// `Step`. Required vias permit only required steps; optional vias permit both. +#[diagnostic::on_unimplemented( + message = "`#[has_one(via = ...)]` result `{Self}` cannot traverse a step targeting `{Step}`", + label = "incompatible via step", + note = "a required via cannot traverse optional relations; declare the via field as `Option<_>` or `Deferred>`" +)] +pub trait AllowsViaStep {} + +// Report the via compatibility error instead of a missing `Model` impl for +// an optional step. +#[diagnostic::do_not_recommend] +impl AllowsViaStep for Declared {} +#[diagnostic::do_not_recommend] +impl AllowsViaStep for Option {} +#[diagnostic::do_not_recommend] +impl AllowsViaStep> for Option {} + +/// Check a singular via step before navigating to the next field, which may +/// have different optionality. Borrowing the handle leaves it available for +/// the next accessor call. +pub fn check_one_step(_: &impl Into>) +where + F: RelationOneField, + F::Expr: AllowsViaStep, + Target: RelationOneField, +{ +} + +/// Validate that a singular via reaches the declared model and convert its +/// typed path to a schema path. Each step's optionality is checked separately +/// by [`check_one_step`]. +pub fn into_one_path(path: Path) -> stmt::Path +where + F: RelationOneField, + Target: RelationOneField, +{ + path.into() +} + +/// Build the [`FieldTy::Via`] for a relation-terminal `#[has_many(via = …)]` +/// field reaching model `M`. The per-model +/// [`ViaTarget`](crate::schema::ViaTarget) impl the derive emits +/// delegates here so the construction (and `Box`/type plumbing) stays in this +/// crate. +pub fn model_via_field_ty(singular: Name, path: stmt::Path) -> FieldTy { + let target = ::id(); + let expr_ty = stmt::Type::List(Box::new(stmt::Type::Model(target))); + FieldTy::Via(Via::new( + target, + expr_ty, + app::Cardinality::Many { singular }, + path, + None, + )) +} diff --git a/packages/toasty/src/db.rs b/packages/toasty/src/db.rs new file mode 100644 index 00000000000..10ef0044bec --- /dev/null +++ b/packages/toasty/src/db.rs @@ -0,0 +1,192 @@ +mod builder; +mod connect; +mod connection; +mod connection_task; +mod executor; +mod pool; +mod tx; + +pub use builder::Builder; +pub use connect::Connect; +pub use connection::Connection; +pub use executor::Executor; +pub use pool::{Pool, PoolStatus}; +pub use toasty_core::driver::{Capability, ConnectContext, Dialect, Driver, SqlPlaceholder}; +pub use tx::{Transaction, TransactionBuilder}; + +/// Response from executing a statement, including pagination metadata. +pub use toasty_core::driver::ExecResponse; + +pub(crate) use connection_task::ConnectionOperation; +pub(crate) use tx::ConnRef; + +use crate::{Result, engine::Engine}; + +use async_trait::async_trait; +use toasty_core::{Schema, driver::operation::RawSql, stmt}; + +use std::sync::Arc; + +/// Shared state between all `Db` clones. +pub(crate) struct Shared { + pub(crate) engine: Engine, + pub(crate) pool: Pool, +} + +/// A database handle backed by a connection pool. +/// +/// Each operation acquires a connection from the pool, executes, and returns +/// the connection. Use [`Db::connection`] to obtain a dedicated +/// [`Connection`] when you need multiple statements to share the same +/// physical connection (e.g. temporary tables or session-level state). +/// +/// Cloning a `Db` is cheap — it shares the underlying pool. +pub struct Db { + shared: Arc, +} + +impl Clone for Db { + fn clone(&self) -> Self { + Db { + shared: self.shared.clone(), + } + } +} + +impl Db { + /// Create a new [`Builder`] for configuring and opening a database. + /// + /// # Examples + /// + /// ``` + /// # tokio::runtime::Runtime::new().unwrap().block_on(async { + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// let driver = toasty_driver_sqlite::Sqlite::in_memory(); + /// let db = toasty::Db::builder() + /// .models(toasty::models!(User)) + /// .build(driver) + /// .await + /// .unwrap(); + /// # }); + /// ``` + pub fn builder() -> Builder { + Builder::default() + } + + /// Acquire a dedicated connection from the pool. + /// + /// The returned [`Connection`] implements [`Executor`] and pins all + /// operations to the same physical connection. This is useful when + /// multiple statements must share connection-level state such as + /// temporary tables or session variables. + /// + /// When the `Connection` is dropped it is returned to the pool for reuse. + pub async fn connection(&self) -> Result { + self.shared.pool.get(self.shared.clone()).await + } + + pub(crate) async fn exec_stmt( + &self, + stmt: stmt::Statement, + in_transaction: bool, + ) -> Result { + let conn = self.connection().await?; + conn.exec_stmt(stmt, in_transaction).await + } + + /// Creates tables and indices defined in the schema on the database. + pub async fn push_schema(&self) -> Result<()> { + let conn = self.connection().await?; + conn.push_schema().await + } + + /// Drops the entire database and recreates an empty one without applying migrations. + pub async fn reset_db(&self) -> Result<()> { + self.shared.pool.driver().reset_db().await + } + + /// Returns a reference to the underlying database driver. + pub fn driver(&self) -> &dyn Driver { + self.shared.pool.driver() + } + + /// Returns the compiled schema used by this database handle. + pub fn schema(&self) -> &Arc { + &self.shared.engine.schema + } + + /// Returns the capability flags reported by the driver. + /// + /// The query engine uses these to decide which operation types to generate + /// (e.g., SQL vs. key-value). + pub fn capability(&self) -> &Capability { + self.shared.engine.capability() + } + + /// Begin a transaction, acquiring a connection from the pool. + /// + /// Takes `&mut self` so the `Db` handle is exclusively borrowed while the + /// transaction is open. This prevents accidentally running a statement + /// against the pool — which would execute on a separate connection and + /// bypass the transaction — when you meant to run it against `&mut tx`. + /// + /// If you need a second handle while the transaction is open, clone the + /// `Db` before calling this method. Clones share the same pool. + pub async fn transaction(&mut self) -> Result> { + ::transaction(self).await + } + + /// Returns a [`TransactionBuilder`] that will acquire a connection from + /// the pool when [`begin`](TransactionBuilder::begin) is called. + /// + /// Like [`transaction`](Self::transaction), this takes `&mut self` so the + /// `Db` handle stays locked for the lifetime of the transaction. Clone + /// the `Db` beforehand if you need a separate handle. + pub fn transaction_builder(&mut self) -> TransactionBuilder<'_> { + TransactionBuilder::new(tx::TxSource::Db(self)) + } + + /// Returns a reference to the connection pool backing this handle. + #[doc(hidden)] + pub fn pool(&self) -> &Pool { + &self.shared.pool + } +} + +impl std::fmt::Debug for Db { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Db") + .field("engine", &self.shared.engine) + .finish() + } +} + +#[async_trait] +impl Executor for Db { + async fn transaction(&mut self) -> Result> { + let conn = self.connection().await?; + Transaction::begin(ConnRef::owned(conn)).await + } + + async fn exec_untyped(&mut self, stmt: stmt::Statement) -> Result { + self.exec_stmt(stmt, false).await + } + + async fn exec_raw_sql(&mut self, raw: RawSql) -> Result { + let conn = self.connection().await?; + conn.exec_raw_sql(raw).await + } + + fn capability(&mut self) -> &Capability { + Db::capability(self) + } + + fn schema(&mut self) -> &Arc { + Db::schema(self) + } +} diff --git a/packages/toasty/src/db/builder.rs b/packages/toasty/src/db/builder.rs new file mode 100644 index 00000000000..3a90fa80965 --- /dev/null +++ b/packages/toasty/src/db/builder.rs @@ -0,0 +1,310 @@ +use crate::{ + Db, Result, + db::{Connect, Pool, Shared, pool::PoolConfig}, + engine::Engine, +}; + +use toasty_core::{ + driver::Driver, + schema::{ + self, + app::{self, ModelSet}, + }, +}; + +use std::{sync::Arc, time::Duration}; + +/// Configures the schema and driver for a [`Db`] instance. +/// +/// Provide model types with [`models`](Self::models) (using the +/// [`models!`](crate::models!) macro), optionally set a table name prefix with +/// [`table_name_prefix`](Self::table_name_prefix), then call +/// [`connect`](Self::connect) (URL-based) or [`build`](Self::build) (driver +/// instance) to open the database. +/// +/// # Examples +/// +/// ``` +/// # tokio::runtime::Runtime::new().unwrap().block_on(async { +/// # #[derive(Debug, toasty::Model)] +/// # struct User { +/// # #[key] +/// # id: i64, +/// # name: String, +/// # } +/// let driver = toasty_driver_sqlite::Sqlite::in_memory(); +/// let db = toasty::Db::builder() +/// .models(toasty::models!(User)) +/// .build(driver) +/// .await +/// .unwrap(); +/// # }); +/// ``` +#[derive(Default)] +pub struct Builder { + /// Model definitions from macro + /// + /// TODO: move this into `core::schema::Builder` after old schema file + /// implementatin is removed. + models: app::ModelSet, + + /// Schema builder + core: schema::Builder, + + /// Connection pool configuration + pool: PoolConfig, +} + +impl Builder { + /// Set the models whose schemas will be included when the database is + /// built. Related and embedded models are discovered automatically through + /// field traversal, so you only need to list your entry-point models. + /// + /// Use the [`models!`](crate::models!) macro to build a [`ModelSet`] from + /// your `#[derive(Model)]` types. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// let mut builder = toasty::Db::builder(); + /// builder.models(toasty::models!(User)); + /// ``` + pub fn models(&mut self, models: ModelSet) -> &mut Self { + self.models = models; + self + } + + /// Set the table name prefix for all tables + pub fn table_name_prefix(&mut self, prefix: &str) -> &mut Self { + self.core.table_name_prefix(prefix); + self + } + + /// Set the maximum number of connections the pool will maintain. + /// + /// Defaults to `num_cpus * 2`. + /// + /// Drivers may cap this below the requested value. For example, an + /// in-memory SQLite database forces a single connection. When that + /// happens a warning is emitted and the driver cap is used. + pub fn max_pool_size(&mut self, max_size: usize) -> &mut Self { + self.pool.max_size = max_size; + self + } + + /// Set the maximum time to wait for a free connection from the pool + /// before returning an error. Passing `None` disables the timeout, + /// which is the default. + pub fn pool_wait_timeout(&mut self, timeout: Option) -> &mut Self { + self.pool.timeouts.wait = timeout; + self + } + + /// Set the maximum time allowed for establishing a new database + /// connection. Passing `None` disables the timeout, which is the + /// default. + pub fn pool_create_timeout(&mut self, timeout: Option) -> &mut Self { + self.pool.timeouts.create = timeout; + self + } + + /// Configure how often the pool's background sweep pings an idle + /// connection to detect a silently-broken backend (a database + /// restart, a load-balancer-closed socket, a session timeout). On + /// success the connection is returned as most-recently-used; on + /// failure the pool eagerly pings every other idle connection and + /// drops the ones that fail, so a single bad result drains every + /// dead connection in one pass. + /// + /// The same eager sweep also fires when a user query observes + /// [`Error::is_connection_lost`](toasty_core::Error::is_connection_lost) + /// — so a restart usually costs at most one failed query rather + /// than one per pooled connection. + /// + /// Defaults to `Some(60s)`. Pass `None` to disable the sweep and + /// rely on passive error-driven recovery only. + pub fn pool_health_check_interval(&mut self, interval: Option) -> &mut Self { + self.pool.health_check_interval = interval; + self + } + + /// Validate every connection with an active ping before handing it + /// to the caller. Useful for deployments that cannot tolerate even + /// one failed user query — a public API behind a flaky network, an + /// idempotent worker that does not implement retry. A failing ping + /// evicts the connection and the pool reuses another idle slot or + /// opens a fresh one; the caller sees either a healthy connection + /// or a clean `connection_pool` error if no slot can be opened + /// within [`pool_create_timeout`](Self::pool_create_timeout). + /// + /// The trade-off is one round-trip per checkout. Combine with a + /// larger [`max_pool_size`](Self::max_pool_size) if the extra + /// latency starts queueing requests. Independent of the background + /// sweep — most deployments want one or the other, but enabling + /// both is safe. + /// + /// Defaults to `false`. + pub fn pool_pre_ping(&mut self, pre_ping: bool) -> &mut Self { + self.pool.pre_ping = pre_ping; + self + } + + /// Evict any pooled connection older than this duration when the + /// pool considers reusing it. Useful when an idle timeout on the + /// server, a load balancer, or a NAT in front of the database + /// silently closes long-lived sockets — capping the lifetime + /// bounds how long a connection can survive past such a close. + /// + /// The check runs in `recycle` (when the pool considers handing + /// the connection back out), not in the background. A query that + /// holds a connection past the cap is allowed to finish; the + /// connection is evicted on its next return. + /// + /// Recommended for any deployment that talks to a remote database: + /// pick a duration shorter than every idle timeout in the path + /// (server, load balancer, NAT). 30 minutes works for most clouds. + /// + /// Defaults to `None` (no cap). + pub fn pool_max_connection_lifetime(&mut self, lifetime: Option) -> &mut Self { + self.pool.max_connection_lifetime = lifetime; + self + } + + /// Include bound parameter values in the per-query `toasty::query` + /// tracing event. + /// + /// Off by default: parameter values are application data and may + /// contain secrets (password hashes, personal information). Enable + /// only when the log destination is trusted. Values are rendered in + /// a bounded form — long strings are truncated and byte blobs are + /// summarized by length. + pub fn log_statement_params(&mut self, enabled: bool) -> &mut Self { + self.pool.query_log.params = enabled; + self + } + + /// Emit the per-query `toasty::query` tracing event at `WARN` instead + /// of `DEBUG` when a statement takes at least this long, so slow + /// queries surface in production logs without enabling debug output. + /// + /// Defaults to `Some(1s)`. Pass `None` to disable the escalation. + pub fn slow_statement_threshold(&mut self, threshold: Option) -> &mut Self { + self.pool.query_log.slow_statement_threshold = threshold; + self + } + + /// Evict any pooled connection that has been sitting unused for + /// longer than this duration. Complements + /// [`pool_max_connection_lifetime`](Self::pool_max_connection_lifetime): + /// the idle cap targets connections specifically held idle past a + /// known timeout, the lifetime cap targets all connections + /// regardless of recent use. + /// + /// Checked in `recycle`, not in the background. + /// + /// Defaults to `None` (no cap). + pub fn pool_max_connection_idle_time(&mut self, idle: Option) -> &mut Self { + self.pool.max_connection_idle_time = idle; + self + } + + /// Build and return the app-level schema from the registered models + /// without opening a database connection. + /// + /// This is useful for tooling that needs the schema without a running + /// database (e.g., migration generators). + pub fn build_app_schema(&self) -> Result { + app::Schema::from_macro(self.models.clone()) + } + + /// Open a database connection using a URL string. + /// + /// The URL scheme selects the driver (`sqlite://`, `postgresql://`, + /// `mysql://`, `dynamodb://`). The corresponding feature flag must be + /// enabled. + /// + /// # Errors + /// + /// Returns an error if the URL cannot be parsed, the scheme is + /// unsupported or its feature flag is not enabled, or the initial + /// connection fails. + /// + /// # Examples + /// + /// ``` + /// # tokio::runtime::Runtime::new().unwrap().block_on(async { + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// # #[cfg(feature = "sqlite")] + /// let db = toasty::Db::builder() + /// .models(toasty::models!(User)) + /// .connect("sqlite::memory:") + /// .await + /// .unwrap(); + /// # }); + /// ``` + pub async fn connect(&mut self, url: &str) -> Result { + let con = Connect::new(url).await?; + self.build(con).await + } + + /// Build a [`Db`] from an already-constructed driver instance. + /// + /// Use this instead of [`connect`](Self::connect) when you need to + /// configure the driver yourself (e.g., an in-memory SQLite database for + /// tests). + /// + /// # Errors + /// + /// Returns an error if the driver reports invalid capabilities or the + /// initial connection cannot be acquired. + /// + /// # Examples + /// + /// ``` + /// # tokio::runtime::Runtime::new().unwrap().block_on(async { + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// let driver = toasty_driver_sqlite::Sqlite::in_memory(); + /// let db = toasty::Db::builder() + /// .models(toasty::models!(User)) + /// .build(driver) + /// .await + /// .unwrap(); + /// # }); + /// ``` + pub async fn build(&mut self, driver: impl Driver) -> Result { + tracing::info!(models = self.models.len(), "building database schema"); + let capability = driver.capability(); + capability.validate()?; + let schema = self.core.build(self.build_app_schema()?, capability)?; + + tracing::info!(tables = schema.db.tables.len(), "schema built successfully"); + + let engine = Engine::new(Arc::new(schema), capability); + let pool = Pool::new(driver, engine.clone(), self.pool.clone())?; + + let shared = Arc::new(Shared { engine, pool }); + + // see if we're able to acquire a valid connection + let conn = shared.pool.get(shared.clone()).await?; + std::mem::drop(conn); + + tracing::info!("database ready"); + Ok(Db { shared }) + } +} diff --git a/packages/toasty/src/db/connect.rs b/packages/toasty/src/db/connect.rs new file mode 100644 index 00000000000..fbaa9220209 --- /dev/null +++ b/packages/toasty/src/db/connect.rs @@ -0,0 +1,160 @@ +use crate::Result; + +use async_trait::async_trait; +use std::borrow::Cow; +use toasty_core::driver::{Capability, ConnectContext, ConnectionUrl, Driver}; +use toasty_core::{ + driver::Connection, + schema::{db::Migration, diff}, +}; + +/// A connection to a database, wrapping the specific driver implementation. +pub struct Connect { + driver: Box, +} + +impl std::fmt::Debug for Connect { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Connect") + .field("driver", &self.driver) + .finish() + } +} + +impl Connect { + /// Create a new connection by parsing a database URL and constructing the + /// appropriate driver. + /// + /// The URL scheme determines which driver is used: + /// + /// | Scheme | Driver | Feature flag | + /// |---|---|---| + + /// | `postgresql` / `postgres` | PostgreSQL | `postgresql` | + /// | `mysql` | MySQL | `mysql` | + /// | `mariadb` | MariaDB | `mariadb` | + /// | `dynamodb` | DynamoDB | `dynamodb` | + /// | `turso` | Turso | `turso` | + /// + /// # Errors + /// + /// Returns an error if the URL is malformed, the scheme is unrecognized, + /// or the required feature flag is not enabled. + pub async fn new(url: &str) -> Result { + #![cfg_attr( + not(any( + feature = "dynamodb", + feature = "mysql", + feature = "mariadb", + feature = "postgresql", + feature = "turso" + )), + allow(unused_variables, unreachable_code) + )] + + let url = ConnectionUrl::parse(url)?; + let scheme = url.scheme().to_ascii_lowercase(); + + let driver: Box = match scheme.as_str() { + #[cfg(feature = "dynamodb")] + "dynamodb" => { + // DynamoDB driver requires async initialization to load AWS config from environment + // Spawn a new thread to avoid runtime context issues + let url = url.as_str().to_string(); + let driver = + toasty_driver_dynamodb::DynamoDb::from_env(url).await?; + Box::new(driver) + } + #[cfg(not(feature = "dynamodb"))] + "dynamodb" => { + return Err(toasty_core::Error::unsupported_feature( + "`dynamodb` feature not enabled", + )); + } + + #[cfg(feature = "mysql")] + "mysql" => Box::new(toasty_driver_mysql::MySQL::new(url.as_str())?), + #[cfg(not(feature = "mysql"))] + "mysql" => { + return Err(toasty_core::Error::unsupported_feature( + "`mysql` feature not enabled", + )); + } + + #[cfg(feature = "mariadb")] + "mariadb" => { + Box::new(toasty_driver_mysql::MariaDB::new(url.as_str())?) + } + #[cfg(not(feature = "mariadb"))] + "mariadb" => { + return Err(toasty_core::Error::unsupported_feature( + "`mariadb` feature not enabled", + )); + } + + #[cfg(feature = "postgresql")] + "postgresql" | "postgres" => Box::new( + toasty_driver_postgresql::PostgreSQL::new(url.as_str())?, + ), + #[cfg(not(feature = "postgresql"))] + "postgresql" | "postgres" => { + return Err(toasty_core::Error::unsupported_feature( + "`postgresql` feature not enabled", + )); + } + + "sqlite" => { + return Err(toasty_core::Error::unsupported_feature( + "SQLite support is disabled in the vendored Toasty crate", + )); + } + + #[cfg(feature = "turso")] + "turso" => Box::new(toasty_driver_turso::Turso::new(url.as_str())?), + #[cfg(not(feature = "turso"))] + "turso" => { + return Err(toasty_core::Error::unsupported_feature( + "`turso` feature not enabled", + )); + } + + scheme => { + return Err(toasty_core::Error::unsupported_feature(format!( + "unsupported database scheme `{scheme}`" + ))); + } + }; + + Ok(Self { driver }) + } +} + +#[async_trait] +impl Driver for Connect { + fn url(&self) -> Cow<'_, str> { + self.driver.url() + } + + fn capability(&self) -> &'static Capability { + self.driver.capability() + } + + async fn connect( + &self, + cx: &ConnectContext, + ) -> Result> { + self.driver.connect(cx).await + } + + fn max_connections(&self) -> Option { + self.driver.max_connections() + } + + fn generate_migration(&self, schema_diff: &diff::Schema<'_>) -> Migration { + self.driver.generate_migration(schema_diff) + } + + async fn reset_db(&self) -> Result<()> { + self.driver.reset_db().await + } +} diff --git a/packages/toasty/src/db/connection.rs b/packages/toasty/src/db/connection.rs new file mode 100644 index 00000000000..d0a97592392 --- /dev/null +++ b/packages/toasty/src/db/connection.rs @@ -0,0 +1,200 @@ +use super::Transaction; +use super::connection_task::{ConnectionHandle, ConnectionOperation}; +use super::pool::Manager; +use super::tx::ConnRef; + +use async_trait::async_trait; +use std::sync::Arc; +#[cfg(feature = "migration")] +use toasty_core::schema::db::{AppliedMigration, Migration}; +use toasty_core::{ + Schema, + driver::{ + Capability, ExecResponse, + operation::{Operation, RawSql}, + }, + stmt, +}; +use tokio::sync::oneshot; +use tracing::Instrument; + +/// A dedicated database connection retrieved from a pool. +/// +/// Holding a `Connection` guarantees that all operations are executed on the +/// same physical connection. This is useful when multiple statements must +/// share connection-level state such as temporary tables or session variables. +/// +/// When dropped, the connection is returned to the pool for reuse. +pub struct Connection { + pub(super) inner: deadpool::managed::Object, + pub(super) shared: Arc, +} + +impl Connection { + /// Access the underlying connection handle. + pub(crate) fn handle(&self) -> &ConnectionHandle { + &self.inner + } + + /// Returns the compiled schema used by this connection. + pub fn schema(&self) -> &Arc { + &self.shared.engine.schema + } + + pub(crate) async fn exec_stmt( + &self, + stmt: stmt::Statement, + in_transaction: bool, + ) -> crate::Result { + // Created on the caller's task so the span parents to the caller's + // current span; the worker task enters it while executing. + let span = crate::instrument::query_span(&self.shared.engine.schema, &stmt); + + let (tx, rx) = oneshot::channel(); + + self.handle() + .in_tx + .send(ConnectionOperation::ExecStatement { + stmt: Box::new(stmt), + in_transaction, + span: span.clone(), + tx, + }) + .unwrap(); + + rx.instrument(span).await.unwrap() + } + + pub(crate) async fn exec_operation(&self, operation: Operation) -> crate::Result { + let (tx, rx) = oneshot::channel(); + + self.handle() + .in_tx + .send(ConnectionOperation::ExecOperation { + operation: Box::new(operation), + span: tracing::Span::current(), + tx, + }) + .unwrap(); + + rx.await.unwrap() + } + + pub(crate) async fn exec_raw_sql(&self, raw: RawSql) -> crate::Result { + let span = crate::instrument::raw_sql_span(); + + let (tx, rx) = oneshot::channel(); + + self.handle() + .in_tx + .send(ConnectionOperation::ExecRawSql { + raw: Box::new(raw), + span: span.clone(), + tx, + }) + .unwrap(); + + rx.instrument(span).await.unwrap() + } + + /// Begin a transaction on this connection. + /// + /// Takes `&mut self` so the `Connection` is exclusively borrowed while + /// the transaction is open. This prevents statements from running on the + /// connection directly — bypassing the transaction — when they should + /// have gone through `&mut tx`. + pub async fn transaction(&mut self) -> crate::Result> { + ::transaction(self).await + } + + /// Returns a [`TransactionBuilder`](super::TransactionBuilder) that will + /// use this connection. + /// + /// Like [`transaction`](Self::transaction), this takes `&mut self` so the + /// `Connection` stays locked for the lifetime of the transaction. + pub fn transaction_builder(&mut self) -> super::TransactionBuilder<'_> { + super::TransactionBuilder::new(super::tx::TxSource::Connection(self)) + } + + /// Creates tables and indices defined in the schema on the database. + pub async fn push_schema(&self) -> crate::Result<()> { + tracing::info!("pushing schema to database"); + let (tx, rx) = oneshot::channel(); + self.handle() + .in_tx + .send(ConnectionOperation::PushSchema { + span: tracing::Span::current(), + tx, + }) + .unwrap(); + rx.await.unwrap() + } + + #[cfg(feature = "migration")] + pub(crate) async fn applied_migrations(&self) -> crate::Result> { + let (tx, rx) = oneshot::channel(); + self.handle() + .in_tx + .send(ConnectionOperation::AppliedMigrations { + span: tracing::Span::current(), + tx, + }) + .unwrap(); + rx.await.unwrap() + } + + #[cfg(feature = "migration")] + pub(crate) async fn apply_migration( + &self, + id: u64, + name: &str, + migration: Migration, + ) -> crate::Result<()> { + let (tx, rx) = oneshot::channel(); + self.handle() + .in_tx + .send(ConnectionOperation::ApplyMigration { + id, + name: name.to_owned(), + migration: Box::new(migration), + span: tracing::Span::current(), + tx, + }) + .unwrap(); + rx.await.unwrap() + } +} + +#[async_trait] +impl super::Executor for Connection { + async fn transaction(&mut self) -> crate::Result> { + Transaction::begin(ConnRef::Borrowed(self)).await + } + + async fn exec_untyped( + &mut self, + stmt: toasty_core::stmt::Statement, + ) -> crate::Result { + self.exec_stmt(stmt, false).await + } + + async fn exec_raw_sql(&mut self, raw: RawSql) -> crate::Result { + Connection::exec_raw_sql(self, raw).await + } + + fn capability(&mut self) -> &Capability { + self.shared.engine.capability() + } + + fn schema(&mut self) -> &Arc { + Connection::schema(self) + } +} + +impl std::fmt::Debug for Connection { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Connection") + .field("handle", &*self.inner) + .finish() + } +} diff --git a/packages/toasty/src/db/connection_task.rs b/packages/toasty/src/db/connection_task.rs new file mode 100644 index 00000000000..0af611e76b9 --- /dev/null +++ b/packages/toasty/src/db/connection_task.rs @@ -0,0 +1,273 @@ +//! Per-connection worker task and its message protocol. +//! +//! Each pooled connection is owned by a [`ConnectionTask`] running in a +//! dedicated tokio task; callers interact with it through a +//! [`ConnectionHandle`] by sending [`ConnectionOperation`] messages over +//! an mpsc channel and awaiting a oneshot response. This serializes +//! access to the underlying driver connection — drivers are not +//! required to be `Sync` on `exec` — and gives the connection its own +//! lifecycle independent of the pool object that holds the handle. + +use std::sync::Arc; + +use toasty_core::driver::operation::RawSql; +use toasty_core::driver::{Connection, Rows}; +#[cfg(feature = "migration")] +use toasty_core::schema::db::{AppliedMigration, Migration}; +use toasty_core::stmt::Value; +use tokio::{ + sync::{mpsc, oneshot}, + task::JoinHandle, +}; +use tracing::Instrument; + +use super::pool::SweepWaker; +use crate::engine::Engine; + +/// Operations sent to the connection task. +/// +/// Each variant carries the caller's `tracing::Span`, captured at send time. +/// The worker task runs on its own tokio task, so span context does not +/// propagate to it implicitly; the task enters the carried span while +/// handling the operation so engine and driver events land inside the +/// caller's span tree. +pub(crate) enum ConnectionOperation { + /// Execute a statement (compile + run on the connection). + ExecStatement { + stmt: Box, + in_transaction: bool, + span: tracing::Span, + tx: oneshot::Sender>, + }, + ExecOperation { + operation: Box, + span: tracing::Span, + tx: oneshot::Sender>, + }, + /// Execute user-authored SQL through the engine's SQL capability guard. + ExecRawSql { + raw: Box, + span: tracing::Span, + tx: oneshot::Sender>, + }, + /// Push schema to the database. + PushSchema { + span: tracing::Span, + tx: oneshot::Sender>, + }, + /// Read migration IDs recorded by the driver. + #[cfg(feature = "migration")] + AppliedMigrations { + span: tracing::Span, + tx: oneshot::Sender>>, + }, + /// Apply and record one migration. + #[cfg(feature = "migration")] + ApplyMigration { + id: u64, + name: String, + migration: Box, + span: tracing::Span, + tx: oneshot::Sender>, + }, + /// Active liveness probe issued by the pool's background sweep + /// (and, eventually, per-acquire pre-ping). Routes through the + /// connection task so the ping is serialized against any in-flight + /// `exec` on the same connection. + Ping { + span: tracing::Span, + tx: oneshot::Sender>, + }, +} + +/// Handle to a dedicated connection task. +/// +/// When dropped, `in_tx` closes the channel, causing the background task to +/// finish processing remaining messages and exit gracefully. +pub(crate) struct ConnectionHandle { + pub(crate) in_tx: mpsc::UnboundedSender, + join_handle: JoinHandle<()>, +} + +impl ConnectionHandle { + /// Spawn a worker task that owns `connection` and serializes + /// operations against it. `sweep_waker` is woken whenever the task + /// observes that the connection went bad so the pool's health-check + /// sweep can escalate immediately rather than waiting for its next + /// periodic tick. + pub(crate) fn spawn( + connection: Box, + engine: Engine, + sweep_waker: Arc, + ) -> Self { + let (in_tx, in_rx) = mpsc::unbounded_channel::(); + let task = ConnectionTask { + connection, + engine, + in_rx, + sweep_waker, + }; + let join_handle = tokio::spawn(task.run()); + Self { in_tx, join_handle } + } + + /// Returns true once the worker task has exited. Used by the pool's + /// `recycle` to detect dead slots. + pub(crate) fn is_finished(&self) -> bool { + self.join_handle.is_finished() + } +} + +impl std::fmt::Debug for ConnectionHandle { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ConnectionHandle") + .field("channel_closed", &self.in_tx.is_closed()) + .field("task_finished", &self.join_handle.is_finished()) + .finish() + } +} + +/// Owns one database connection and serializes operations against it. +/// Spawned by [`ConnectionHandle::spawn`]; runs until `in_rx` closes +/// (sender side dropped) or the connection reports `is_valid() == false`. +struct ConnectionTask { + connection: Box, + engine: Engine, + in_rx: mpsc::UnboundedReceiver, + sweep_waker: Arc, +} + +impl ConnectionTask { + async fn run(mut self) { + while let Some(op) = self.in_rx.recv().await { + if !self.handle(op).await { + return; + } + } + } + + /// Dispatch one operation, entering the caller's span for its duration. + /// Returns `false` if the connection went bad during it and the task + /// should exit. + async fn handle(&mut self, op: ConnectionOperation) -> bool { + match op { + ConnectionOperation::ExecStatement { + stmt, + in_transaction, + span, + tx, + } => { + let result = self + .exec_statement(*stmt, in_transaction) + .instrument(span) + .await; + self.respond(tx, result) + } + ConnectionOperation::ExecOperation { + operation, + span, + tx, + } => { + let result = self + .connection + .exec(&self.engine.schema, *operation) + .instrument(span) + .await; + self.respond(tx, result) + } + ConnectionOperation::ExecRawSql { raw, span, tx } => { + let result = self + .engine + .exec_raw_sql(&mut *self.connection, *raw) + .instrument(span) + .await; + self.respond(tx, result) + } + ConnectionOperation::PushSchema { span, tx } => { + let result = self + .connection + .push_schema(&self.engine.schema) + .instrument(span) + .await; + self.respond(tx, result) + } + #[cfg(feature = "migration")] + ConnectionOperation::AppliedMigrations { span, tx } => { + let result = self.connection.applied_migrations().instrument(span).await; + self.respond(tx, result) + } + #[cfg(feature = "migration")] + ConnectionOperation::ApplyMigration { + id, + name, + migration, + span, + tx, + } => { + let result = self + .connection + .apply_migration(id, &name, &migration) + .instrument(span) + .await; + self.respond(tx, result) + } + ConnectionOperation::Ping { span, tx } => { + let result = self.connection.ping().instrument(span).await; + self.respond(tx, result) + } + } + } + + async fn exec_statement( + &mut self, + stmt: toasty_core::stmt::Statement, + in_transaction: bool, + ) -> crate::Result { + let single = stmt.is_single(); + let mut response = self + .engine + .exec(&mut *self.connection, stmt, in_transaction) + .await?; + response.values.buffer().await?; + + if single { + let Rows::Value(Value::List(mut items)) = response.values else { + unreachable!() + }; + assert!( + items.len() <= 1, + "expected at most 1 row for single statement, got {}", + items.len() + ); + response.values = Rows::Value(items.pop().unwrap_or(Value::Null)); + } + + Ok(response) + } + + /// Send `result` to the caller, but if the connection reported + /// invalid during the op, close `in_rx` *first* so the mpsc transitions + /// to closed synchronously. A consumer woken by the response that then + /// re-enters the pool sees `in_tx.is_closed() == true` in + /// `Manager::recycle` and the slot is evicted — no race with this + /// task's exit. Returns whether the task should keep running. + /// + /// Also wakes the pool's background sweep so it can escalate to a + /// full idle-pool ping pass without waiting for the next periodic + /// tick. A real connection-lost error usually means more than one + /// connection in the pool is affected (database restart, network + /// event), and catching the rest eagerly turns a multi-second + /// recovery window into one round-trip per idle slot. + fn respond(&mut self, tx: oneshot::Sender, result: T) -> bool { + if self.connection.is_valid() { + let _ = tx.send(result); + true + } else { + tracing::debug!("connection reported invalid; closing channel and exiting"); + self.in_rx.close(); + self.sweep_waker.wake(); + let _ = tx.send(result); + false + } + } +} diff --git a/packages/toasty/src/db/executor.rs b/packages/toasty/src/db/executor.rs new file mode 100644 index 00000000000..da1c05abe39 --- /dev/null +++ b/packages/toasty/src/db/executor.rs @@ -0,0 +1,86 @@ +use crate::{Result, Statement, db::Transaction, schema::Load}; + +use async_trait::async_trait; +use std::sync::Arc; +use toasty_core::{ + Schema, + driver::{Capability, ExecResponse, operation::RawSql}, +}; + +/// Anything that can execute queries — [`Db`](crate::Db), +/// [`Connection`](crate::Connection), or [`Transaction`](crate::Transaction). +/// +/// This trait is dyn-compatible. The generic [`exec`](dyn Executor::exec) +/// method lives on `dyn Executor` and accepts any typed +/// [`Statement`](crate::Statement). +#[async_trait] +pub trait Executor: Send + Sync { + /// Starts a (potentially nested) transaction. + /// + /// Takes `&mut self` so the executor is exclusively borrowed while the + /// transaction is open. This prevents statements from accidentally + /// executing on the parent handle rather than through `&mut tx`, which + /// would bypass the transaction. + async fn transaction(&mut self) -> Result>; + + /// Execute an untyped statement, returning the full execution response. + #[doc(hidden)] + async fn exec_untyped(&mut self, stmt: toasty_core::stmt::Statement) -> Result; + + /// Execute user-authored SQL, returning the full execution response. + #[doc(hidden)] + async fn exec_raw_sql(&mut self, raw: RawSql) -> Result; + + /// Returns the capabilities associated with this executor. + #[doc(hidden)] + fn capability(&mut self) -> &Capability; + + /// Returns the schema associated with this executor. + #[doc(hidden)] + fn schema(&mut self) -> &Arc; +} + +impl dyn Executor + '_ { + /// Execute a typed [`Statement`] and deserialize the result. + /// + /// This is the primary entry point for running queries, inserts, updates, + /// and deletes. The return type is determined by the statement's type + /// parameter `T`: + /// + /// - `Statement>` returns `Vec`. + /// - `Statement` returns `M`. + /// - `Statement>` returns `Option`. + /// - `Statement<()>` returns `()`. + /// + /// Most users call `exec` on the query/update/delete builders directly + /// (e.g., [`Query::exec`](crate::stmt::Query::exec)) rather than calling + /// this method, but it is available for working with [`Statement`] values + /// directly. + /// + /// # Examples + /// + /// ``` + /// # tokio::runtime::Runtime::new().unwrap().block_on(async { + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// # let driver = toasty_driver_sqlite::Sqlite::in_memory(); + /// # let mut db = toasty::Db::builder().models(toasty::models!(User)).build(driver).await.unwrap(); + /// # db.push_schema().await.unwrap(); + /// use toasty::stmt::{IntoStatement, List, Query}; + /// use toasty::Executor; + /// + /// let stmt = Query::>::all().into_statement(); + /// let executor: &mut dyn Executor = &mut db; + /// let users: Vec = executor.exec(stmt).await.unwrap(); + /// # }); + /// ``` + pub async fn exec(&mut self, stmt: Statement) -> Result { + let response = self.exec_untyped(stmt.untyped).await?; + let value = response.values.collect_as_value().await?; + T::load(value) + } +} diff --git a/packages/toasty/src/db/pool.rs b/packages/toasty/src/db/pool.rs new file mode 100644 index 00000000000..7d943d7d6b5 --- /dev/null +++ b/packages/toasty/src/db/pool.rs @@ -0,0 +1,471 @@ +//! Connection pooling for database connections. + +use deadpool::managed::{Object, Timeouts}; +use std::{ + sync::{ + Arc, + atomic::{AtomicU64, Ordering}, + }, + time::Duration, +}; +use toasty_core::driver::{Capability, ConnectContext, Driver, QueryLogConfig}; +use tokio::{sync::Notify, task::JoinHandle}; + +use super::connection_task::{ConnectionHandle, ConnectionOperation}; +use crate::engine::Engine; + +/// Get the default maximum size of a pool, which is `cpu_core_count * 2` +/// including logical cores (Hyper-Threading). +fn get_default_pool_max_size() -> usize { + deadpool::managed::PoolConfig::default().max_size +} + +/// How long the sweep task waits for a `Ping` to come back before +/// treating the connection as dead. Short enough that one stuck +/// connection cannot stall the sweep loop; long enough to ride out a +/// normal round-trip on a busy database. +const DEFAULT_SWEEP_PING_TIMEOUT: Duration = Duration::from_secs(5); + +/// Configuration for connection pool behavior. +#[derive(Debug, Clone)] +pub(crate) struct PoolConfig { + pub(crate) max_size: usize, + pub(crate) timeouts: Timeouts, + pub(crate) health_check_interval: Option, + pub(crate) pre_ping: bool, + pub(crate) max_connection_lifetime: Option, + pub(crate) max_connection_idle_time: Option, + /// Per-query tracing configuration applied to each connection the + /// pool creates. + pub(crate) query_log: QueryLogConfig, +} + +impl Default for PoolConfig { + fn default() -> Self { + Self { + max_size: get_default_pool_max_size(), + timeouts: Default::default(), + health_check_interval: Some(Duration::from_secs(60)), + pre_ping: false, + max_connection_lifetime: None, + max_connection_idle_time: None, + query_log: QueryLogConfig::default(), + } + } +} + +/// A connection pool that manages database connections with background tasks. +#[derive(Debug)] +pub struct Pool { + inner: deadpool::managed::Pool, + capability: &'static Capability, + /// Handle for the background health-check sweep, if one was spawned. + /// Aborted on `Pool::drop` so the task does not outlive the pool. + sweep_task: Option>, +} + +impl Drop for Pool { + fn drop(&mut self) { + if let Some(handle) = self.sweep_task.take() { + handle.abort(); + } + } +} + +impl Pool { + /// Creates a new connection pool from the given driver, engine, and + /// configuration. + pub(crate) fn new( + driver: impl Driver, + engine: Engine, + config: PoolConfig, + ) -> crate::Result { + let capability = driver.capability(); + let driver_cap = driver.max_connections(); + + let effective_max = match driver_cap { + Some(cap) if cap < config.max_size => { + tracing::warn!( + requested = config.max_size, + cap, + "driver caps max pool size below the requested value; using driver cap" + ); + cap + } + _ => config.max_size, + }; + + let sweep_waker = Arc::new(SweepWaker::new()); + + let mut connect_cx = ConnectContext::default(); + connect_cx.query_log = config.query_log; + + let inner = deadpool::managed::Pool::builder(Manager { + driver: Box::new(driver), + engine, + sweep_waker: sweep_waker.clone(), + pre_ping: config.pre_ping, + max_connection_lifetime: config.max_connection_lifetime, + max_connection_idle_time: config.max_connection_idle_time, + connect_cx, + }) + .runtime(deadpool::Runtime::Tokio1) + .max_size(effective_max) + .timeouts(config.timeouts) + .build() + .map_err(|e| { + tracing::error!(error = %e, "failed to build connection pool"); + toasty_core::Error::connection_pool(e) + })?; + + let sweep_task = config.health_check_interval.map(|interval| { + let task = SweepTask { + pool: inner.clone(), + waker: sweep_waker, + interval, + last_serviced: 0, + }; + tokio::spawn(task.run()) + }); + + Ok(Self { + inner, + capability, + sweep_task, + }) + } + + /// Retrieves a connection from the pool. + pub(crate) async fn get(&self, shared: Arc) -> crate::Result { + let connection = self.inner.get().await.map_err(|e| { + tracing::debug!(error = %e, "failed to acquire connection from pool"); + toasty_core::Error::connection_pool(e) + })?; + Ok(super::Connection { + inner: connection, + shared, + }) + } + + /// Returns the database driver this pool uses to create connections. + pub fn driver(&self) -> &dyn Driver { + self.inner.manager().driver.as_ref() + } + + /// Returns the database driver's capabilities. + pub fn capability(&self) -> &'static Capability { + self.capability + } + + /// Returns the current status of the pool, including the number of + /// connections, how many are available, and how many waiters are queued. + pub fn status(&self) -> PoolStatus { + let s = self.inner.status(); + PoolStatus { + max_size: s.max_size, + size: s.size, + available: s.available, + waiting: s.waiting, + } + } +} + +pub(super) struct Manager { + driver: Box, + engine: Engine, + sweep_waker: Arc, + pre_ping: bool, + // Enforcing these caps needs the connection's age, which deadpool only + // tracks off wasm32 (`std::time::Instant` has no wasm32 implementation). + #[cfg_attr( + target_arch = "wasm32", + expect(dead_code, reason = "connection age is unavailable on wasm32") + )] + max_connection_lifetime: Option, + #[cfg_attr( + target_arch = "wasm32", + expect(dead_code, reason = "connection age is unavailable on wasm32") + )] + max_connection_idle_time: Option, + /// Per-connection configuration handed to `Driver::connect` for every + /// connection the pool creates. + connect_cx: ConnectContext, +} + +impl std::fmt::Debug for Manager { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Manager") + .field("driver", &self.driver) + .finish() + } +} + +impl deadpool::managed::Manager for Manager { + type Type = ConnectionHandle; + type Error = crate::Error; + + async fn create(&self) -> Result { + tracing::debug!("creating new pooled connection"); + let connection = self + .driver + .connect(&self.connect_cx) + .await + .inspect_err(|e| { + tracing::debug!(error = %e, "failed to create database connection"); + })?; + Ok(ConnectionHandle::spawn( + connection, + self.engine.clone(), + self.sweep_waker.clone(), + )) + } + + async fn recycle( + &self, + obj: &mut Self::Type, + #[cfg_attr( + target_arch = "wasm32", + expect(unused_variables, reason = "no timestamps on wasm32") + )] + metrics: &deadpool::managed::Metrics, + ) -> deadpool::managed::RecycleResult { + // `Metrics::age()` and `Metrics::last_used()` only exist off wasm32: + // deadpool records no timestamps there because `std::time::Instant` + // has no wasm32 implementation, so neither cap can be enforced. + #[cfg(not(target_arch = "wasm32"))] + if let Some(max) = self.max_connection_lifetime + && metrics.age() >= max + { + tracing::debug!(?max, "discarding pooled connection past max lifetime"); + return Err(deadpool::managed::RecycleError::message( + "connection exceeded max lifetime", + )); + } + #[cfg(not(target_arch = "wasm32"))] + if let Some(max) = self.max_connection_idle_time + && metrics.last_used() >= max + { + tracing::debug!(?max, "discarding pooled connection past max idle time"); + return Err(deadpool::managed::RecycleError::message( + "connection exceeded max idle time", + )); + } + if obj.in_tx.is_closed() || obj.is_finished() { + tracing::debug!("discarding dead pooled connection"); + return Err(deadpool::managed::RecycleError::message( + "background task is no longer running", + )); + } + if self.pre_ping { + let (tx, rx) = tokio::sync::oneshot::channel(); + let ping = ConnectionOperation::Ping { + span: tracing::Span::current(), + tx, + }; + if obj.in_tx.send(ping).is_err() { + tracing::debug!("pre-ping channel closed; discarding pooled connection"); + return Err(deadpool::managed::RecycleError::message( + "background task is no longer running", + )); + } + match rx.await { + Ok(Ok(())) => {} + Ok(Err(err)) => { + tracing::debug!(error = %err, "pre-ping failed; discarding pooled connection"); + return Err(deadpool::managed::RecycleError::Backend(err)); + } + Err(_) => { + tracing::debug!( + "pre-ping response channel dropped; discarding pooled connection" + ); + return Err(deadpool::managed::RecycleError::message( + "background task exited during pre-ping", + )); + } + } + } + tracing::trace!("recycling pooled connection"); + Ok(()) + } +} + +/// Bundles the sweep `Notify` with a monotonic request counter. A +/// user-task that observes connection-lost calls `wake`, which bumps +/// the counter *before* notifying. The sweep task compares the +/// post-notify counter against the snapshot taken at the start of its +/// most recent escalate to decide whether the notify has already been +/// covered by an in-flight or just-completed sweep. +pub(crate) struct SweepWaker { + requests: AtomicU64, + notify: Notify, +} + +impl SweepWaker { + fn new() -> Self { + Self { + requests: AtomicU64::new(0), + notify: Notify::new(), + } + } + + /// Called when a caller observes `connection_lost` and wants the + /// sweep to escalate. The counter bump happens-before the + /// `notify_one`, so the sweep task is guaranteed to load a value + /// at least as large as the one this caller produced. + pub(crate) fn wake(&self) { + self.requests.fetch_add(1, Ordering::Relaxed); + self.notify.notify_one(); + } +} + +/// Background task that periodically pings the longest-idle connection +/// and escalates to a full idle-pool sweep on failure (either a failing +/// periodic ping or a notify from a user-observed connection-lost). +struct SweepTask { + pool: deadpool::managed::Pool, + waker: Arc, + interval: Duration, + /// Highest `waker.requests` value covered by a completed escalate. + /// The notify branch skips when this is already ≥ the current + /// counter, since some earlier escalate has already serviced every + /// outstanding request. + last_serviced: u64, +} + +impl SweepTask { + async fn run(mut self) { + let mut ticker = tokio::time::interval(self.interval); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + // `interval` fires immediately on first poll; skip that tick so + // the first real ping happens one interval into the pool's life. + ticker.tick().await; + + loop { + tokio::select! { + _ = ticker.tick() => { + self.periodic_iteration().await; + } + _ = self.waker.notify.notified() => { + if self.waker.requests.load(Ordering::Relaxed) <= self.last_serviced { + // An earlier escalate (periodic-failure path or + // a prior notify) already covered every request + // outstanding when this notify was issued. + tracing::trace!("sweep notify already serviced; skipping"); + continue; + } + tracing::debug!("sweep woken by observed connection_lost; escalating"); + self.escalate().await; + } + } + } + } + + /// One periodic tick: pop the longest-idle connection, ping it, + /// return it on success or drop it (and escalate) on failure. + async fn periodic_iteration(&mut self) { + if self.pool.status().available == 0 { + return; + } + let Some(conn) = self.try_get_idle().await else { + return; + }; + if Self::ping_conn(&conn).await { + // Healthy — drop returns the connection to the pool. A + // successful ping has just touched the connection, so the + // longest-idle selector naturally picks a different slot + // on the next tick. + drop(conn); + } else { + // Failed — detach the slot from the pool. The task has + // already exited (respond closed the channel). + let _ = Object::take(conn); + self.escalate().await; + } + } + + /// Walk every currently-idle connection, ping each, drop the + /// failures and return the healthy ones. Bounded by the snapshot + /// of `status().available` at entry so a busy producer cannot keep + /// the sweep looping indefinitely. + async fn escalate(&mut self) { + // Snapshot the request counter *before* any pings start. Any + // user observation that arrives during the loop below + // increments past this value and will trigger another escalate + // on the next iteration; observations from before the snapshot + // are covered by this pass. + let snap = self.waker.requests.load(Ordering::Relaxed); + + let budget = self.pool.status().available; + let mut healthy = Vec::with_capacity(budget); + for _ in 0..budget { + let Some(conn) = self.try_get_idle().await else { + break; + }; + if Self::ping_conn(&conn).await { + healthy.push(conn); + } else { + let _ = Object::take(conn); + } + } + // Healthy connections return to the pool when `healthy` drops. + drop(healthy); + + self.last_serviced = snap; + } + + /// Non-blocking acquire that only returns an existing idle slot — + /// never creates a new connection. `wait = ZERO` makes the + /// semaphore acquire non-blocking; if no permit is available + /// (every slot is checked out by a user), `timeout_get` returns + /// `Timeout(Wait)` and we skip. + async fn try_get_idle(&self) -> Option> { + let timeouts = Timeouts { + wait: Some(Duration::ZERO), + create: Some(Duration::ZERO), + recycle: self.pool.timeouts().recycle, + }; + self.pool.timeout_get(&timeouts).await.ok() + } + + /// Send a `Ping` operation through the connection task, bounded by + /// `DEFAULT_SWEEP_PING_TIMEOUT`. Returns `true` iff the ping reported a + /// healthy connection. + async fn ping_conn(handle: &ConnectionHandle) -> bool { + let (tx, rx) = tokio::sync::oneshot::channel(); + let ping = ConnectionOperation::Ping { + span: tracing::Span::current(), + tx, + }; + if handle.in_tx.send(ping).is_err() { + return false; + } + match tokio::time::timeout(DEFAULT_SWEEP_PING_TIMEOUT, rx).await { + Ok(Ok(Ok(()))) => true, + Ok(Ok(Err(err))) => { + tracing::debug!(error = %err, "sweep ping failed; discarding pooled connection"); + false + } + Ok(Err(_)) => false, // connection task dropped tx + Err(_) => { + tracing::debug!("sweep ping timed out; discarding pooled connection"); + false + } + } + } +} + +/// Snapshot of the pool's current state. +#[derive(Clone, Copy, Debug)] +pub struct PoolStatus { + /// The maximum number of connections the pool will manage. + pub max_size: usize, + + /// The current number of connections (both in-use and idle). + pub size: usize, + + /// The number of idle connections ready to be checked out. + pub available: usize, + + /// The number of tasks waiting for a connection to become available. + pub waiting: usize, +} diff --git a/packages/toasty/src/db/tx.rs b/packages/toasty/src/db/tx.rs new file mode 100644 index 00000000000..724c194b922 --- /dev/null +++ b/packages/toasty/src/db/tx.rs @@ -0,0 +1,284 @@ +use std::marker::PhantomData; +use std::ops::{Deref, DerefMut}; +use std::sync::Arc; + +use crate::{Result, db::ConnectionOperation, db::Executor}; + +use async_trait::async_trait; +use toasty_core::{ + Schema, + driver::{ + Capability, + operation::{self, IsolationLevel, RawSql, TransactionMode}, + }, +}; +use tokio::sync::oneshot; + +/// Either a `&mut Db` or `&mut Connection`, used by [`TransactionBuilder`] to +/// defer connection acquisition until [`begin`](TransactionBuilder::begin). +pub(crate) enum TxSource<'a> { + Db(&'a mut super::Db), + Connection(&'a mut super::Connection), +} + +/// Builder for configuring a transaction before starting it. +/// +/// Obtain one via [`Db::transaction_builder`](super::Db::transaction_builder) +/// or [`Connection::transaction_builder`](super::Connection::transaction_builder), +/// configure isolation level, read-only settings, and lock-acquisition +/// mode, then call [`begin`](Self::begin) to start the transaction. +pub struct TransactionBuilder<'a> { + source: TxSource<'a>, + isolation: Option, + read_only: bool, + mode: TransactionMode, +} + +impl<'a> TransactionBuilder<'a> { + pub(crate) fn new(source: TxSource<'a>) -> Self { + TransactionBuilder { + source, + isolation: None, + read_only: false, + mode: TransactionMode::Default, + } + } + + /// Set the isolation level for this transaction. + pub fn isolation(mut self, level: IsolationLevel) -> Self { + self.isolation = Some(level); + self + } + + /// Set whether this transaction is read-only. + pub fn read_only(mut self, read_only: bool) -> Self { + self.read_only = read_only; + self + } + + /// Set the lock-acquisition mode for this transaction. See + /// [`TransactionMode`] for the supported modes and which drivers + /// implement them. + pub fn mode(mut self, mode: TransactionMode) -> Self { + self.mode = mode; + self + } + + /// Begin the transaction. + /// + /// When built from a [`Db`](super::Db), this acquires a connection from + /// the pool. The connection is owned by the returned [`Transaction`] and + /// will be returned to the pool when the transaction is dropped. + pub async fn begin(self) -> Result> { + let conn = match self.source { + TxSource::Db(db) => ConnRef::owned(db.connection().await?), + TxSource::Connection(conn) => ConnRef::Borrowed(conn), + }; + Transaction::begin_with(conn, self.isolation, self.read_only, self.mode).await + } +} + +/// An active database transaction. +/// +/// All operations executed through a `Transaction` are guaranteed to use the +/// same physical connection. +/// +/// If dropped without calling [`commit`](Self::commit) or +/// [`rollback`](Self::rollback), the transaction is automatically rolled back. +pub struct Transaction<'a> { + /// The connection this transaction operates on. + conn: ConnRef<'a>, + + /// Whether commit or rollback has been called. + finalized: bool, + + /// If this is a nested transaction (implemented through savepoints), + /// this holds the savepoint stack depth to be used as an identifier. + savepoint: Option, +} + +/// Either a borrowed or owned reference to a [`Connection`](super::Connection). +pub(crate) enum ConnRef<'a> { + Borrowed(&'a mut super::Connection), + Owned(super::Connection, PhantomData<&'a ()>), +} + +impl<'a> ConnRef<'a> { + pub(crate) fn owned(conn: super::Connection) -> ConnRef<'a> { + ConnRef::Owned(conn, PhantomData) + } +} + +impl Deref for ConnRef<'_> { + type Target = super::Connection; + + fn deref(&self) -> &Self::Target { + match self { + ConnRef::Borrowed(c) => c, + ConnRef::Owned(c, _) => c, + } + } +} + +impl DerefMut for ConnRef<'_> { + fn deref_mut(&mut self) -> &mut Self::Target { + match self { + ConnRef::Borrowed(c) => c, + ConnRef::Owned(c, _) => c, + } + } +} + +impl<'a> Transaction<'a> { + pub(crate) async fn begin(conn: ConnRef<'a>) -> Result> { + Self::begin_with(conn, None, false, TransactionMode::Default).await + } + + pub(crate) async fn begin_with( + conn: ConnRef<'a>, + isolation: Option, + read_only: bool, + mode: TransactionMode, + ) -> Result> { + tracing::debug!( + isolation = ?isolation, + read_only = read_only, + mode = ?mode, + "beginning transaction" + ); + + // We're creating the Transaction struct before actually starting the transaction. If the + // future is cancelled while waiting on the response of the start command, the transaction + // is still rolled back. + let tx = Transaction { + conn, + finalized: false, + savepoint: None, + }; + + tx.conn + .exec_operation( + operation::Transaction::Start { + isolation, + read_only, + mode, + } + .into(), + ) + .await?; + Ok(tx) + } + + /// Create a nested transaction (savepoint). + /// + /// Takes `&mut self` so the outer transaction is exclusively borrowed + /// while the nested one is open. This prevents statements from running + /// against the outer transaction — bypassing the savepoint — when they + /// should have gone through the nested handle. + pub async fn transaction(&mut self) -> Result> { + ::transaction(self).await + } + + /// Commit the transaction. + pub async fn commit(mut self) -> Result<()> { + tracing::debug!("committing transaction"); + match self.savepoint { + Some(_) => self + .conn + .exec_operation(operation::Transaction::ReleaseSavepoint(self.savepoint()).into()), + None => self + .conn + .exec_operation(operation::Transaction::Commit.into()), + } + .await?; + self.finalized = true; + Ok(()) + } + + /// Roll back the transaction. + pub async fn rollback(mut self) -> Result<()> { + tracing::debug!("rolling back transaction"); + match self.savepoint { + Some(_) => self.conn.exec_operation( + operation::Transaction::RollbackToSavepoint(self.savepoint()).into(), + ), + None => self + .conn + .exec_operation(operation::Transaction::Rollback.into()), + } + .await?; + self.finalized = true; + Ok(()) + } + + fn savepoint(&self) -> String { + format!("tx_{}", self.savepoint.unwrap()) + } +} + +impl Drop for Transaction<'_> { + fn drop(&mut self) { + if !self.finalized { + let op = match self.savepoint { + Some(_) => operation::Transaction::RollbackToSavepoint(self.savepoint()), + None => operation::Transaction::Rollback, + }; + + // Fire-and-forget rollback: send the operation to the background + // connection task without awaiting the response. + let (tx, _rx) = oneshot::channel(); + let _ = self + .conn + .handle() + .in_tx + .send(ConnectionOperation::ExecOperation { + operation: Box::new(op.into()), + span: tracing::Span::current(), + tx, + }); + } + } +} + +#[async_trait] +impl<'a> Executor for Transaction<'a> { + async fn transaction(&mut self) -> Result> { + let depth = match self.savepoint { + Some(savepoint) => savepoint + 1, + None => 1, + }; + tracing::debug!(depth = depth, "creating nested transaction (savepoint)"); + + let transaction = Transaction { + conn: ConnRef::Borrowed(&mut self.conn), + finalized: false, + savepoint: Some(depth), + }; + + transaction + .conn + .exec_operation(operation::Transaction::Savepoint(transaction.savepoint()).into()) + .await?; + + Ok(transaction) + } + + async fn exec_untyped( + &mut self, + stmt: toasty_core::stmt::Statement, + ) -> Result { + self.conn.exec_stmt(stmt, true).await + } + + async fn exec_raw_sql(&mut self, raw: RawSql) -> Result { + self.conn.exec_raw_sql(raw).await + } + + fn capability(&mut self) -> &Capability { + self.conn.shared.engine.capability() + } + + fn schema(&mut self) -> &Arc { + self.conn.schema() + } +} diff --git a/packages/toasty/src/engine.rs b/packages/toasty/src/engine.rs new file mode 100644 index 00000000000..25663206dce --- /dev/null +++ b/packages/toasty/src/engine.rs @@ -0,0 +1,127 @@ +// Statement effect classification. Used by the planner's mutation +// classification (`mir::Operation::is_effectful`); the pool retry wrapper +// consumer lands in a follow-up PR per +// `docs/dev/design/retry-safe-recovery.md`. +pub(crate) mod effect; +pub(crate) mod eval; +pub(crate) mod exec; + +mod bind; +#[cfg(test)] +pub(crate) mod test_util; + +mod fold; +mod hir; +use hir::HirStatement; + +mod index; +mod legalize; +mod lower; +mod mir; +mod normalize; +mod plan; +mod select_item; +pub(crate) use select_item::{SelectItem, SelectItems}; +mod simplify; +mod ty; +mod upsert; +mod verify; + +use crate::Result; +use std::sync::Arc; +use toasty_core::{ + Connection, Schema, + driver::{Capability, operation::RawSql}, + stmt::{self, Statement}, +}; + +/// The query execution engine. +/// +/// [`Engine`] orchestrates the multi-phase compilation pipeline that transforms +/// user queries into database operations. It owns the schema and capability +/// reference, and provides the main entry point ([`exec`](Self::exec)) for +/// executing statements. +/// +/// The execution pipeline follows this process: +/// +/// 1. **Normalization.** Expand implicit application-level semantics. +/// 2. **Verification.** Validate statement structure and reject AST shapes +/// the driver does not support. +/// 3. **Lowering.** Convert to HIR with dependency tracking. +/// 4. **Planning.** Build MIR operation graph. +/// 5. **Execution.** Run actions against the database driver. Each +/// driver-bound statement is legalized for the target backend and its +/// bind parameters extracted ([`prepare_for_driver`](Self::prepare_for_driver)) +/// immediately before it crosses to the driver. +#[derive(Debug, Clone)] +pub(crate) struct Engine { + /// The schema being managed by this database instance. + pub(crate) schema: Arc, + + /// Driver capabilities, used during planning. + pub(crate) capability: &'static Capability, +} + +impl Engine { + /// Creates a new [`Engine`] with the given schema and capability. + pub(crate) fn new(schema: Arc, capability: &'static Capability) -> Engine { + Engine { schema, capability } + } + + /// Returns the driver's capabilities. + pub(crate) fn capability(&self) -> &Capability { + self.capability + } + + /// Executes a statement and returns the full response including pagination metadata. + /// + /// The statement passes through the full compilation pipeline + /// (lowering -> planning -> execution) before being sent to the database + /// driver via the provided connection. + pub(crate) async fn exec( + &self, + connection: &mut dyn Connection, + mut stmt: Statement, + in_transaction: bool, + ) -> Result { + self.normalize_stmt(&mut stmt)?; + self.verify(&stmt)?; + + // Lower the statement to High-level intermediate representation + let hir = self.lower_stmt(stmt)?; + + // Translate the optimized statement into a series of driver operations. + let plan = self.plan_hir_statement(hir)?; + + tracing::trace!( + steps = plan.steps.len(), + needs_transaction = plan.needs_transaction, + "execution plan ready" + ); + + // The plan is called once (single entry record stream) with no arguments + // (empty record). + self.exec_plan(connection, plan, in_transaction).await + } + + /// Executes user-authored SQL through the driver SQL path. + pub(crate) async fn exec_raw_sql( + &self, + connection: &mut dyn Connection, + raw: RawSql, + ) -> Result { + if !self.capability.sql() { + return Err(toasty_core::Error::unsupported_feature(format!( + "{} does not support raw SQL", + self.capability.driver_name + ))); + } + + connection.exec(&self.schema, raw.into()).await + } + + /// Returns a new [`ExprContext`](stmt::ExprContext) for a specific target. + fn expr_cx_for<'a>(&'a self, target: impl stmt::IntoExprTarget<'a>) -> stmt::ExprContext<'a> { + stmt::ExprContext::new_with_target(&self.schema, target) + } +} diff --git a/packages/toasty/src/engine/bind.rs b/packages/toasty/src/engine/bind.rs new file mode 100644 index 00000000000..22d2a24998a --- /dev/null +++ b/packages/toasty/src/engine/bind.rs @@ -0,0 +1,126 @@ +//! Bind a fully-resolved statement's inline values into typed parameters. +//! +//! Three phases: +//! 1. **Extract** ([`extract`]): replace scalar `Value` nodes with `Arg(n)` +//! placeholders, initializing each param's type from the value itself. +//! 2. **Synthesize** (bottom-up): compute each node's inferred type from its +//! children (column refs get their storage type from the schema, records get +//! a tuple of field types, etc.). +//! 3. **Check** (top-down): push refined types into `Arg(n)` nodes, upgrading a +//! param when context is more precise (e.g. `Enum` over `Text`). +//! +//! Synthesize and check ([`infer`]) run together in a single recursive walk. +//! Types carry **provenance** (`Column` vs `Inferred`) so schema-authoritative +//! column types win over value-inferred guesses when merging. +//! +//! `#[document]` values are named into their `Value::Object` form before this +//! runs: the mapping's lowering casts convert them during statement +//! lowering/simplification, and document *paths* are resolved by legalization +//! ([`super::legalize`]) — `Engine::prepare_for_driver` runs both in order. + +use toasty_core::{ + driver::{Capability, operation::TypedValue}, + schema::db, + stmt, +}; + +mod extract; +mod infer; + +#[cfg(test)] +mod tests; + +/// Bind a statement's inline values: replace scalar `Value` nodes with +/// `Expr::Arg(n)` placeholders and infer a precise `db::Type` for each. The +/// returned `Vec` is indexed by the `n` in each placeholder. +/// +/// The statement must already be legalized ([`super::legalize`]): document +/// values named, document paths resolved. Runs via +/// `Engine::prepare_for_driver`, as the final engine step before a driver +/// serializes the statement. +pub(crate) fn run( + stmt: &mut stmt::Statement, + db_schema: &db::Schema, + capability: &Capability, +) -> Vec { + // Phase 1: mechanical extraction — replace values with Arg(n). + let mut params: Vec = Vec::new(); + extract::extract_values(stmt, &mut params, capability); + + // Phases 2+3: bidirectional type inference — refine param types. + infer::refine_param_types(stmt, db_schema, &mut params); + + // Materialize the final TypedValues. `finalize_ty` panics if any param is + // still unresolved — synthesize/check is expected to type every param. + params + .into_iter() + .map(|p| { + let Param { value, ty } = p; + TypedValue { + ty: finalize_ty(&value, ty), + value, + } + }) + .collect() +} + +/// A bind parameter being inferred. Once inference completes, the `Ty` is +/// converted to a concrete `db::Type` for the `TypedValue`. +struct Param { + value: stmt::Value, + ty: Ty, +} + +/// Resolve a `Ty` to a concrete `db::Type`. Panics on `Unknown` / `Record` — +/// every param should be fully inferred by the synthesize/check pass; if a +/// statement reaches here with an unresolved param, that's a bug worth +/// surfacing so we can evaluate the specific case. +fn finalize_ty(value: &stmt::Value, ty: Ty) -> db::Type { + match ty { + Ty::Column(t) | Ty::Inferred(t) => t, + Ty::List(elem) => db::Type::list(finalize_ty(value, *elem)), + Ty::Unknown => panic!("bind left {value:?} with unresolved type"), + Ty::Record(_) => panic!( + "bind left {value:?} typed as a record; only scalars and lists are extracted as params" + ), + } +} + +// ============================================================================ +// Inferred type representation +// ============================================================================ + +/// The inferred database-level type of an expression node. +/// +/// Each scalar type carries **provenance**: `Column` means the type came from +/// the schema (authoritative), `Inferred` means it was guessed from the value. +/// Column types always win when merging. +#[derive(Debug, Clone)] +enum Ty { + /// Type from a column reference or schema (authoritative). + Column(db::Type), + /// Type inferred from a value (initial guess — may be less specific). + Inferred(db::Type), + /// A tuple of types (one per field). + Record(Vec), + /// A homogeneous list where all elements share a type. + List(Box), + /// Type could not be determined. + Unknown, +} + +impl Ty { + /// Extract the `db::Type`, regardless of provenance. + fn db_type(&self) -> Option<&db::Type> { + match self { + Ty::Column(ty) | Ty::Inferred(ty) => Some(ty), + _ => None, + } + } + + /// Returns true if this type comes from the schema (authoritative). + #[cfg(test)] + fn is_column(&self) -> bool { + matches!(self, Ty::Column(_)) + } +} diff --git a/packages/toasty/src/engine/bind/extract.rs b/packages/toasty/src/engine/bind/extract.rs new file mode 100644 index 00000000000..c8df602cf9e --- /dev/null +++ b/packages/toasty/src/engine/bind/extract.rs @@ -0,0 +1,385 @@ +//! Phase 1: mechanical value extraction — replace scalar `Value` nodes with +//! `Arg(n)` placeholders, initializing each param's type from the value itself. + +use toasty_core::{ + driver::Capability, + schema::db, + stmt::{self, VisitMut}, +}; + +use super::{Param, Ty}; + +/// Replace all scalar `Value` nodes with `Arg(n)` placeholders, initializing +/// each param's `ty` from the value itself. +pub(super) fn extract_values( + stmt: &mut stmt::Statement, + params: &mut Vec, + capability: &Capability, +) { + struct Extract<'a> { + params: &'a mut Vec, + bind_list_param: bool, + glob_starts_with: bool, + binary_like_starts_with: bool, + } + + impl stmt::VisitMut for Extract<'_> { + fn visit_expr_mut(&mut self, expr: &mut stmt::Expr) { + // Intercept ANY/ALL: bind their array operand as one Value::List + // param rather than visiting the rhs and extracting each element + // separately. The element type is refined to the column type by + // the synthesize/check pass. + match expr { + stmt::Expr::AnyOp(e) => { + self.visit_expr_mut(&mut e.lhs); + if let Some(arg) = extract_array_operand(&mut e.rhs, self.params) { + *e.rhs = arg; + } else { + self.visit_expr_mut(&mut e.rhs); + } + return; + } + stmt::Expr::AllOp(e) => { + self.visit_expr_mut(&mut e.lhs); + if let Some(arg) = extract_array_operand(&mut e.rhs, self.params) { + *e.rhs = arg; + } else { + self.visit_expr_mut(&mut e.rhs); + } + return; + } + // `IN (...)` always renders as N separate placeholders on + // backends without `predicate_match_any` (the ones that support + // ANY were rewritten to it during lowering). Force per-element + // expansion of the rhs list regardless of `bind_list_param`, so + // `id IN (1, 2, 3)` stays N placeholders rather than collapsing + // to `id IN ?` on backends that bind a `Vec` column as + // one param. + stmt::Expr::InList(e) => { + self.visit_expr_mut(&mut e.expr); + if let stmt::Expr::Value(stmt::Value::List(_)) = e.list.as_ref() { + let stmt::Expr::Value(stmt::Value::List(items)) = + std::mem::replace(e.list.as_mut(), stmt::Expr::null()) + else { + unreachable!() + }; + let items = items + .into_iter() + .map(|v| value_to_extracted_expr(v, self.params, false)) + .collect(); + *e.list = stmt::Expr::List(stmt::ExprList { items }); + } else { + self.visit_expr_mut(&mut e.list); + } + return; + } + // For SQLite/MySQL, transform the prefix into the final search + // pattern before binding it: GLOB needs `*`/`?`/`[` escaped and + // a `*` appended; BINARY LIKE needs `%`/`_`/`!` escaped and a + // `%` appended. The column expression is visited normally. + stmt::Expr::StartsWith(e) + if self.glob_starts_with || self.binary_like_starts_with => + { + self.visit_expr_mut(&mut e.expr); + let stmt::Expr::Value(stmt::Value::String(prefix)) = e.prefix.as_ref() else { + panic!("starts_with prefix must be a string literal"); + }; + let pattern = if self.glob_starts_with { + glob_prefix_pattern(prefix) + } else { + binary_like_prefix_pattern(prefix) + }; + let position = self.params.len(); + self.params.push(Param { + value: stmt::Value::String(pattern), + ty: Ty::Inferred(db::Type::Text), + }); + *e.prefix = stmt::Expr::arg(position); + return; + } + _ => {} + } + + // On backends that bind arrays as a single protocol parameter + // (PostgreSQL, see `Capability::bind_list_param`), a literal + // list of scalar values is the value of a `Vec` model + // field — extract as one `Value::List` arg so it round-trips + // through the driver as a `text[]` / `int8[]` bind. Without + // this, recursion would expand the list to one arg per item + // and render it as a SQL record literal. + if self.bind_list_param + && is_scalar_list(expr) + && let Some(arg) = extract_array_operand(expr, self.params) + { + *expr = arg; + return; + } + + // Default post-order: recurse first, then maybe extract this node. + stmt::visit_mut::visit_expr_mut(self, expr); + + match expr { + stmt::Expr::Value(value) if is_extractable_scalar(value) => { + let ty = infer_ty(value); + let position = self.params.len(); + let value = std::mem::replace(value, stmt::Value::Null); + self.params.push(Param { value, ty }); + *expr = stmt::Expr::arg(position); + } + // A bare `#[document]` embed value (named by the mapping's + // lowering cast) binds as one param with an unknown type; the + // synthesize/check pass resolves it to the document column + // type. + stmt::Expr::Value(value @ stmt::Value::Object(_)) => { + let owned = std::mem::replace(value, stmt::Value::Null); + let position = self.params.len(); + self.params.push(Param { + value: owned, + ty: Ty::Unknown, + }); + *expr = stmt::Expr::arg(position); + } + stmt::Expr::Value(value @ (stmt::Value::Record(_) | stmt::Value::List(_))) => { + let owned = std::mem::replace(value, stmt::Value::Null); + *expr = value_to_extracted_expr(owned, self.params, self.bind_list_param); + } + _ => {} + } + } + } + + Extract { + params, + bind_list_param: capability.bind_list_param, + glob_starts_with: capability.glob_starts_with, + binary_like_starts_with: capability.binary_like_starts_with, + } + .visit_mut(stmt); +} + +/// Whether `expr` is an `Expr::Value` carrying an extractable scalar. +fn is_extractable_scalar_expr(expr: &stmt::Expr) -> bool { + matches!(expr, stmt::Expr::Value(v) if is_extractable_scalar(v)) +} + +/// Whether `expr` is a literal list of scalar values — either an +/// `Expr::List` of `Expr::Value(...)` items, or an already-collapsed +/// `Expr::Value(Value::List(...))`. The canonicalizer (`fold::expr_list`) +/// produces the latter shape, but lowering can still emit the former, so +/// we cover both. +fn is_scalar_list(expr: &stmt::Expr) -> bool { + match expr { + stmt::Expr::List(list) => list.items.iter().all(is_extractable_scalar_expr), + stmt::Expr::Value(stmt::Value::List(items)) => items.iter().all(is_extractable_scalar), + _ => false, + } +} + +/// If `expr` is a list literal of values, take it out, push one +/// `Param { value: Value::List(items), ty: Ty::List() }` onto `params`, +/// and return an `Expr::Arg(n)` to put back in its place. Used for both the +/// `ANY/ALL` rhs operand and `Vec` field literals on backends that +/// bind arrays as a single protocol parameter. +/// +/// The element type starts as the value-inferred type of the first non-null +/// item — or `Ty::Unknown` for empty / all-null lists. The synthesize/check +/// pass refines it to the column type when one is known. +fn extract_array_operand(expr: &mut stmt::Expr, params: &mut Vec) -> Option { + let items: Vec = match expr { + stmt::Expr::Value(stmt::Value::List(_)) => { + let stmt::Expr::Value(stmt::Value::List(items)) = + std::mem::replace(expr, stmt::Expr::null()) + else { + unreachable!() + }; + items + } + stmt::Expr::List(list) if list.items.iter().all(|i| matches!(i, stmt::Expr::Value(_))) => { + let stmt::Expr::List(list) = std::mem::replace(expr, stmt::Expr::null()) else { + unreachable!() + }; + list.items + .into_iter() + .map(|e| match e { + stmt::Expr::Value(v) => v, + _ => unreachable!(), + }) + .collect() + } + _ => return None, + }; + + let value = stmt::Value::List(items); + let ty = infer_ty(&value); + + let position = params.len(); + params.push(Param { value, ty }); + Some(stmt::Expr::arg(position)) +} + +/// Recursively convert a `Value` into an `Expr`, extracting scalar values. +/// Takes ownership to avoid cloning. +/// +/// On backends that bind arrays as a single protocol parameter (`bind_list_param`), +/// a `Value::List` of all extractable scalars is captured as a single param of +/// `Value::List` shape so it round-trips through the driver as one array bind. +/// Other lists fall through to per-element expansion to preserve the existing +/// record/tuple semantics on backends without native array binds. +fn value_to_extracted_expr( + value: stmt::Value, + params: &mut Vec, + bind_list_param: bool, +) -> stmt::Expr { + match value { + stmt::Value::Null => stmt::Expr::Value(stmt::Value::Null), + stmt::Value::Record(record) => { + let fields = record + .fields + .into_iter() + .map(|f| value_to_extracted_expr(f, params, bind_list_param)) + .collect(); + stmt::Expr::Record(stmt::ExprRecord::from_vec(fields)) + } + stmt::Value::List(values) + if bind_list_param + && values.iter().all(|v| { + // A `Vec` collection, or a `#[document]` collection + // of embedded structs (named `Value::Object`s by the + // mapping's lowering cast). Either way the whole list binds + // as one parameter; the synthesize/check pass resolves its + // type. + is_extractable_scalar(v) || matches!(v, stmt::Value::Object(_)) + }) => + { + let value = stmt::Value::List(values); + let ty = infer_ty(&value); + let position = params.len(); + params.push(Param { value, ty }); + stmt::Expr::arg(position) + } + stmt::Value::List(values) => { + let items = values + .into_iter() + .map(|v| value_to_extracted_expr(v, params, bind_list_param)) + .collect(); + stmt::Expr::List(stmt::ExprList { items }) + } + scalar => { + let ty = infer_ty(&scalar); + let position = params.len(); + params.push(Param { value: scalar, ty }); + stmt::Expr::arg(position) + } + } +} + +fn is_extractable_scalar(value: &stmt::Value) -> bool { + // A `Value::Object` is a named document blob, not a scalar: it binds as one + // param via the dedicated `Expr::Value(Object)` arm / `value_to_extracted_expr`, + // never through scalar extraction. + !matches!( + value, + stmt::Value::Null | stmt::Value::Record(_) | stmt::Value::List(_) | stmt::Value::Object(_) + ) +} + +/// Initial type guess for a value, used as the starting point for inference. +/// +/// Returns the most precise `Ty` derivable from the value alone: +/// - Scalars become `Ty::Inferred()`. +/// - Lists become `Ty::List()`, recursing into the first non-null item. +/// Empty / all-null lists yield `Ty::List(Ty::Unknown)`; the element type is +/// refined by synthesize/check when a column context is available. +/// - Anything we can't classify (`Null`, `Record`, `F32`/`F64`, `Zoned`, +/// `BigDecimal`, `SparseRecord`) becomes `Ty::Unknown`. +fn infer_ty(value: &stmt::Value) -> Ty { + use stmt::Value; + match value { + Value::Bool(_) => Ty::Inferred(db::Type::Boolean), + Value::I8(_) => Ty::Inferred(db::Type::Integer(1)), + Value::I16(_) => Ty::Inferred(db::Type::Integer(2)), + Value::I32(_) => Ty::Inferred(db::Type::Integer(4)), + Value::I64(_) => Ty::Inferred(db::Type::Integer(8)), + Value::U8(_) => Ty::Inferred(db::Type::UnsignedInteger(1)), + Value::U16(_) => Ty::Inferred(db::Type::UnsignedInteger(2)), + Value::U32(_) => Ty::Inferred(db::Type::UnsignedInteger(4)), + Value::U64(_) => Ty::Inferred(db::Type::UnsignedInteger(8)), + Value::String(_) => Ty::Inferred(db::Type::Text), + Value::Uuid(_) => Ty::Inferred(db::Type::Uuid), + Value::Bytes(_) => Ty::Inferred(db::Type::Blob), + #[cfg(feature = "rust_decimal")] + Value::Decimal(_) => Ty::Inferred(db::Type::Numeric(None)), + #[cfg(feature = "jiff")] + Value::Timestamp(_) => Ty::Inferred(db::Type::Timestamp(6)), + #[cfg(feature = "jiff")] + Value::Date(_) => Ty::Inferred(db::Type::Date), + #[cfg(feature = "jiff")] + Value::Time(_) => Ty::Inferred(db::Type::Time(6)), + #[cfg(feature = "jiff")] + Value::DateTime(_) => Ty::Inferred(db::Type::DateTime(6)), + #[cfg(feature = "net")] + Value::Cidr(_) => Ty::Inferred(db::Type::Cidr), + #[cfg(feature = "net")] + Value::Inet(_) => Ty::Inferred(db::Type::Inet), + #[cfg(feature = "net")] + Value::MacAddr(_) => Ty::Inferred(db::Type::MacAddr), + #[cfg(feature = "net")] + Value::MacAddr8(_) => Ty::Inferred(db::Type::MacAddr8), + Value::List(items) => { + let elem = items + .iter() + .find(|v| !v.is_null()) + .map(infer_ty) + .unwrap_or(Ty::Unknown); + Ty::List(Box::new(elem)) + } + _ => Ty::Unknown, + } +} + +// ============================================================================ +// `starts_with` pattern builders (used when a backend lowers it to GLOB / LIKE) +// ============================================================================ + +/// Build a SQLite GLOB pattern for a `starts_with(prefix)` expression. +/// +/// GLOB metacharacters (`*`, `?`, `[`) are escaped by wrapping them in a +/// bracket class: `*` → `[*]`, `?` → `[?]`, `[` → `[[]`. A trailing `*` +/// wildcard is appended so the pattern matches any string starting with +/// `prefix`. GLOB has no ESCAPE clause, so bracket-class escaping is the only +/// available mechanism. +pub(super) fn glob_prefix_pattern(prefix: &str) -> String { + // Each metachar expands to 3 chars; over-allocate slightly rather than + // under-allocate and trigger a realloc on prefixes with wildcards. + let mut pattern = String::with_capacity(prefix.len() * 3 + 1); + for c in prefix.chars() { + match c { + '*' => pattern.push_str("[*]"), + '?' => pattern.push_str("[?]"), + '[' => pattern.push_str("[[]"), + c => pattern.push(c), + } + } + pattern.push('*'); + pattern +} + +/// Build a MySQL `BINARY col LIKE ? ESCAPE '!'` pattern for `starts_with(prefix)`. +/// +/// `!` is the hardcoded escape character. In a single pass, `!`, `%`, and `_` +/// are all prefixed with `!` (so `!` → `!!`, `%` → `!%`, `_` → `!_`). A +/// trailing `%` wildcard is appended. +pub(super) fn binary_like_prefix_pattern(prefix: &str) -> String { + let mut pattern = String::with_capacity(prefix.len() + 1); + for c in prefix.chars() { + match c { + '!' | '%' | '_' => { + pattern.push('!'); + pattern.push(c); + } + c => pattern.push(c), + } + } + pattern.push('%'); + pattern +} diff --git a/packages/toasty/src/engine/bind/infer.rs b/packages/toasty/src/engine/bind/infer.rs new file mode 100644 index 00000000000..f16313ae9f0 --- /dev/null +++ b/packages/toasty/src/engine/bind/infer.rs @@ -0,0 +1,485 @@ +//! Phases 2 & 3: bidirectional type inference. A single recursive walk +//! synthesizes each node's type bottom-up and checks refined types back down +//! into `Arg` params, so each param ends up with a precise `db::Type`. + +use toasty_core::{schema::db, stmt}; + +use super::{Param, Ty}; + +/// Expression context bound to the database schema. +type Cx<'a> = stmt::ExprContext<'a, db::Schema>; + +/// Refine param types by walking the statement with synthesize + check. +pub(super) fn refine_param_types( + stmt: &stmt::Statement, + db_schema: &db::Schema, + params: &mut [Param], +) { + let cx = stmt::ExprContext::new(db_schema); + refine_stmt(stmt, &cx, db_schema, params); +} + +fn refine_stmt(stmt: &stmt::Statement, cx: &Cx<'_>, db_schema: &db::Schema, params: &mut [Param]) { + match stmt { + stmt::Statement::Insert(insert) => { + let cx = cx.scope(insert); + refine_insert(insert, &cx, db_schema, params); + } + stmt::Statement::Update(update) => { + let cx = cx.scope(update); + refine_update(update, &cx, db_schema, params); + } + stmt::Statement::Delete(delete) => { + let cx = cx.scope(delete); + refine_filter(&delete.filter, &cx, params); + } + stmt::Statement::Query(query) => { + refine_query(query, cx, params); + } + } +} + +/// Lift a column's `db::Type` into the inferred-type form. List columns +/// expand to `Ty::List(Ty::Column(elem))` so they unify with values whose +/// inferred shape is also `Ty::List(_)`; everything else stays as a flat +/// `Ty::Column(_)`. +/// +/// # Why this shape instead of widening `Ty::Column` to hold a `db::Type::List`? +/// +/// `Ty` exists to carry *provenance* alongside the inferred type: +/// `Ty::Column(_)` is authoritative (from the schema), `Ty::Inferred(_)` is a +/// guess from a value. Merging the two is what propagates the column type +/// down into argument placeholders. +/// +/// A list arg comes in as `Ty::List(Ty::Inferred(elem))` because the +/// element type is guessed from the first non-null value. When the schema +/// knows the column type, we need to merge the column-provenance element type +/// *into* the list. That requires the two sides to agree on shape — +/// `Ty::List(_)` vs `Ty::List(_)` — and merge element-wise via the list branch +/// in [`merge`]. +/// +/// The alternative of carrying `Ty::Column(db::Type::List(_))` would put a +/// list inside a "scalar" variant; merging it against `Ty::List(Inferred(_))` +/// from a value would either require a special case or lose the element-level +/// provenance the synthesize/check pass relies on. Expanding into +/// `Ty::List(Ty::Column(_))` keeps the data structure uniform — every list +/// is `Ty::List`, every scalar is `Ty::Column`/`Ty::Inferred` — and lets +/// `merge` handle the cases with no extra branches. +fn ty_from_column(storage_ty: db::Type) -> Ty { + match storage_ty { + db::Type::List(elem) => Ty::List(Box::new(ty_from_column(*elem))), + scalar => Ty::Column(scalar), + } +} + +fn refine_insert( + insert: &stmt::Insert, + _cx: &Cx<'_>, + db_schema: &db::Schema, + params: &mut [Param], +) { + let stmt::InsertTarget::Table(table) = &insert.target else { + return; + }; + let db_table = &db_schema.tables[table.table.0]; + + // Expected type from the column list (authoritative). `check` pushes it + // down into each VALUES row, field by field, typing every `Arg` param — + // including a `#[document]` column, whose list/object value resolves to its + // scalar `Document` storage type in `merge`. + let expected = Ty::Record( + table + .columns + .iter() + .map(|col_id| ty_from_column(db_table.columns[col_id.index].storage_ty.clone())) + .collect(), + ); + + if let stmt::ExprSet::Values(values) = &insert.source.body { + for row in &values.rows { + check(row, &expected, params); + } + } + + if let Some(upsert) = &insert.upsert { + refine_assignments(&upsert.shared, db_table, params); + refine_assignments(&upsert.defaults, db_table, params); + refine_assignments(&upsert.update_defaults, db_table, params); + } +} + +fn refine_update(update: &stmt::Update, cx: &Cx<'_>, db_schema: &db::Schema, params: &mut [Param]) { + // Refine assignment types from target columns + if let stmt::UpdateTarget::Table(table_id) = &update.target { + let db_table = &db_schema.tables[table_id.0]; + + refine_assignments(&update.assignments, db_table, params); + } + + // Refine filter types + refine_filter(&update.filter, cx, params); +} + +fn refine_assignments(assignments: &stmt::Assignments, db_table: &db::Table, params: &mut [Param]) { + for (projection, assignment) in assignments.iter() { + let steps = projection.as_slice(); + assert_eq!( + steps.len(), + 1, + "UPDATE assignment projection should be a single column index, got {steps:?}" + ); + let col_idx = steps[0]; + let Some(col) = db_table.columns.get(col_idx) else { + continue; + }; + + match assignment { + stmt::Assignment::Set(expr) | stmt::Assignment::Append(expr) => { + // The expression takes the column's full type (the whole + // column for `Set`, the elements for `Append`). A + // `#[document]` column's `Value::List`/`Value::Object` param + // collapses to its scalar `Document` storage type via `merge`. + let expected = ty_from_column(col.storage_ty.clone()); + check(expr, &expected, params); + } + // `Remove` is `array_remove(col, $1)`-shaped: the rhs binds + // as the column's element type, not the list type. Pull the + // element out of the list column type so the param is bound + // correctly. + stmt::Assignment::Remove(expr) => { + if let db::Type::List(elem) = &col.storage_ty { + let expected = ty_from_column((**elem).clone()); + check(expr, &expected, params); + } + } + // `RemoveAt` binds an integer index, not a column value: + // the column's element type is unrelated to the index's + // type. Skip the column-driven `check` — the value-side + // inference from `infer_ty` (e.g. + // `Ty::Inferred(UnsignedInteger(8))` for a `usize` + // converted to `Value::U64`) is enough to bind the param. + stmt::Assignment::RemoveAt(_) | stmt::Assignment::Pop => {} + // `Add` / `Subtract` bind a scalar of the column's type + // (`col = col + $1`). + stmt::Assignment::Add(expr) | stmt::Assignment::Subtract(expr) => { + let expected = ty_from_column(col.storage_ty.clone()); + check(expr, &expected, params); + } + stmt::Assignment::Insert(_) | stmt::Assignment::Batch(_) => continue, + } + } +} + +fn refine_query(query: &stmt::Query, cx: &Cx<'_>, params: &mut [Param]) { + // One scope per query — matching the `ExprColumn::nesting` model and the + // SQL serializer (which also scopes once per `Query`). `Query`'s target + // resolves through its body to the `Select` source, so this single scope + // is the source scope. Scoping the `Select` again would double-count a + // level, so a column inside a subquery that references an outer column + // (e.g. a JOIN-include's linking column lifted into an `EXISTS`) would + // resolve against the wrong source. + let cx = cx.scope(query); + + match &query.body { + stmt::ExprSet::Select(select) => { + refine_filter(&select.filter, &cx, params); + } + stmt::ExprSet::Values(values) => { + for row in &values.rows { + synthesize(row, &cx, params); + } + } + // Data-modifying CTE bodies (a conditional write compiled to a CTE): + // the write's assignments and filter carry params that need the same + // column-driven refinement as a top-level UPDATE/DELETE. + stmt::ExprSet::Update(update) => { + refine_update(update, &cx, cx.schema(), params); + } + stmt::ExprSet::Delete(delete) => { + refine_filter(&delete.filter, &cx, params); + } + _ => {} + } + + // Handle CTEs + if let Some(with) = &query.with { + for cte in &with.ctes { + refine_query(&cte.query, &cx, params); + } + } +} + +fn refine_filter(filter: &stmt::Filter, cx: &Cx<'_>, params: &mut [Param]) { + if let Some(expr) = &filter.expr { + synthesize(expr, cx, params); + } +} + +// ============================================================================ +// Synthesize (bottom-up) — returns the inferred type with provenance +// ============================================================================ + +/// Compute the inferred type of an expression from its children. +/// +/// For comparison operators, this also triggers `check()` to push refined +/// types down into both sides (bidirectional inference). +pub(super) fn synthesize(expr: &stmt::Expr, cx: &Cx<'_>, params: &mut [Param]) -> Ty { + match expr { + // Arg — type comes from the extracted param (whatever the current + // inference state is — `Inferred(...)` from the value, possibly + // already upgraded to `Column(...)` by a prior `check`). + stmt::Expr::Arg(arg) => params[arg.position].ty.clone(), + + // Column reference — authoritative from schema + stmt::Expr::Reference(expr_ref @ stmt::ExprReference::Column(_)) => { + match cx.resolve_expr_reference(expr_ref) { + stmt::ResolvedRef::Column(col) => ty_from_column(col.storage_ty.clone()), + _ => Ty::Unknown, + } + } + + // Projection — walk each step to reach the projected field's type + stmt::Expr::Project(project) => { + let mut ty = synthesize(&project.base, cx, params); + for &step in project.projection.as_slice() { + ty = match ty { + Ty::Record(fields) => { + assert!( + step < fields.len(), + "projection step {step} out of range for record with {} fields", + fields.len() + ); + fields.into_iter().nth(step).unwrap() + } + other => panic!("cannot project from non-record type: {other:?}"), + }; + } + ty + } + + // Record — synthesize each field + stmt::Expr::Record(record) => { + let fields: Vec = record + .fields + .iter() + .map(|f| synthesize(f, cx, params)) + .collect(); + Ty::Record(fields) + } + + // List — synthesize each item, merge to a common type + stmt::Expr::List(list) => { + let mut merged = Ty::Unknown; + for item in &list.items { + let item_ty = synthesize(item, cx, params); + merged = merge(&merged, &item_ty); + } + Ty::List(Box::new(merged)) + } + + // BinaryOp (comparison) — synthesize both sides, merge, check both + stmt::Expr::BinaryOp(binary) => { + let lhs_ty = synthesize(&binary.lhs, cx, params); + let rhs_ty = synthesize(&binary.rhs, cx, params); + let merged = merge(&lhs_ty, &rhs_ty); + check(&binary.lhs, &merged, params); + check(&binary.rhs, &merged, params); + Ty::Inferred(db::Type::Boolean) + } + + // InList — synthesize expr, check list items against it + stmt::Expr::InList(in_list) => { + let expr_ty = synthesize(&in_list.expr, cx, params); + synthesize(&in_list.list, cx, params); + check_list(&in_list.list, &expr_ty, params); + Ty::Inferred(db::Type::Boolean) + } + + // AnyOp / AllOp — synthesize lhs, then push `List(lhs_ty)` down so + // the rhs Arg's param type lifts to `db::Type::List()` with + // the column-known element type. + stmt::Expr::AnyOp(e) => { + let lhs_ty = synthesize(&e.lhs, cx, params); + check(&e.rhs, &Ty::List(Box::new(lhs_ty)), params); + Ty::Inferred(db::Type::Boolean) + } + stmt::Expr::AllOp(e) => { + let lhs_ty = synthesize(&e.lhs, cx, params); + check(&e.rhs, &Ty::List(Box::new(lhs_ty)), params); + Ty::Inferred(db::Type::Boolean) + } + + // InSubquery — synthesize the expression, recurse into subquery + stmt::Expr::InSubquery(in_sub) => { + synthesize(&in_sub.expr, cx, params); + refine_query(&in_sub.query, cx, params); + Ty::Inferred(db::Type::Boolean) + } + + // Exists — recurse into subquery + stmt::Expr::Exists(exists) => { + refine_query(&exists.subquery, cx, params); + Ty::Inferred(db::Type::Boolean) + } + + // Nested statement + stmt::Expr::Stmt(expr_stmt) => { + refine_stmt(&expr_stmt.stmt, cx, cx.schema(), params); + Ty::Unknown + } + + // Logical operators — recurse, return boolean + stmt::Expr::And(and) => { + for op in &and.operands { + synthesize(op, cx, params); + } + Ty::Inferred(db::Type::Boolean) + } + stmt::Expr::Or(or) => { + for op in &or.operands { + synthesize(op, cx, params); + } + Ty::Inferred(db::Type::Boolean) + } + stmt::Expr::Not(not) => { + synthesize(¬.expr, cx, params); + Ty::Inferred(db::Type::Boolean) + } + stmt::Expr::IsNull(is_null) => { + synthesize(&is_null.expr, cx, params); + Ty::Inferred(db::Type::Boolean) + } + + // StartsWith — both sides are strings. Reaches here only on drivers + // that natively support it (e.g., DynamoDB); SQL drivers lower it to + // Like during the lowering phase. + stmt::Expr::StartsWith(e) => { + check(&e.expr, &Ty::Inferred(db::Type::Text), params); + check(&e.prefix, &Ty::Inferred(db::Type::Text), params); + Ty::Inferred(db::Type::Boolean) + } + + // Like — both sides are strings + stmt::Expr::Like(e) => { + check(&e.expr, &Ty::Inferred(db::Type::Text), params); + check(&e.pattern, &Ty::Inferred(db::Type::Text), params); + Ty::Inferred(db::Type::Boolean) + } + + // Values that weren't extracted (Null, Default) + stmt::Expr::Value(stmt::Value::Null) => Ty::Unknown, + stmt::Expr::Default => Ty::Unknown, + + // Anything else + _ => Ty::Unknown, + } +} + +// ============================================================================ +// Check (top-down) — pushes refined types into Arg nodes +// ============================================================================ + +/// Push an expected type down into an expression. When it reaches `Arg(n)`, +/// merge the expected type into `params[n].ty` so column provenance and +/// concrete element types propagate down (e.g. `List(Unknown) → List(Column(_))`). +fn check(expr: &stmt::Expr, expected: &Ty, params: &mut [Param]) { + match (expr, expected) { + // Arg — merge expected into the param's current type. `merge` handles + // provenance (column wins over inferred) and unknowns (any type wins + // over Unknown), including recursively for list element types. + (stmt::Expr::Arg(arg), ty) => { + let current = params[arg.position].ty.clone(); + params[arg.position].ty = merge(¤t, ty); + } + + // Record — check each field against its expected type + (stmt::Expr::Record(record), Ty::Record(field_types)) => { + for (field, field_ty) in record.fields.iter().zip(field_types) { + check(field, field_ty, params); + } + } + + // List — check each item against the expected element type + (stmt::Expr::List(list), Ty::List(elem_ty)) => { + for item in &list.items { + check(item, elem_ty, params); + } + } + (stmt::Expr::List(list), ty) if ty.db_type().is_some() => { + // Scalar expected for each item (e.g., from InList) + for item in &list.items { + check(item, ty, params); + } + } + + // For other nodes, no downward propagation needed + _ => {} + } +} + +/// Check all items in a list expression against an expected element type. +fn check_list(list_expr: &stmt::Expr, elem_ty: &Ty, params: &mut [Param]) { + match list_expr { + stmt::Expr::List(list) => { + for item in &list.items { + check(item, elem_ty, params); + } + } + _ => { + check(list_expr, elem_ty, params); + } + } +} + +// ============================================================================ +// Merge — combines two types, column provenance wins +// ============================================================================ + +/// Merge two inferred types. Column provenance wins over Inferred. +pub(super) fn merge(a: &Ty, b: &Ty) -> Ty { + match (a, b) { + (Ty::Unknown, other) | (other, Ty::Unknown) => other.clone(), + + // Both are scalars — column provenance wins + (Ty::Column(a_ty), Ty::Column(b_ty)) => { + assert_eq!( + a_ty, b_ty, + "two column types in the same expression disagree: {a_ty:?} vs {b_ty:?}" + ); + a.clone() + } + (Ty::Column(_), Ty::Inferred(_)) => a.clone(), + (Ty::Inferred(_), Ty::Column(_)) => b.clone(), + (Ty::Inferred(a_ty), Ty::Inferred(b_ty)) => { + assert_eq!( + a_ty, b_ty, + "two inferred types in the same expression disagree: {a_ty:?} vs {b_ty:?}" + ); + a.clone() + } + + // Records — merge field-by-field + (Ty::Record(a_fields), Ty::Record(b_fields)) if a_fields.len() == b_fields.len() => { + Ty::Record( + a_fields + .iter() + .zip(b_fields) + .map(|(a, b)| merge(a, b)) + .collect(), + ) + } + + // Lists — merge element types + (Ty::List(a_elem), Ty::List(b_elem)) => Ty::List(Box::new(merge(a_elem, b_elem))), + + // A `#[document]` collection binds as a `Value::List` (the JSON array's + // internal shape) while its column storage type is the opaque scalar + // `Document`. The schema type is authoritative, so the list resolves to + // the document column type — the value keeps its list shape, only its + // type resolves. This lets `check` type a document column like any other. + (Ty::List(_), col @ Ty::Column(db::Type::Document { .. })) + | (col @ Ty::Column(db::Type::Document { .. }), Ty::List(_)) => col.clone(), + + _ => panic!("cannot merge incompatible types: {a:?} and {b:?}"), + } +} diff --git a/packages/toasty/src/engine/bind/tests.rs b/packages/toasty/src/engine/bind/tests.rs new file mode 100644 index 00000000000..c23bbd23359 --- /dev/null +++ b/packages/toasty/src/engine/bind/tests.rs @@ -0,0 +1,508 @@ +use crate as toasty; +use crate::engine::test_util::*; +use crate::schema::{Embed, Model}; +use toasty_core::{ + schema::db, + stmt::{self, Expr, Value}, +}; + +use super::extract::{binary_like_prefix_pattern, glob_prefix_pattern}; +use super::{Param, Ty, run}; + +// ============================================================================ +// Basic extraction +// ============================================================================ + +#[test] +fn extract_scalar_from_simple_insert() { + #[derive(toasty::Model)] + struct Item { + #[key] + id: String, + name: String, + } + + let schema = test_schema_with(&[Item::schema()]); + + // Build: INSERT INTO items (id, name) VALUES ('abc', 'hello') + let mut stmt = stmt::Statement::Insert(stmt::Insert { + target: insert_target(&schema, "items"), + source: stmt::Query::values(stmt::Values::new(vec![Expr::from(Value::Record( + stmt::ValueRecord::from_vec(vec![Value::from("abc"), Value::from("hello")]), + ))])), + upsert: None, + returning: None, + }); + + let params = run( + &mut stmt, + &schema.db, + &toasty_core::driver::Capability::SQLITE, + ); + + assert_eq!(params.len(), 2); + assert_eq!(params[0].value, Value::from("abc")); + assert_eq!(params[1].value, Value::from("hello")); + + // Statement should have Arg placeholders + if let stmt::Statement::Insert(insert) = &stmt + && let stmt::ExprSet::Values(values) = &insert.source.body + { + let row = &values.rows[0]; + assert!(matches!(row, Expr::Record(_))); + if let Expr::Record(record) = row { + assert!(matches!(record.fields[0], Expr::Arg(_))); + assert!(matches!(record.fields[1], Expr::Arg(_))); + } + } +} + +#[test] +fn null_values_not_extracted() { + #[derive(toasty::Model)] + struct Item { + #[key] + id: String, + name: Option, + } + + let schema = test_schema_with(&[Item::schema()]); + + let mut stmt = stmt::Statement::Insert(stmt::Insert { + target: insert_target(&schema, "items"), + source: stmt::Query::values(stmt::Values::new(vec![Expr::from(Value::Record( + stmt::ValueRecord::from_vec(vec![Value::from("abc"), Value::Null]), + ))])), + upsert: None, + returning: None, + }); + + let params = run( + &mut stmt, + &schema.db, + &toasty_core::driver::Capability::SQLITE, + ); + + // Only 'abc' should be extracted; NULL stays as literal + assert_eq!(params.len(), 1); + assert_eq!(params[0].value, Value::from("abc")); +} + +#[test] +fn extract_from_where_clause() { + let schema = test_schema(); + + // Build: SELECT ... WHERE col = 'hello' + let _filter_expr = Expr::BinaryOp(stmt::ExprBinaryOp { + lhs: Box::new(Expr::Value(Value::from("hello"))), + op: stmt::BinaryOp::Eq, + rhs: Box::new(Expr::Value(Value::from(42i64))), + }); + + let mut stmt = stmt::Statement::Query(stmt::Query { + with: None, + body: stmt::ExprSet::Values(stmt::Values::new(vec![])), + single: false, + order_by: None, + limit: None, + locks: vec![], + }); + + // Can't easily build a full SELECT with filter without a schema, + // but we can test that bind handles an empty query + let params = run( + &mut stmt, + &schema.db, + &toasty_core::driver::Capability::SQLITE, + ); + assert_eq!(params.len(), 0); +} + +// ============================================================================ +// Type refinement for enums +// ============================================================================ + +#[test] +fn enum_insert_refines_type_to_enum() { + #[derive(Debug, toasty::Embed)] + enum Status { + Active, + Inactive, + } + + #[derive(toasty::Model)] + struct Task { + #[key] + id: String, + status: Status, + } + + let schema = test_schema_postgresql(&[Task::schema(), Status::schema()]); + + let mut stmt = stmt::Statement::Insert(stmt::Insert { + target: insert_target(&schema, "tasks"), + source: stmt::Query::values(stmt::Values::new(vec![Expr::from(Value::Record( + stmt::ValueRecord::from_vec(vec![Value::from("task-1"), Value::from("active")]), + ))])), + upsert: None, + returning: None, + }); + + let params = run( + &mut stmt, + &schema.db, + &toasty_core::driver::Capability::SQLITE, + ); + + assert_eq!(params.len(), 2); + + // The status param should be refined to Enum type, not plain Text + assert!( + matches!(¶ms[1].ty, db::Type::Enum(_)), + "expected Enum type for status param, got {:?}", + params[1].ty + ); +} + +#[test] +fn non_enum_insert_keeps_default_types() { + #[derive(toasty::Model)] + struct Item { + #[key] + id: String, + count: i64, + } + + let schema = test_schema_with(&[Item::schema()]); + + let mut stmt = stmt::Statement::Insert(stmt::Insert { + target: insert_target(&schema, "items"), + source: stmt::Query::values(stmt::Values::new(vec![Expr::from(Value::Record( + stmt::ValueRecord::from_vec(vec![Value::from("item-1"), Value::from(42i64)]), + ))])), + upsert: None, + returning: None, + }); + + let params = run( + &mut stmt, + &schema.db, + &toasty_core::driver::Capability::SQLITE, + ); + + assert_eq!(params.len(), 2); + assert!(matches!(¶ms[0].ty, db::Type::Text)); + assert!(matches!(¶ms[1].ty, db::Type::Integer(8))); +} + +// ============================================================================ +// Projection type inference +// ============================================================================ + +#[test] +fn synthesize_multi_step_projection() { + // Test that a multi-step projection like project(record, [1, 0]) + // correctly walks through nested record types. + // + // Given: Record([Text, Record([Integer(4), Boolean])]) + // Project [1, 0] should yield Integer(4) + + use super::infer::synthesize; + + let schema = test_schema(); + + // Directly test the synthesize function with a constructed expression + // that has a multi-step projection + let mut params = vec![ + Param { + value: Value::from("a"), + ty: Ty::Inferred(db::Type::Text), + }, + Param { + value: Value::from(1i32), + ty: Ty::Inferred(db::Type::Integer(4)), + }, + Param { + value: Value::from(true), + ty: Ty::Inferred(db::Type::Boolean), + }, + ]; + + // Build: Record([Arg(0), Record([Arg(1), Arg(2)])]) + let expr = Expr::Record(stmt::ExprRecord::from_vec(vec![ + Expr::arg(0), + Expr::Record(stmt::ExprRecord::from_vec(vec![Expr::arg(1), Expr::arg(2)])), + ])); + + // Project with steps [1, 0] — field 1 of outer (inner record), then field 0 (Integer) + let mut projection = stmt::Projection::single(1); + projection.push(0); + let project_expr = Expr::Project(stmt::ExprProject { + base: Box::new(expr), + projection, + }); + + let cx = stmt::ExprContext::new(&schema.db); + let ty = synthesize(&project_expr, &cx, &mut params); + + assert!( + matches!(ty, Ty::Inferred(db::Type::Integer(4))), + "expected Inferred(Integer(4)), got {:?}", + ty + ); +} + +// ============================================================================ +// Invalid state panics +// ============================================================================ + +#[test] +#[should_panic(expected = "index out of bounds")] +fn synthesize_arg_out_of_range_panics() { + use super::infer::synthesize; + + let schema = test_schema(); + let cx = stmt::ExprContext::new(&schema.db); + let mut params = vec![]; + + // Arg(5) but params is empty + synthesize(&Expr::arg(5), &cx, &mut params); +} + +#[test] +#[should_panic(expected = "out of range")] +fn synthesize_project_step_out_of_bounds_panics() { + use super::infer::synthesize; + + let schema = test_schema(); + let cx = stmt::ExprContext::new(&schema.db); + let mut params = vec![Param { + value: Value::from("a"), + ty: Ty::Inferred(db::Type::Text), + }]; + + // Record with 1 field, projecting step 5 + let expr = Expr::Project(stmt::ExprProject { + base: Box::new(Expr::Record(stmt::ExprRecord::from_vec(vec![Expr::arg(0)]))), + projection: stmt::Projection::single(5), + }); + + synthesize(&expr, &cx, &mut params); +} + +#[test] +#[should_panic(expected = "non-record")] +fn synthesize_project_from_scalar_panics() { + use super::infer::synthesize; + + let schema = test_schema(); + let cx = stmt::ExprContext::new(&schema.db); + let mut params = vec![Param { + value: Value::from(42i64), + ty: Ty::Inferred(db::Type::Integer(8)), + }]; + + // Project from a scalar Arg (not a record) + let expr = Expr::Project(stmt::ExprProject { + base: Box::new(Expr::arg(0)), + projection: stmt::Projection::single(0), + }); + + synthesize(&expr, &cx, &mut params); +} + +#[test] +#[should_panic(expected = "incompatible")] +fn merge_incompatible_structures_panics() { + use super::infer::merge; + + // Record vs Inferred scalar + let a = Ty::Record(vec![Ty::Inferred(db::Type::Text)]); + let b = Ty::Inferred(db::Type::Integer(8)); + + merge(&a, &b); +} + +#[test] +#[should_panic(expected = "incompatible")] +fn merge_records_different_lengths_panics() { + use super::infer::merge; + + let a = Ty::Record(vec![Ty::Inferred(db::Type::Text)]); + let b = Ty::Record(vec![ + Ty::Inferred(db::Type::Text), + Ty::Inferred(db::Type::Integer(8)), + ]); + + merge(&a, &b); +} + +#[test] +fn merge_column_wins_over_inferred() { + use super::infer::merge; + + let col = Ty::Column(db::Type::Enum(db::TypeEnum { + name: Some("status".to_string()), + variants: vec![], + })); + let inferred = Ty::Inferred(db::Type::Text); + + // Column should win regardless of argument order + let result = merge(&col, &inferred); + assert!(result.is_column(), "expected Column, got {:?}", result); + + let result = merge(&inferred, &col); + assert!(result.is_column(), "expected Column, got {:?}", result); +} + +// ============================================================================ +// Helpers +// ============================================================================ + +/// Build an InsertTarget::Table for the named table in the schema. +fn insert_target(schema: &toasty_core::Schema, table_name: &str) -> stmt::InsertTarget { + let table = schema + .db + .tables + .iter() + .find(|t| t.name.ends_with(table_name)) + .unwrap_or_else(|| panic!("table '{table_name}' not found in schema")); + + stmt::InsertTarget::Table(stmt::InsertTable { + table: table.id, + columns: table.columns.iter().map(|c| c.id).collect(), + }) +} + +// ============================================================================ +// glob_prefix_pattern +// ============================================================================ + +#[test] +fn glob_plain_prefix() { + assert_eq!(glob_prefix_pattern("alpha"), "alpha*"); +} + +#[test] +fn glob_empty_prefix() { + assert_eq!(glob_prefix_pattern(""), "*"); +} + +#[test] +fn glob_escapes_star() { + assert_eq!(glob_prefix_pattern("100*off"), "100[*]off*"); +} + +#[test] +fn glob_escapes_question_mark() { + assert_eq!(glob_prefix_pattern("a?b"), "a[?]b*"); +} + +#[test] +fn glob_escapes_open_bracket() { + assert_eq!(glob_prefix_pattern("a[b"), "a[[]b*"); +} + +#[test] +fn glob_like_wildcards_are_literal() { + // `%` and `_` are LIKE wildcards but not GLOB metacharacters. + assert_eq!(glob_prefix_pattern("100%"), "100%*"); + assert_eq!(glob_prefix_pattern("a_b"), "a_b*"); +} + +// ============================================================================ +// binary_like_prefix_pattern +// ============================================================================ + +#[test] +fn binary_like_plain_prefix() { + assert_eq!(binary_like_prefix_pattern("alpha"), "alpha%"); +} + +#[test] +fn binary_like_empty_prefix() { + assert_eq!(binary_like_prefix_pattern(""), "%"); +} + +#[test] +fn binary_like_escapes_percent() { + assert_eq!(binary_like_prefix_pattern("100%"), "100!%%"); +} + +#[test] +fn binary_like_escapes_underscore() { + assert_eq!(binary_like_prefix_pattern("a_b"), "a!_b%"); +} + +#[test] +fn binary_like_escapes_escape_char() { + assert_eq!(binary_like_prefix_pattern("!bang"), "!!bang%"); +} + +#[test] +fn binary_like_glob_wildcards_are_literal() { + // `*` and `?` are GLOB metacharacters but not LIKE metacharacters. + assert_eq!(binary_like_prefix_pattern("foo*"), "foo*%"); + assert_eq!(binary_like_prefix_pattern("a?b"), "a?b%"); +} + +// ============================================================================ +// Expr::Static survives binding +// ============================================================================ + +#[test] +fn static_value_in_insert_row_is_not_extracted() { + // Mixed row: one user-supplied `Expr::Value` (extracted) and one + // synthetic `Expr::Static` (passes through). Confirms the bind pass + // keys on the variant, not the value content. + #[derive(toasty::Model)] + struct Item { + #[key] + id: String, + name: String, + } + + let schema = test_schema_with(&[Item::schema()]); + + let mut stmt = stmt::Statement::Insert(stmt::Insert { + target: insert_target(&schema, "items"), + source: stmt::Query::values(stmt::Values::new(vec![Expr::Record(stmt::ExprRecord { + fields: vec![ + Expr::Static(Value::from("static-key")), + Expr::Value(Value::from("user-name")), + ], + })])), + upsert: None, + returning: None, + }); + + let params = run( + &mut stmt, + &schema.db, + &toasty_core::driver::Capability::SQLITE, + ); + + // Only the `Expr::Value` is extracted; the `Expr::Static` passes through. + assert_eq!(params.len(), 1); + assert_eq!(params[0].value, Value::from("user-name")); + + let stmt::Statement::Insert(insert) = &stmt else { + unreachable!() + }; + let stmt::ExprSet::Values(values) = &insert.source.body else { + unreachable!() + }; + let Expr::Record(record) = &values.rows[0] else { + unreachable!() + }; + assert!( + matches!(&record.fields[0], Expr::Static(Value::String(s)) if s == "static-key"), + "Static leaf should survive binding, got {:#?}", + record.fields[0] + ); + assert!( + matches!(&record.fields[1], Expr::Arg(_)), + "Value leaf should be extracted to Arg, got {:#?}", + record.fields[1] + ); +} diff --git a/packages/toasty/src/engine/effect.rs b/packages/toasty/src/engine/effect.rs new file mode 100644 index 00000000000..4c5d97f7584 --- /dev/null +++ b/packages/toasty/src/engine/effect.rs @@ -0,0 +1,72 @@ +//! Statement effect classification. +//! +//! [`classify`] walks a [`Statement`] AST and returns whether the +//! statement can be retried transparently on `connection_lost`. The +//! pool consults this before attempting a retry; downstream consumers +//! (e.g. a runtime-checked read-only API surface) read the same +//! classification. +//! +//! See [`docs/dev/design/retry-safe-recovery.md`] for the full +//! contract. +//! +//! [`Statement`]: toasty_core::stmt::Statement +//! [`docs/dev/design/retry-safe-recovery.md`]: ../../../docs/dev/design/retry-safe-recovery.md + +use toasty_core::stmt::{Delete, Insert, Statement, Update, Visit}; + +/// Whether a statement mutates database state. +/// +/// A statement is [`Effect::ReadOnly`] if it is a [`Statement::Query`] +/// and contains no `Insert`, `Update`, or `Delete` anywhere in its +/// tree. Otherwise it is [`Effect::Mutating`]. +/// +/// CTE-with-mutation queries (e.g. +/// `WITH ins AS (INSERT ... RETURNING *) SELECT * FROM ins`) parse as +/// `Statement::Query` values whose `WITH` clauses contain an +/// `ExprSet::Insert`, `ExprSet::Update`, or `ExprSet::Delete` and are +/// correctly classified as `Mutating`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Effect { + /// The statement reads but does not mutate state. Safe to retry + /// after `connection_lost`. + ReadOnly, + + /// The statement mutates state, either directly (top-level + /// `Insert` / `Update` / `Delete`) or via an embedded sub-statement. + /// Not safe to retry without further analysis. + Mutating, +} + +/// Classify a statement's effect on database state. +/// +/// O(n) in the size of the statement tree; no schema access. +pub(crate) fn classify(stmt: &Statement) -> Effect { + let mut walker = Walker { mutating: false }; + walker.visit_stmt(stmt); + if walker.mutating { + Effect::Mutating + } else { + Effect::ReadOnly + } +} + +struct Walker { + mutating: bool, +} + +impl Visit for Walker { + fn visit_stmt_delete(&mut self, _: &Delete) { + self.mutating = true; + } + + fn visit_stmt_insert(&mut self, _: &Insert) { + self.mutating = true; + } + + fn visit_stmt_update(&mut self, _: &Update) { + self.mutating = true; + } +} + +#[cfg(test)] +mod tests; diff --git a/packages/toasty/src/engine/effect/tests.rs b/packages/toasty/src/engine/effect/tests.rs new file mode 100644 index 00000000000..b2e92d9667d --- /dev/null +++ b/packages/toasty/src/engine/effect/tests.rs @@ -0,0 +1,203 @@ +use super::{Effect, classify}; +use toasty_core::schema::app::ModelId; +use toasty_core::schema::db::TableId; +use toasty_core::stmt::{ + self, Assignments, Condition, Delete, Expr, ExprStmt, Filter, Insert, InsertTarget, Query, + Source, Statement, Update, UpdateTarget, With, +}; + +fn model() -> ModelId { + ModelId(0) +} + +fn simple_query() -> Query { + Query::new_select(Source::from(model()), Filter::default()) +} + +fn simple_insert() -> Insert { + Insert { + target: InsertTarget::Model(model()), + source: Query::unit(), + upsert: None, + returning: None, + } +} + +fn simple_update() -> Update { + Update { + target: UpdateTarget::Model(model()), + assignments: Assignments::default(), + filter: Filter::default(), + condition: Condition::default(), + returning: None, + } +} + +fn simple_delete() -> Delete { + Delete { + from: Source::from(model()), + filter: Filter::default(), + returning: None, + condition: Condition::default(), + } +} + +fn query_with_cte(body: impl Into) -> Statement { + let mut query = simple_query(); + query.with = Some(With { + ctes: vec![stmt::Cte { + query: Query::new(body), + }], + }); + Statement::Query(query) +} + +// --- top-level variants --- + +#[test] +fn top_level_query_is_read_only() { + let stmt = Statement::Query(simple_query()); + assert_eq!(classify(&stmt), Effect::ReadOnly); +} + +#[test] +fn top_level_insert_is_mutating() { + let stmt = Statement::Insert(simple_insert()); + assert_eq!(classify(&stmt), Effect::Mutating); +} + +#[test] +fn top_level_update_is_mutating() { + let stmt = Statement::Update(simple_update()); + assert_eq!(classify(&stmt), Effect::Mutating); +} + +#[test] +fn top_level_delete_is_mutating() { + let stmt = Statement::Delete(simple_delete()); + assert_eq!(classify(&stmt), Effect::Mutating); +} + +// --- embedded mutation sub-statements --- + +#[test] +fn query_with_insert_in_filter_is_mutating() { + // SELECT ... WHERE id = (INSERT ... RETURNING id) + // Build by injecting an Expr::Stmt(Insert) into the filter expression. + let mut query = simple_query(); + let stmt::ExprSet::Select(select) = &mut query.body else { + unreachable!() + }; + select.filter = Filter::from(Expr::Stmt(ExprStmt { + stmt: Box::new(Statement::Insert(simple_insert())), + })); + + let stmt = Statement::Query(query); + assert_eq!(classify(&stmt), Effect::Mutating); +} + +#[test] +fn query_with_update_in_filter_is_mutating() { + let mut query = simple_query(); + let stmt::ExprSet::Select(select) = &mut query.body else { + unreachable!() + }; + select.filter = Filter::from(Expr::Stmt(ExprStmt { + stmt: Box::new(Statement::Update(simple_update())), + })); + + let stmt = Statement::Query(query); + assert_eq!(classify(&stmt), Effect::Mutating); +} + +#[test] +fn query_with_delete_in_filter_is_mutating() { + let mut query = simple_query(); + let stmt::ExprSet::Select(select) = &mut query.body else { + unreachable!() + }; + select.filter = Filter::from(Expr::Stmt(ExprStmt { + stmt: Box::new(Statement::Delete(simple_delete())), + })); + + let stmt = Statement::Query(query); + assert_eq!(classify(&stmt), Effect::Mutating); +} + +#[test] +fn query_with_nested_query_only_is_read_only() { + // SELECT ... WHERE id IN (SELECT id FROM other) — a subquery that + // doesn't mutate keeps the outer statement ReadOnly. + let mut outer = simple_query(); + let stmt::ExprSet::Select(select) = &mut outer.body else { + unreachable!() + }; + select.filter = Filter::from(Expr::Stmt(ExprStmt { + stmt: Box::new(Statement::Query(simple_query())), + })); + + let stmt = Statement::Query(outer); + assert_eq!(classify(&stmt), Effect::ReadOnly); +} + +#[test] +fn query_with_cte_insert_is_mutating() { + // WITH ins AS (INSERT ... RETURNING *) SELECT * FROM ins + let stmt = query_with_cte(simple_insert()); + + assert_eq!(classify(&stmt), Effect::Mutating); +} + +#[test] +fn query_with_cte_update_is_mutating() { + let stmt = query_with_cte(simple_update()); + + assert_eq!(classify(&stmt), Effect::Mutating); +} + +#[test] +fn query_with_cte_delete_is_mutating() { + let stmt = query_with_cte(simple_delete()); + + assert_eq!(classify(&stmt), Effect::Mutating); +} + +#[test] +fn deeply_nested_mutation_is_mutating() { + // SELECT ... WHERE id IN (SELECT id FROM ... WHERE x = (UPDATE ... RETURNING id)) + // Mutation buried two levels of subquery deep. + let mut inner = simple_query(); + let stmt::ExprSet::Select(select) = &mut inner.body else { + unreachable!() + }; + select.filter = Filter::from(Expr::Stmt(ExprStmt { + stmt: Box::new(Statement::Update(simple_update())), + })); + + let mut outer = simple_query(); + let stmt::ExprSet::Select(select) = &mut outer.body else { + unreachable!() + }; + select.filter = Filter::from(Expr::Stmt(ExprStmt { + stmt: Box::new(Statement::Query(inner)), + })); + + let stmt = Statement::Query(outer); + assert_eq!(classify(&stmt), Effect::Mutating); +} + +// --- lowered shapes also classify correctly --- + +#[test] +fn lowered_update_to_table_is_mutating() { + // Post-lowering, UpdateTarget::Model becomes UpdateTarget::Table. + // The top-level variant check still fires regardless of target shape. + let stmt = Statement::Update(Update { + target: UpdateTarget::Table(TableId(0)), + assignments: Assignments::default(), + filter: Filter::default(), + condition: Condition::default(), + returning: None, + }); + assert_eq!(classify(&stmt), Effect::Mutating); +} diff --git a/packages/toasty/src/engine/eval.rs b/packages/toasty/src/engine/eval.rs new file mode 100644 index 00000000000..eceb3484f4e --- /dev/null +++ b/packages/toasty/src/engine/eval.rs @@ -0,0 +1,87 @@ +mod as_expr; +#[cfg(test)] +mod tests; + +use as_expr::AsExpr; + +use crate::Result; +use toasty_core::{ + schema::Schema, + stmt::{self, ExprContext}, +}; + +#[derive(Clone, Debug)] +pub(crate) struct Func { + /// Expression arguments + pub(crate) args: Vec, + + /// Expression return type + pub(crate) ret: stmt::Type, + + /// Expression to evaluate + expr: T, +} + +impl Func { + pub(crate) fn from_stmt(expr: T, args: Vec) -> Self { + assert!(expr.as_expr().is_eval()); + let ret = ExprContext::new_free().infer_expr_ty(expr.as_expr(), &args); + Self { args, ret, expr } + } + + pub(crate) fn from_stmt_typed(expr: T, args: Vec, ret: stmt::Type) -> Self { + Self { args, ret, expr } + } + + /// Returns true if the function has no inputs + pub(crate) fn is_const(&self) -> bool { + self.args.is_empty() + } + + pub(crate) fn is_identity(&self) -> bool { + matches!(self.expr.as_expr(), stmt::Expr::Arg(expr_arg) if expr_arg.position == 0) + } + + /// Evaluates the function against `input`. `schema` directs the + /// schema-directed conversions (a `#[document]` embed's record ↔ object + /// casts); every other operation is schema-free. + pub(crate) fn eval(&self, schema: &Schema, input: impl stmt::Input) -> Result { + use stmt::TypedInput; + + let input = TypedInput::new(stmt::ExprContext::new(schema), &self.args, input); + self.expr.as_expr().eval(input) + } + + pub(crate) fn eval_const(&self, schema: &Schema) -> stmt::Value { + assert!(self.is_const()); + self.eval(schema, stmt::ConstInput::new()).unwrap() + } + + pub(crate) fn eval_bool(&self, schema: &Schema, input: impl stmt::Input) -> Result { + use stmt::TypedInput; + + let input = TypedInput::new(stmt::ExprContext::new(schema), &self.args, input); + self.expr.as_expr().eval_bool(input) + } +} + +impl Func { + /// Consumes the function, returning its expression. + pub(crate) fn into_expr(self) -> stmt::Expr { + self.expr + } +} + +impl Func<&stmt::Expr> { + pub(crate) fn try_from_stmt( + expr: &stmt::Expr, + args: Vec, + ) -> Option> { + if !expr.is_eval() { + return None; + } + + let ret = ExprContext::new_free().infer_expr_ty(expr, &args); + Some(Func::from_stmt_typed(expr, args, ret)) + } +} diff --git a/packages/toasty/src/engine/eval/as_expr.rs b/packages/toasty/src/engine/eval/as_expr.rs new file mode 100644 index 00000000000..53bc8e4ff4c --- /dev/null +++ b/packages/toasty/src/engine/eval/as_expr.rs @@ -0,0 +1,29 @@ +use toasty_core::stmt; + +pub(crate) trait AsExpr { + fn as_expr(&self) -> &stmt::Expr; +} + +impl AsExpr for stmt::Expr { + fn as_expr(&self) -> &stmt::Expr { + self + } +} + +impl AsExpr for &stmt::Expr { + fn as_expr(&self) -> &stmt::Expr { + self + } +} + +impl AsExpr for stmt::Filter { + fn as_expr(&self) -> &stmt::Expr { + self.as_expr() + } +} + +impl AsExpr for &stmt::Filter { + fn as_expr(&self) -> &stmt::Expr { + stmt::Filter::as_expr(self) + } +} diff --git a/packages/toasty/src/engine/eval/tests.rs b/packages/toasty/src/engine/eval/tests.rs new file mode 100644 index 00000000000..ed407703a08 --- /dev/null +++ b/packages/toasty/src/engine/eval/tests.rs @@ -0,0 +1,32 @@ +use super::*; +use crate::engine::test_util::test_schema; +use stmt::{Expr, Query, Value, Values}; + +#[test] +fn evaluable_expressions_build_and_run() { + let schema = test_schema(); + for (expr, expected) in [ + (Expr::Static(Value::I64(42)), Value::I64(42)), + (Expr::in_list(42, Expr::list([42])), Value::Bool(true)), + (Expr::any(Expr::list([false, true])), Value::Bool(true)), + ( + Expr::exists(Query::values(Values::new(vec![42.into()]))), + Value::Bool(true), + ), + ] { + let func = Func::try_from_stmt(&expr, vec![]).unwrap(); + assert_eq!(func.eval_const(&schema), expected); + let func = Func::from_stmt(expr, vec![]); + assert_eq!(func.eval_const(&schema), expected); + } +} + +#[test] +fn unsupported_expressions_return_none() { + for expr in [ + Expr::between(2, 1, 3), + Expr::match_expr(1, vec![], Expr::count_star()), + ] { + assert!(Func::try_from_stmt(&expr, vec![]).is_none()); + } +} diff --git a/packages/toasty/src/engine/exec.rs b/packages/toasty/src/engine/exec.rs new file mode 100644 index 00000000000..46dede5ce5f --- /dev/null +++ b/packages/toasty/src/engine/exec.rs @@ -0,0 +1,210 @@ +mod branch; +pub(crate) use branch::If; + +mod delete_by_key; + +mod eval; + +mod exec_statement; +pub(crate) use exec_statement::{ConditionalOutput, PaginationConfig}; + +mod filter; + +mod find_pk_by_index; + +mod get_by_key; + +mod kv; + +mod nested_merge; +pub(crate) use nested_merge::{MergeIndex, MergeQualification, NestedChild, NestedLevel}; + +mod plan; +pub(crate) use plan::{ExecPlan, Step}; + +mod query_pk; + +mod repeat; + +mod rmw; + +mod scan; + +mod update_by_key; + +mod upsert; + +mod var; +pub(crate) use var::VarStore; + +use crate::{ + Result, + engine::{Engine, mir}, +}; +use toasty_core::{ + Connection, + driver::{ExecResponse, Rows, operation::Transaction}, + stmt::{self, ValueStream}, +}; + +struct Exec<'a> { + engine: &'a Engine, + connection: &'a mut dyn Connection, + vars: VarStore, + /// True when an outer transaction is active on this connection. Used by + /// ReadModifyWrite to decide between savepoints (nested) and its own + /// BEGIN/COMMIT (standalone). + in_transaction: bool, +} + +impl Engine { + pub(crate) async fn exec_plan( + &self, + connection: &mut dyn Connection, + plan: ExecPlan, + in_transaction: bool, + ) -> Result { + let ExecPlan { + plan: logical_plan, + vars, + steps, + returning, + needs_transaction, + } = plan; + + let mut exec = Exec { + engine: self, + connection, + vars, + in_transaction, + }; + + // When nested inside an outer transaction use savepoints so the outer + // transaction can still commit or roll back as a whole. When standalone, + // start our own transaction (MySQL requires an active BEGIN before + // SAVEPOINT can be used, so we can't use savepoints here). + let (begin, commit, rollback) = if exec.in_transaction { + let name = "statement"; + ( + Transaction::Savepoint(name.to_owned()), + Transaction::ReleaseSavepoint(name.to_owned()), + Transaction::RollbackToSavepoint(name.to_owned()), + ) + } else { + ( + Transaction::start(), + Transaction::Commit, + Transaction::Rollback, + ) + }; + + if needs_transaction { + tracing::trace!("beginning plan transaction"); + exec.connection.exec(&self.schema, begin.into()).await?; + exec.in_transaction = true; + } + + for (i, step) in steps.iter().enumerate() { + tracing::trace!(step = i, action = %step.name(&logical_plan), "executing action"); + // Debug, not error: the failure propagates to the caller, who + // decides whether it is an application error. A handled unique + // violation should not error-spam production logs. + if let Err(e) = exec.exec_step(&logical_plan, step).await { + tracing::debug!(step = i, action = %step.name(&logical_plan), error = %e, "action failed"); + if needs_transaction { + tracing::trace!("rolling back plan transaction due to error"); + // Best effort: ignore rollback errors so the original error is returned + let _ = exec.connection.exec(&self.schema, rollback.into()).await; + } + return Err(e); + } + } + + if needs_transaction { + tracing::trace!("committing plan transaction"); + exec.connection.exec(&self.schema, commit.into()).await?; + } + + let response = exec.vars.load(returning).await?; + tracing::trace!("final result from var {:?}:\n{:#?}", returning, response); + + // With exact use counts, the returning load was the last use of + // the last live slot (success path only — the loop above returned + // early on failure). + exec.vars.assert_empty(); + + let value_stream = match response.values { + Rows::Count(_) => ValueStream::default(), + Rows::Value(stmt::Value::List(items)) => ValueStream::from_vec(items), + // TODO have the public API be able to handle single rows + Rows::Value(value) => ValueStream::from_vec(vec![value]), + Rows::Stream(value_stream) => value_stream, + }; + + Ok(ExecResponse { + values: Rows::Stream(value_stream), + next_cursor: response.next_cursor, + prev_cursor: response.prev_cursor, + }) + } +} + +impl Exec<'_> { + async fn exec_step(&mut self, logical_plan: &mir::LogicalPlan, step: &Step) -> Result<()> { + match step { + Step::Run(node_id) => self.exec_node(logical_plan, *node_id).await, + Step::If(action) => self.action_if(logical_plan, action).await, + } + } + + /// Executes one node's operation and stores its output in the node's + /// variable slot. + async fn exec_node( + &mut self, + logical_plan: &mir::LogicalPlan, + node_id: mir::NodeId, + ) -> Result<()> { + use mir::Operation; + + let node = &logical_plan[node_id]; + + let response = match &node.op { + // A pass-through: the response (stream included) relocates + // between slots without buffering. + Operation::Alias(op) => self.vars.load(op.input).await?, + Operation::Const(op) => ExecResponse::from_rows(Rows::Value(op.value.clone())), + Operation::DeleteByKey(op) => self.exec_delete_by_key(op).await?, + Operation::Eval(op) => self.exec_eval(op).await?, + Operation::ExecStatement(op) => self.exec_statement(op).await?, + Operation::Filter(op) => self.exec_filter(op).await?, + Operation::FindPkByIndex(op) => self.exec_find_pk_by_index(op).await?, + Operation::GetByKey(op) => self.exec_get_by_key(op).await?, + Operation::NestedMerge(op) => self.exec_nested_merge(op).await?, + Operation::ReadModifyWrite(op) => self.exec_read_modify_write(op).await?, + Operation::Repeat(op) => self.exec_repeat(op).await?, + Operation::QueryPk(op) => self.exec_query_pk(op).await?, + Operation::Scan(op) => self.exec_scan(op).await?, + Operation::UpdateByKey(op) => self.exec_update_by_key(op).await?, + Operation::Upsert(op) => self.exec_upsert(op).await?, + }; + + self.vars.store(node_id, node.ty(), node.num_uses, response); + + Ok(()) + } + + async fn collect_input( + &mut self, + input: impl IntoIterator, + ) -> Result> { + let mut ret = Vec::new(); + + for node_id in input { + let response = self.vars.load(node_id).await?; + let value = response.values.collect_as_value().await?; + ret.push(value); + } + + Ok(ret) + } +} diff --git a/packages/toasty/src/engine/exec/branch.rs b/packages/toasty/src/engine/exec/branch.rs new file mode 100644 index 00000000000..bc18b4a0ac4 --- /dev/null +++ b/packages/toasty/src/engine/exec/branch.rs @@ -0,0 +1,69 @@ +use crate::{ + Result, + engine::{exec::Exec, mir, mir::LogicalPlan}, +}; + +/// A conditionally executed block of nodes — the exec program's first +/// control-flow construct. +/// +/// The `then` arm holds pure nodes whose outputs are only consumed when the +/// condition holds; mutations never appear in it. Every node in the arm +/// carries the same guard, which is the block's condition. Skipping the arm +/// is pure bookkeeping, declared as data: `skipped_inputs` and +/// `skipped_outputs` keep variable slots consistent so consumers outside the +/// block never see an unset slot and use counts stay exact on both paths. +#[derive(Debug)] +pub(crate) struct If { + /// Nodes run when the condition holds. Non-empty; the condition is the + /// nodes' shared guard. + pub(crate) then: Vec, + + /// Nodes produced outside the block whose outputs the `then` arm would + /// have loaded — one entry per declined load. Released when the arm is + /// skipped. + pub(crate) skipped_inputs: Vec, + + /// The `then` arm's escaping outputs, each with its external use count. + /// When the arm is skipped, each receives a non-loadable slot used only + /// for use counting. + pub(crate) skipped_outputs: Vec<(mir::NodeId, usize)>, +} + +impl If { + fn guard(&self, logical_plan: &LogicalPlan) -> mir::NodeId { + // Every node in the branch has the same guard, so the first node's + // guard is the branch's guard. + logical_plan[self.then[0]] + .guard + .expect("`If` block nodes carry the block's guard") + } +} + +impl Exec<'_> { + pub(super) async fn action_if( + &mut self, + logical_plan: &LogicalPlan, + action: &If, + ) -> Result<()> { + let cond = action.guard(logical_plan); + + // A non-consuming peek: buffers the condition variable's stream in + // place, leaving its use count untouched. + let pass = self.vars.peek_non_empty(cond).await?; + + if pass { + for &node_id in &action.then { + self.exec_node(logical_plan, node_id).await?; + } + } else { + for &node_id in &action.skipped_inputs { + self.vars.release(node_id); + } + for &(node_id, num_uses) in &action.skipped_outputs { + self.vars.store_skipped(node_id, num_uses); + } + } + + Ok(()) + } +} diff --git a/packages/toasty/src/engine/exec/delete_by_key.rs b/packages/toasty/src/engine/exec/delete_by_key.rs new file mode 100644 index 00000000000..99b2746c859 --- /dev/null +++ b/packages/toasty/src/engine/exec/delete_by_key.rs @@ -0,0 +1,47 @@ +use toasty_core::driver::{ExecResponse, Rows, operation}; + +use crate::engine::{exec::Exec, mir}; + +use super::Result; + +impl Exec<'_> { + pub(super) async fn exec_delete_by_key( + &mut self, + action: &mir::DeleteByKey, + ) -> Result { + let keys = self + .vars + .load(action.input) + .await? + .values + .collect_as_value() + .await? + .into_list_unwrap(); + + let res = if keys.is_empty() { + Rows::Count(0) + } else { + let mut total_count = 0u64; + + for key in keys { + let op = operation::DeleteByKey { + table: action.table, + keys: vec![key], + filter: action.filter.clone(), + condition: action.condition.clone(), + }; + + let res = self.connection.exec(&self.engine.schema, op.into()).await?; + + match res.values { + Rows::Count(n) => total_count += n, + _ => panic!("expected Count from DeleteByKey"), + } + } + + Rows::Count(total_count) + }; + + Ok(ExecResponse::from_rows(res)) + } +} diff --git a/packages/toasty/src/engine/exec/eval.rs b/packages/toasty/src/engine/exec/eval.rs new file mode 100644 index 00000000000..3ebaa8a7596 --- /dev/null +++ b/packages/toasty/src/engine/exec/eval.rs @@ -0,0 +1,101 @@ +use crate::{ + Result, + engine::{exec::Exec, mir}, +}; +use toasty_core::{ + driver::{ExecResponse, Rows}, + stmt::Value, +}; + +impl Exec<'_> { + pub(super) async fn exec_eval(&mut self, action: &mir::Eval) -> Result { + match action.row_input { + Some(row_input) => self.exec_eval_map_over(action, row_input).await, + None => self.exec_eval_compute(action).await, + } + } + + async fn exec_eval_compute(&mut self, action: &mir::Eval) -> Result { + // This form evaluates the body once with each complete input. + // For example, two input nodes produce this call: + // + // body.eval([input_0, input_1]) + let mut input = Vec::with_capacity(action.inputs.len()); + + for &node_id in &action.inputs { + let response = self.vars.load(node_id).await?; + // Only a row input can pass page cursors to an `Eval` result. + debug_assert!(response.is_unpaginated()); + input.push(response.values.collect_as_value().await?); + } + + let result = action.body.eval(&self.engine.schema, &input)?; + + Ok(ExecResponse::from_rows(Rows::Value(result))) + } + + async fn exec_eval_map_over( + &mut self, + action: &mir::Eval, + row_input: mir::NodeId, + ) -> Result { + // This form maps the body over the rows from `row_input`. + // For example, two rows produce this result: + // + // [row_body.eval(row_input[0], inputs...), + // row_body.eval(row_input[1], inputs...)] + // + // Load `row_input` first because it decides whether the other inputs + // are needed. + let ExecResponse { + values, + next_cursor, + prev_cursor, + } = self.vars.load(row_input).await?; + let input_rows = values.collect_as_value().await?; + + if input_rows.is_list_empty() { + return Ok(self.exec_eval_map_over_empty_input(action, next_cursor, prev_cursor)); + } + + let mut input = Vec::with_capacity(1 + action.inputs.len()); + input.push(input_rows); + + for &node_id in &action.inputs { + let input_response = self.vars.load(node_id).await?; + // Only a row input can pass page cursors to an `Eval` result. + debug_assert!(input_response.is_unpaginated()); + input.push(input_response.values.collect_as_value().await?); + } + + let result = action.body.eval(&self.engine.schema, &input)?; + + // The output has one item for each input row. Its cursors must match + // the input cursors so the caller can fetch the next or previous page. + Ok(ExecResponse { + values: Rows::Value(result), + next_cursor, + prev_cursor, + }) + } + + fn exec_eval_map_over_empty_input( + &mut self, + action: &mir::Eval, + next_cursor: Option>, + prev_cursor: Option>, + ) -> ExecResponse { + // There are no rows to evaluate. Do not load the other inputs. + // Loading a value consumes one recorded use. Since this path skips + // those loads, release the uses instead. + for &node_id in &action.inputs { + self.vars.release(node_id); + } + + ExecResponse { + values: Rows::Value(Value::List(vec![])), + next_cursor, + prev_cursor, + } + } +} diff --git a/packages/toasty/src/engine/exec/exec_statement.rs b/packages/toasty/src/engine/exec/exec_statement.rs new file mode 100644 index 00000000000..723ecc05429 --- /dev/null +++ b/packages/toasty/src/engine/exec/exec_statement.rs @@ -0,0 +1,356 @@ +use toasty_core::{ + driver::{ExecResponse, Rows, operation}, + stmt, +}; + +use crate::{ + Result, + engine::{eval, exec::Exec, mir}, +}; + +/// How to interpret a statement's output rows. +/// +/// A conditional write (the SQL `#[version]` / OCC path compiled as a single +/// CTE statement) prefixes its result with two probe columns: the number of +/// rows matching the filter and, of those, the number satisfying the condition. +/// The write applied only when the two agree; a mismatch is a condition +/// failure, and zero matched rows means the record no longer exists. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ConditionalOutput { + /// Not a conditional write. Output is passed through unchanged. + None, + + /// Conditional write with no `RETURNING`. The two probe columns are the + /// only output; the action reports the matched-row count. + Count, + + /// Conditional write with a `RETURNING`. The two probe columns are followed + /// by the changed rows' columns, which become the action's output. + Returning, +} + +/// Configuration for pagination at the execution level. +#[derive(Debug, Clone)] +pub(crate) struct PaginationConfig { + /// Number of items per page + pub page_size: i64, + /// Whether the query starts after a cursor and can have a previous page. + pub has_previous_page: bool, + /// Function to extract cursor from a row (SQL only). + /// For NoSQL drivers, this is None (driver provides cursor). + pub extract_cursor: Option, +} + +/// Information about a MySQL UPDATE with RETURNING that needs special handling. +/// +/// MySQL doesn't support `RETURNING` on `UPDATE`. The workaround is to strip +/// the returning, run the UPDATE, then run a follow-up `SELECT` over the same +/// table and filter to fetch the post-update column values. The two +/// statements are not atomic relative to concurrent writers — see #881 for +/// the broader design discussion. +#[derive(Debug)] +pub(super) struct MySQLUpdateReturning { + /// The `SELECT` statement that returns the post-update values. Carries + /// the same filter as the original `UPDATE` plus the projected + /// returning expression. + select_stmt: stmt::Statement, +} + +impl Exec<'_> { + pub(super) async fn exec_statement( + &mut self, + action: &mir::ExecStatement, + ) -> Result { + // Databases always return rows as a vec of values; this specifies the + // type of each value. `None` means the statement returns only a count. + let output_ty = mir::row_field_types(&action.ty); + + let mut stmt = action.stmt.clone(); + + // Collect input values and substitute into the statement + if !action.inputs.is_empty() { + let input_values = self.collect_input(action.inputs.iter().copied()).await?; + stmt.substitute(&input_values); + + self.engine.simplify_stmt(&mut stmt); + } + + debug_assert!( + stmt.returning() + .and_then(|returning| returning.as_project()) + .map(|expr| expr.is_record()) + .unwrap_or(true), + "stmt={stmt:#?}" + ); + + // MySQL does not support `RETURNING` on `UPDATE`. Strip the returning + // and capture an equivalent `SELECT` to run after the UPDATE. + let mysql_update_returning = self.process_stmt_update_with_returning_on_mysql(&mut stmt); + + // Short circuit if we can statically determine there are no results + if let stmt::Statement::Query(query) = &stmt + && let stmt::ExprSet::Values(values) = &query.body + && values.is_empty() + { + assert_eq!(action.conditional, ConditionalOutput::None); + + let rows = if output_ty.is_some() { + Rows::Stream(stmt::ValueStream::default()) + } else { + Rows::Count(0) + }; + + return Ok(ExecResponse::from_rows(rows)); + } + + // Legalize the statement for the target backend and extract bind + // parameters (SQL drivers only; key-value drivers read values + // directly from the statement). + let params = self.engine.prepare_for_driver(&mut stmt); + + let ret = match action.conditional { + // A conditional write prefixes its result with two `I64` probe + // counts; the `Returning` variant follows them with the changed + // rows' columns. + ConditionalOutput::Count => Some(vec![stmt::Type::I64, stmt::Type::I64]), + ConditionalOutput::Returning => { + let mut tys = vec![stmt::Type::I64, stmt::Type::I64]; + tys.extend( + output_ty + .clone() + .expect("conditional write with RETURNING has output columns"), + ); + Some(tys) + } + ConditionalOutput::None if mysql_update_returning.is_some() => { + // The UPDATE has had its RETURNING stripped; the driver runs + // a plain UPDATE that returns no rows. The follow-up SELECT + // below produces the returning values. + None + } + ConditionalOutput::None => output_ty.clone(), + }; + + let op: toasty_core::driver::Operation = if stmt.is_insert() { + operation::Insert { stmt, params, ret }.into() + } else { + operation::QuerySql { stmt, params, ret }.into() + }; + + let mut res = self.connection.exec(&self.engine.schema, op).await?; + + match action.conditional { + ConditionalOutput::None => { + if let Some(mysql_update) = mysql_update_returning { + res = self + .run_mysql_update_returning_select(mysql_update, output_ty.clone()) + .await?; + } + } + ConditionalOutput::Count | ConditionalOutput::Returning => { + let rows = collect_conditional_probe(res.values).await?; + let (matched, conditioned) = conditional_probe_counts(&rows[0])?; + + // A conditional write targets a row the caller holds an + // instance of: zero matched rows means it has since been + // deleted. + if matched == 0 { + return Err(toasty_core::Error::record_not_found( + "conditional write matched no rows", + )); + } + if matched != conditioned { + return Err(toasty_core::Error::condition_failed( + "write condition did not match", + )); + } + + res.values = match action.conditional { + ConditionalOutput::Count => Rows::Count(matched as u64), + _ => { + // The probe locked the matched rows, so the write + // applied to exactly those rows and every result row is + // a real changed row — strip the two leading probe + // columns. + let changed = rows + .into_iter() + .map(|row| { + let stmt::Value::Record(record) = row else { + return Err(toasty_core::Error::invalid_result( + "conditional write expected Record", + )); + }; + Ok(stmt::Value::record_from_vec( + record.fields.into_iter().skip(2).collect(), + )) + }) + .collect::>>()?; + + Rows::value_stream(changed) + } + }; + } + } + + // Apply pagination if configured + if let Some(pagination) = &action.pagination { + assert!(res.is_unpaginated()); + res.values.buffer().await?; + self.apply_sql_pagination(&mut res, pagination)?; + } + + Ok(res) + } + + /// Apply SQL pagination by extracting cursor from last row. + /// If we got a full page (page_size rows), extract cursor for potential next page. + /// The client will naturally discover there's no more data when the next request returns empty. + /// + /// The response values must already be buffered (via `Rows::buffer()`). + fn apply_sql_pagination( + &mut self, + res: &mut ExecResponse, + pagination: &PaginationConfig, + ) -> Result<()> { + let Some(extract_cursor) = &pagination.extract_cursor else { + return Ok(()); + }; + + let Rows::Value(stmt::Value::List(ref row_vec)) = res.values else { + return Ok(()); + }; + + let page_size = pagination.page_size as usize; + + // Extract cursors for potential next/prev pages + res.next_cursor = if row_vec.len() == page_size { + let cursor_row = &row_vec[page_size - 1]; + Some(Box::new(extract_cursor.eval( + &self.engine.schema, + std::slice::from_ref(cursor_row), + )?)) + } else { + // Got fewer than page_size rows, no more data + None + }; + + // Extract a previous cursor only when this query started after another page. + res.prev_cursor = if pagination.has_previous_page + && !row_vec.is_empty() + && self.engine.capability().backward_pagination + { + let cursor_row = &row_vec[0]; + Some(Box::new(extract_cursor.eval( + &self.engine.schema, + std::slice::from_ref(cursor_row), + )?)) + } else { + None + }; + + Ok(()) + } +} + +impl Exec<'_> { + /// Detects an UPDATE with a non-empty `RETURNING` on a MySQL backend + /// and rewrites the statement for the workaround path: + /// + /// - The returning clause is stripped from the UPDATE so the SQL + /// serializer doesn't reject it. + /// - An equivalent `SELECT` over the same table + filter is captured, + /// carrying the original returning expression as its projection. + /// + /// Returns `None` when the backend supports `RETURNING` natively (PG, + /// SQLite) or when the statement is not an UPDATE with a returning + /// project. The two-statement path is not atomic relative to concurrent + /// writers — see #881. + pub(super) fn process_stmt_update_with_returning_on_mysql( + &self, + stmt: &mut stmt::Statement, + ) -> Option { + if self.engine.capability().returning_from_update || !self.engine.capability().sql() { + return None; + } + + let stmt::Statement::Update(update) = stmt else { + return None; + }; + + let table_id = match &update.target { + stmt::UpdateTarget::Table(table_id) => *table_id, + _ => return None, + }; + + let returning = update.returning.take()?; + + let select = stmt::Select { + returning, + source: stmt::Source::table(table_id), + filter: update.filter.clone(), + distinct: false, + }; + let select_stmt = + stmt::Statement::Query(stmt::Query::new(stmt::ExprSet::Select(Box::new(select)))); + + Some(MySQLUpdateReturning { select_stmt }) + } + + /// Runs the follow-up `SELECT` for a MySQL UPDATE with stripped + /// `RETURNING`. The driver receives a plain query whose result rows + /// take the place of the original RETURNING output. + pub(super) async fn run_mysql_update_returning_select( + &mut self, + mysql_update: MySQLUpdateReturning, + ret_ty: Option>, + ) -> Result { + let mut select_stmt = mysql_update.select_stmt; + let select_params = self.engine.prepare_for_driver(&mut select_stmt); + + let op = operation::QuerySql { + stmt: select_stmt, + params: select_params, + ret: ret_ty, + }; + + self.connection.exec(&self.engine.schema, op.into()).await + } +} + +/// Collects a conditional write's result rows. The probe (a `COUNT` aggregate) +/// always yields at least one row, so an empty result is a driver bug. +async fn collect_conditional_probe(rows: Rows) -> Result> { + let Rows::Stream(rows) = rows else { + return Err(toasty_core::Error::invalid_result(format!( + "conditional write expected Stream, got {rows:?}" + ))); + }; + + let rows = rows.collect().await?; + if rows.is_empty() { + return Err(toasty_core::Error::invalid_result( + "conditional write probe returned no rows", + )); + } + + Ok(rows) +} + +/// Reads the two leading probe counts (`matched`, `conditioned`) from a +/// conditional write's result row. +fn conditional_probe_counts(row: &stmt::Value) -> Result<(i64, i64)> { + let stmt::Value::Record(record) = row else { + return Err(toasty_core::Error::invalid_result(format!( + "conditional write expected Record, got {row:?}" + ))); + }; + + match (record.fields.first(), record.fields.get(1)) { + (Some(stmt::Value::I64(matched)), Some(stmt::Value::I64(conditioned))) => { + Ok((*matched, *conditioned)) + } + _ => Err(toasty_core::Error::invalid_result(format!( + "conditional write probe columns are not I64; row={row:?}" + ))), + } +} diff --git a/packages/toasty/src/engine/exec/filter.rs b/packages/toasty/src/engine/exec/filter.rs new file mode 100644 index 00000000000..00ba8932b5c --- /dev/null +++ b/packages/toasty/src/engine/exec/filter.rs @@ -0,0 +1,46 @@ +use crate::{ + Result, + engine::{exec::Exec, mir}, +}; +use toasty_core::{ + driver::{ExecResponse, Rows}, + stmt, +}; + +impl Exec<'_> { + pub(super) async fn exec_filter(&mut self, action: &mir::Filter) -> Result { + // Attached args are loaded whether or not any row is read, keeping + // use counts exact. + let args = self.collect_input(action.args.iter().copied()).await?; + + // Load the input variable with metadata + let input_response = self.vars.load(action.input).await?; + let mut input_stream = input_response.values.into_value_stream(); + + // Predicate input: `arg(0)` = current row, `arg(1 + i)` = `args[i]`. + let mut eval_input = Vec::with_capacity(1 + args.len()); + eval_input.push(stmt::Value::Null); + eval_input.extend(args); + + let mut filtered_rows = vec![]; + + // Iterate through the input stream and apply the filter + while let Some(res) = input_stream.next().await { + eval_input[0] = res?; + + if action + .predicate + .eval_bool(&self.engine.schema, &eval_input)? + { + filtered_rows.push(std::mem::replace(&mut eval_input[0], stmt::Value::Null)); + } + } + + // Return the filtered stream with preserved pagination metadata + Ok(ExecResponse { + values: Rows::value_stream(filtered_rows), + next_cursor: input_response.next_cursor, + prev_cursor: input_response.prev_cursor, + }) + } +} diff --git a/packages/toasty/src/engine/exec/find_pk_by_index.rs b/packages/toasty/src/engine/exec/find_pk_by_index.rs new file mode 100644 index 00000000000..f41f142b1b1 --- /dev/null +++ b/packages/toasty/src/engine/exec/find_pk_by_index.rs @@ -0,0 +1,48 @@ +use toasty_core::{ + driver::{ExecResponse, Rows, operation}, + stmt, +}; + +use crate::{ + Result, + engine::{exec::Exec, mir}, +}; + +impl Exec<'_> { + pub(super) async fn exec_find_pk_by_index( + &mut self, + action: &mir::FindPkByIndex, + ) -> Result { + let mut filter = action.filter.clone(); + + if !action.inputs.is_empty() { + assert!(action.inputs.len() == 1); + let input = self.collect_input(action.inputs.iter().copied()).await?; + filter.substitute(&input); + } + + let filters = self.split_filter(filter, action.table); + let mut all_rows = Vec::new(); + + for f in filters { + let res = self + .connection + .exec( + &self.engine.schema, + operation::FindPkByIndex { + table: action.table, + index: action.index, + filter: f, + } + .into(), + ) + .await?; + + all_rows.extend(res.values.into_value_stream().collect().await?); + } + + Ok(ExecResponse::from_rows(Rows::Stream( + stmt::ValueStream::from_vec(all_rows), + ))) + } +} diff --git a/packages/toasty/src/engine/exec/get_by_key.rs b/packages/toasty/src/engine/exec/get_by_key.rs new file mode 100644 index 00000000000..d09d434b0a6 --- /dev/null +++ b/packages/toasty/src/engine/exec/get_by_key.rs @@ -0,0 +1,41 @@ +use crate::{ + Result, + engine::{exec::Exec, mir}, +}; +use toasty_core::{ + driver::{ExecResponse, Rows, operation}, + stmt::{ValueSet, ValueStream}, +}; + +impl Exec<'_> { + pub(super) async fn exec_get_by_key(&mut self, action: &mir::GetByKey) -> Result { + let mut seen = ValueSet::new(); + let keys: Vec<_> = self + .vars + .load(action.input) + .await? + .values + .collect_as_value() + .await? + .into_list_unwrap() + .into_iter() + .filter(|k| !k.is_null()) + .filter(|k| seen.insert(k.clone())) + .collect(); + + let res = if keys.is_empty() { + Rows::value_stream(ValueStream::default()) + } else { + let op = operation::GetByKey { + table: action.table, + select: mir::column_ids(action.table, &action.columns).collect(), + keys, + }; + + let res = self.connection.exec(&self.engine.schema, op.into()).await?; + res.values + }; + + Ok(ExecResponse::from_rows(res)) + } +} diff --git a/packages/toasty/src/engine/exec/kv.rs b/packages/toasty/src/engine/exec/kv.rs new file mode 100644 index 00000000000..5119d324f8a --- /dev/null +++ b/packages/toasty/src/engine/exec/kv.rs @@ -0,0 +1,106 @@ +use toasty_core::{ + driver::Capability, schema::db::TableId, stmt, stmt::ExprContext, stmt::ValueSet, +}; + +use crate::engine::simplify; + +use super::Exec; + +impl Exec<'_> { + /// Split a composite filter into individual key predicates. + /// + /// Recognizes these forms and decomposes them: + /// - `ANY(MAP(Value::List([v1, v2, ...]), pred))` — substitutes each vi + /// into pred + /// - `InList(expr, Value::List([v1, v2, ...]))` — produces `expr == vi` + /// for each value + /// + /// For any other form (including a single equality), simplifies and returns + /// it as a single-element vec (or empty if unsatisfiable). + /// + /// Each returned predicate has been simplified and is guaranteed + /// satisfiable. + pub(super) fn split_filter(&self, filter: stmt::Expr, table: TableId) -> Vec { + let db_table = self.engine.schema.db.table(table); + let cx = self.engine.expr_cx_for(db_table); + + let capability = self.engine.capability; + match filter { + stmt::Expr::Any(any) => Self::split_filter_any_map(*any.expr, cx, capability), + stmt::Expr::InList(in_list) => { + Self::split_filter_in_list(*in_list.expr, *in_list.list, cx, capability) + } + mut other => { + simplify::simplify_expr(cx, self.engine.capability, &mut other); + if other.is_unsatisfiable() { + vec![] + } else { + vec![other] + } + } + } + } + + /// `ANY(MAP(Value::List([v1, v2, ...]), pred))` — substitutes each value + /// into the predicate template. + /// + /// Duplicate values are collapsed: each kv-layer fan-out becomes one + /// driver call per partition key, and a downstream `HashIndex` build + /// over the merged rows requires unique keys. + fn split_filter_any_map( + map_expr: stmt::Expr, + cx: ExprContext<'_>, + capability: &Capability, + ) -> Vec { + let stmt::Expr::Map(map) = map_expr else { + unreachable!() + }; + let stmt::Expr::Value(stmt::Value::List(items)) = *map.base else { + unreachable!() + }; + + let mut seen = ValueSet::with_capacity(items.len()); + items + .into_iter() + .filter(|item| !item.is_null()) + .filter(|item| seen.insert(item.clone())) + .filter_map(|item| { + let mut pred = *map.map.clone(); + // Unpack Record fields so arg(i) binds to field i. + match item { + stmt::Value::Record(r) => pred.substitute(&r.fields[..]), + item => pred.substitute([item]), + } + simplify::simplify_expr(cx, capability, &mut pred); + (!pred.is_unsatisfiable()).then_some(pred) + }) + .collect() + } + + /// `InList(expr, Value::List([v1, v2, ...]))` — produces `expr == vi` for + /// each value. + /// + /// Duplicate values are collapsed; see `split_filter_any_map`. + fn split_filter_in_list( + expr: stmt::Expr, + list: stmt::Expr, + cx: ExprContext<'_>, + capability: &Capability, + ) -> Vec { + let stmt::Expr::Value(stmt::Value::List(values)) = list else { + unreachable!() + }; + + let mut seen = ValueSet::with_capacity(values.len()); + values + .into_iter() + .filter(|v| !v.is_null()) + .filter(|v| seen.insert(v.clone())) + .filter_map(|v| { + let mut pred = stmt::Expr::binary_op(expr.clone(), stmt::BinaryOp::Eq, v); + simplify::simplify_expr(cx, capability, &mut pred); + (!pred.is_unsatisfiable()).then_some(pred) + }) + .collect() + } +} diff --git a/packages/toasty/src/engine/exec/nested_merge.rs b/packages/toasty/src/engine/exec/nested_merge.rs new file mode 100644 index 00000000000..6d50e0eed09 --- /dev/null +++ b/packages/toasty/src/engine/exec/nested_merge.rs @@ -0,0 +1,418 @@ +use toasty_core::driver::{ExecResponse, Rows}; +use toasty_core::stmt; + +use crate::Result; +use crate::engine::eval; +use crate::engine::exec::Exec; +use crate::engine::mir; + +/// Describes one hash index to build over an input before the merge begins. +/// +/// Computed entirely at plan time; execution builds the index mechanically. +#[derive(Debug, Clone)] +pub(crate) struct MergeIndex { + /// Index into `NestedMerge::inputs` — which input to build the index over. + pub(crate) source: usize, + + /// Which field(s) of the child record form the hash key. + pub(crate) child_projections: Vec, +} + +/// A single level in the nesting hierarchy +/// +/// Each level represents one child relationship and contains: +/// - How to load the child data +/// - How to filter it for each parent (can reference ANY ancestor) +/// - How to project it (may include ExprArgs for its own children) +/// - Its own children (recursive nesting) +#[derive(Debug, Clone)] +pub(crate) struct NestedLevel { + /// Input for this level as an index in `NestedMerge::inputs` + pub(crate) source: usize, + + /// Projection for this level (before passing to parent) Argument 0 is the + /// row for the current level, all other arguments are the results of the + /// recursive nested merge. + pub(crate) projection: eval::Func, + + /// This level's children (recursive nesting) + /// Empty for leaf nodes + pub(crate) nested: Vec, +} + +#[derive(Debug, Clone)] +pub(crate) struct NestedChild { + /// The nested level for this child + pub(crate) level: NestedLevel, + + /// How to filter rows to match the parent request + pub(crate) qualification: MergeQualification, + + /// True if single value + pub(crate) single: bool, +} + +/// How to filter nested records for a parent record +#[derive(Debug, Clone)] +pub(crate) enum MergeQualification { + /// Parent selects all records provided from sub statements. + /// + /// This is typically used when the parent query returns only one row, so all + /// nested records end up associated with that single parent record. + All, + + /// Hash index lookup — O(1) per parent row after a one-time O(M) index build. + /// + /// For `single=true` (has_one / belongs_to) relationships only, since + /// `HashIndex` requires unique keys across the child set. + /// + /// `index` is a position into `NestedMerge::hash_indexes`. + /// `lookup_key` evaluates against the ancestor row stack to produce the key. + HashLookup { + /// Position in `NestedMerge::hash_indexes`. + index: usize, + /// Evaluates against the ancestor `RowStack` to produce the lookup key. + lookup_key: eval::Func, + }, + + /// Sorted index lookup — O(log M + k) per parent row after a one-time O(M log M) build. + /// + /// For `single=false` (has_many) relationships. Supports duplicate keys: + /// `find_range(Included(key), Included(key))` returns all matching children. + /// + /// `index` is a position into `NestedMerge::sort_indexes`. + SortLookup { + /// Position in `NestedMerge::sort_indexes`. + index: usize, + /// Evaluates against the ancestor `RowStack` to produce the lookup key. + lookup_key: eval::Func, + }, + + /// General predicate evaluation (nested loop — O(N×M)). + /// + /// Args: [ancestor_stack..., nested_record] -> bool + /// + /// The ancestor stack contains all ancestors from root to immediate parent: + /// [root, child, grandchild, ..., immediate_parent, nested_record] + /// + /// Execution: For each parent record, evaluate predicate against all + /// nested records. No indexing. + Scan(eval::Func), +} + +#[derive(Debug)] +struct RowStack<'a> { + parent: Option<&'a RowStack<'a>>, + row: &'a stmt::Value, + /// Matches `position` from ExprArg + position: usize, +} + +#[derive(Debug)] +struct RowAndNested<'a> { + row: &'a stmt::Value, + nested: &'a [stmt::Value], +} + +#[derive(Debug)] +enum Input { + Count(u64), + Value(Vec), +} + +struct Indices<'a> { + hash: Vec>, + sort: Vec>, +} + +impl Exec<'_> { + /// Combines parent and child data into nested structures. + /// + /// The nested merge algorithm processes hierarchical data by: + /// + /// 1. Loading all batch data upfront - fetches all input data for all levels + /// before processing + /// 2. Building all hash indexes from the pre-computed `indexes` list + /// 3. Processing each root row: + /// - For each nested child relationship at this level: + /// - Filters batch-loaded child data using the qualification (hash lookup + /// or scan predicate) + /// - Recursively merges each matching child row with its own children + /// - Collects results into a list, or a single value if `single` is `true` + /// - Projects the final row by applying the projection function with the + /// current row and all nested children + /// - Adds the projected row to output + /// 4. Returning all merged rows with their nested data + /// + /// # Note + /// + /// Rows loaded from batch queries are not the final projection. They may include + /// extra fields needed for filtering or projecting nested children. + pub(super) async fn exec_nested_merge( + &mut self, + action: &mir::NestedMerge, + ) -> Result { + // Load all input data upfront + let mut inputs = Vec::with_capacity(action.inputs.len()); + let mut next_cursor = None; + let mut prev_cursor = None; + + for (source, node_id) in action.inputs.iter().enumerate() { + let response = self.vars.load(*node_id).await?; + + if source == action.root.source { + next_cursor = response.next_cursor; + prev_cursor = response.prev_cursor; + } + + inputs.push(match response.values { + Rows::Count(count) => Input::Count(count), + Rows::Value(value) => Input::Value(match value { + stmt::Value::List(items) => items, + value => vec![value], + }), + Rows::Stream(value_stream) => Input::Value(value_stream.collect().await?), + }); + } + + // Build all hash and sorted indexes from the plan-time-computed flat lists. + let indices = Indices { + hash: action + .hash_indexes + .iter() + .map(|mi| { + let Input::Value(values) = &inputs[mi.source] else { + panic!("HashLookup source must be a Value input, not Count") + }; + stmt::HashIndex::new(values, &mi.child_projections) + }) + .collect(), + sort: action + .sort_indexes + .iter() + .map(|mi| { + let Input::Value(values) = &inputs[mi.source] else { + panic!("SortLookup source must be a Value input, not Count") + }; + stmt::SortedIndex::new(values, &mi.child_projections) + }) + .collect(), + }; + + // Load the root rows + let mut merged_rows = vec![]; + + match &inputs[action.root.source] { + Input::Count(count) => { + let row_stack = RowStack { + parent: None, + // Bit of a hack + row: &stmt::Value::Null, + position: 0, + }; + + for _ in 0..*count { + merged_rows.push(self.merge_nested_row( + &row_stack, + &action.root, + &inputs, + &indices, + )?); + } + } + Input::Value(root_rows) => { + // Iterate over each record to perform the nested merge + for row in root_rows { + let stack = RowStack { + parent: None, + row, + position: 0, + }; + merged_rows.push(self.merge_nested_row( + &stack, + &action.root, + &inputs, + &indices, + )?); + } + } + } + + Ok(ExecResponse { + values: Rows::value_stream(merged_rows), + next_cursor, + prev_cursor, + }) + } + + /// Recursively merges a single row with its nested child data. + /// + /// This is the core recursive function that processes one row at a time through + /// the nesting hierarchy. For each row, it: + /// + /// 1. **Processes each child relationship**: Iterates through all nested children + /// defined at this level + /// 2. **Filters child rows**: For each child, uses either a pre-built hash index + /// (O(1) lookup) or a linear scan with the qualification predicate + /// 3. **Recursively merges**: For each matching child row, recursively calls + /// `merge_nested_row` to process its own children + /// 4. **Collects results**: Gathers all processed child rows into a list (or + /// single value if `single: true`) + /// 5. **Projects final result**: Applies the projection function with the current + /// row and all nested children as arguments to produce the final output + /// + /// # Arguments + /// + /// * `row_stack` - The ancestor stack containing this row and all parent rows, + /// used for evaluating predicates that reference parent data + /// * `level` - The current nesting level descriptor with source, projection, and children + /// * `inputs` - All batch-loaded data for the entire merge, indexed by source + /// * `indexes` - Pre-built hash indexes, indexed by position in `NestedMerge::indexes` + /// + /// # Returns + /// + /// The projected value for this row, with all nested children merged in according + /// to the projection function. + fn merge_nested_row( + &self, + row_stack: &RowStack<'_>, + level: &NestedLevel, + inputs: &[Input], + indices: &Indices<'_>, + ) -> Result { + // Collected all nested rows for this row. + let mut nested = vec![]; + + for nested_child in &level.nested { + // Find the batch-loaded input + let Input::Value(nested_input) = &inputs[nested_child.level.source] else { + todo!("input={:#?}", inputs[nested_child.level.source]) + }; + let mut nested_rows_projected = vec![]; + + // Process a single matching child row: recurse and collect the result. + let mut process = |nested_row: &stmt::Value| -> Result<()> { + let nested_stack = RowStack { + parent: Some(row_stack), + row: nested_row, + position: row_stack.position + 1, + }; + nested_rows_projected.push(self.merge_nested_row( + &nested_stack, + &nested_child.level, + inputs, + indices, + )?); + Ok(()) + }; + + match &nested_child.qualification { + MergeQualification::All => { + for row in nested_input { + process(row)?; + } + } + MergeQualification::HashLookup { index, lookup_key } => { + let key_val = lookup_key.eval(&self.engine.schema, row_stack)?; + if let Some(row) = indices.hash[*index].find(key_as_slice(&key_val)) { + process(row)?; + } + } + MergeQualification::SortLookup { index, lookup_key } => { + let key_val = lookup_key.eval(&self.engine.schema, row_stack)?; + let key = key_as_slice(&key_val); + for row in indices.sort[*index].find_range( + std::ops::Bound::Included(key), + std::ops::Bound::Included(key), + ) { + process(row)?; + } + } + MergeQualification::Scan(func) => { + for row in nested_input { + let stack = RowStack { + parent: Some(row_stack), + row, + position: row_stack.position + 1, + }; + if func.eval_bool(&self.engine.schema, &stack)? { + process(row)?; + } + } + } + } + + nested.push(if nested_child.single { + assert!(nested_rows_projected.len() <= 1, "TODO: error handling"); + + if let Some(row) = nested_rows_projected.into_iter().next() { + row + } else { + stmt::Value::Null + } + } else { + stmt::Value::List(nested_rows_projected) + }); + } + + // Project the row with the nested data as arguments. + let eval_input = RowAndNested { + row: row_stack.row, + nested: &nested[..], + }; + + level.projection.eval(&self.engine.schema, &eval_input) + } +} + +/// Extract a `&[Value]` key from a parent key value. +/// +/// `Record` values are flattened to their field slice (composite key). +/// All other values are wrapped in a single-element slice (scalar key). +fn key_as_slice(val: &stmt::Value) -> &[stmt::Value] { + match val { + stmt::Value::Record(r) => r.as_slice(), + v => std::slice::from_ref(v), + } +} + +impl stmt::Input for &RowStack<'_> { + fn resolve_arg( + &mut self, + expr_arg: &stmt::ExprArg, + projection: &stmt::Projection, + ) -> Option { + let mut current: &RowStack<'_> = self; + + // Find the stack level that corresponds with the argument. + loop { + if current.position == expr_arg.position { + break; + } + + let Some(parent) = current.parent else { + todo!() + }; + current = parent; + } + + // Get the value and apply projection + Some(current.row.entry(projection).to_expr()) + } +} + +impl stmt::Input for &RowAndNested<'_> { + fn resolve_arg( + &mut self, + expr_arg: &stmt::ExprArg, + projection: &stmt::Projection, + ) -> Option { + let base = if expr_arg.position == 0 { + self.row + } else { + &self.nested[expr_arg.position - 1] + }; + + Some(base.entry(projection).to_expr()) + } +} diff --git a/packages/toasty/src/engine/exec/plan.rs b/packages/toasty/src/engine/exec/plan.rs new file mode 100644 index 00000000000..27e59dfb8e4 --- /dev/null +++ b/packages/toasty/src/engine/exec/plan.rs @@ -0,0 +1,59 @@ +use crate::engine::{ + exec::{If, VarStore}, + mir, +}; + +/// The executable form of a query: the MIR operation graph plus the step +/// sequence that runs it. Steps reference operations by [`mir::NodeId`], +/// which doubles as the runtime variable slot for the operation's output. +#[derive(Debug)] +pub(crate) struct ExecPlan { + /// The operation graph the steps execute. + pub(crate) plan: mir::LogicalPlan, + + /// Runtime storage for node outputs. + pub(crate) vars: VarStore, + + /// Steps in the pipeline. + pub(crate) steps: Vec, + + /// The node whose output the pipeline returns. + pub(crate) returning: mir::NodeId, + + /// When true, the executor wraps the entire plan in a transaction. + pub(crate) needs_transaction: bool, +} + +/// A single step of the exec program. +#[derive(Debug)] +pub(crate) enum Step { + /// Execute one node's operation and store its output. + Run(mir::NodeId), + + /// Conditionally execute a block of pure nodes. + If(If), +} + +impl Step { + /// Returns the step's name for logging. + pub(crate) fn name(&self, plan: &mir::LogicalPlan) -> &'static str { + match self { + Step::Run(node_id) => plan[node_id].op.name(), + Step::If(_) => "if", + } + } + + /// Returns the number of database operations this step issues, counting + /// into `If` arms. The count is static: a skipped `If` arm can leave a + /// transaction wrapping a single executed operation, which is harmless. + pub(crate) fn db_op_count(&self, plan: &mir::LogicalPlan) -> usize { + match self { + Step::Run(node_id) => plan[node_id].op.is_db_op() as usize, + Step::If(action) => action + .then + .iter() + .filter(|node_id| plan[*node_id].op.is_db_op()) + .count(), + } + } +} diff --git a/packages/toasty/src/engine/exec/query_pk.rs b/packages/toasty/src/engine/exec/query_pk.rs new file mode 100644 index 00000000000..eda027d138a --- /dev/null +++ b/packages/toasty/src/engine/exec/query_pk.rs @@ -0,0 +1,70 @@ +use crate::{ + Result, + engine::{exec::Exec, mir}, +}; +use toasty_core::{ + driver::{ExecResponse, Rows, operation}, + stmt, +}; + +impl Exec<'_> { + pub(super) async fn exec_query_pk(&mut self, action: &mir::QueryPk) -> Result { + let mut pk_filter = action.pk_filter.clone(); + + if let Some(input) = action.input { + let input = self.collect_input([input]).await?; + pk_filter.substitute(&input); + } + + let filters = self.split_filter(pk_filter, action.table); + let mut all_rows = Vec::new(); + let mut response_cursor = None; + + // A limit or pagination clause is only meaningful against a single + // partition key query. With multiple filters, each partition call + // would apply the limit/offset independently and produce wrong + // totals (e.g. `.limit(10)` across 3 partitions could return 30 + // rows, each offset skipping within its own partition). + assert!( + action.limit.is_none() || filters.len() <= 1, + "limit/pagination with multiple partition filters is not supported; filters={}", + filters.len() + ); + + // When there are multiple filters, discard the response cursor since it + // would only apply to the last filter's result set. + let paginated = filters.len() <= 1; + + for f in filters { + let res = self + .connection + .exec( + &self.engine.schema, + operation::QueryPk { + table: action.table, + index: action.index, + select: mir::column_ids(action.table, &action.columns).collect(), + pk_filter: f, + filter: action.row_filter.clone(), + limit: action.limit.clone(), + order: action.order, + } + .into(), + ) + .await?; + + // Only capture cursor when paginating a single filter + if paginated && res.next_cursor.is_some() { + response_cursor = res.next_cursor; + } + + all_rows.extend(res.values.into_value_stream().collect().await?); + } + + Ok(ExecResponse { + values: Rows::Stream(stmt::ValueStream::from_vec(all_rows)), + next_cursor: response_cursor, + prev_cursor: None, + }) + } +} diff --git a/packages/toasty/src/engine/exec/repeat.rs b/packages/toasty/src/engine/exec/repeat.rs new file mode 100644 index 00000000000..51ecc671564 --- /dev/null +++ b/packages/toasty/src/engine/exec/repeat.rs @@ -0,0 +1,44 @@ +use crate::{ + Result, + engine::{exec::Exec, mir}, +}; +use toasty_core::{ + driver::{ExecResponse, Rows}, + stmt, +}; + +impl Exec<'_> { + pub(super) async fn exec_repeat(&mut self, action: &mir::Repeat) -> Result { + let ExecResponse { + values, + next_cursor, + prev_cursor, + } = self.vars.load(action.input).await?; + + // Only the input's cardinality is observed; a count-only driver + // response works the same as one that returned rows. + let count = match values { + Rows::Count(count) => count as usize, + Rows::Value(stmt::Value::List(items)) => items.len(), + Rows::Value(value) => todo!("value={value:#?}"), + Rows::Stream(mut value_stream) => { + let mut count = 0; + while let Some(res) = value_stream.next().await { + res?; + count += 1; + } + count + } + }; + + let rows = vec![action.value.clone(); count]; + + // The output has one item for each input row. Its cursors must match + // the input cursors so the caller can fetch the next or previous page. + Ok(ExecResponse { + values: Rows::value_stream(rows), + next_cursor, + prev_cursor, + }) + } +} diff --git a/packages/toasty/src/engine/exec/rmw.rs b/packages/toasty/src/engine/exec/rmw.rs new file mode 100644 index 00000000000..f622de7548e --- /dev/null +++ b/packages/toasty/src/engine/exec/rmw.rs @@ -0,0 +1,202 @@ +use crate::{ + Result, + engine::{exec::Exec, mir}, +}; +use toasty_core::{ + driver::{ + ExecResponse, Rows, + operation::{self, Transaction}, + }, + stmt, +}; + +impl Exec<'_> { + pub(super) async fn exec_read_modify_write( + &mut self, + action: &mir::ReadModifyWrite, + ) -> Result { + assert!(action.inputs.is_empty(), "TODO"); + + // When nested inside an outer transaction use savepoints so the outer + // transaction can still commit or roll back as a whole. When standalone, + // start our own transaction (MySQL requires an active BEGIN before + // SAVEPOINT can be used, so we can't use savepoints here). + let (begin, commit, rollback) = if self.in_transaction { + let name = "read_modify_write"; + ( + Transaction::Savepoint(name.to_owned()), + Transaction::ReleaseSavepoint(name.to_owned()), + Transaction::RollbackToSavepoint(name.to_owned()), + ) + } else { + ( + Transaction::start(), + Transaction::Commit, + Transaction::Rollback, + ) + }; + + self.connection + .exec(&self.engine.schema, begin.into()) + .await?; + + let rows = match self.rmw_exec(action).await { + Ok(rows) => rows, + Err(e) => { + // Best effort: ignore rollback errors so the original error is returned + let _ = self + .connection + .exec(&self.engine.schema, rollback.into()) + .await; + return Err(e); + } + }; + + self.connection + .exec(&self.engine.schema, commit.into()) + .await?; + + Ok(ExecResponse::from_rows(rows)) + } + + /// Execute the core read-then-write logic, returning the write's output + /// rows on success. Errors on a DB failure or a condition mismatch; + /// rollback (savepoint or transaction) is handled by the caller. + /// + /// The returned [`Rows`] mirrors what the equivalent unconditional write + /// would produce: a `Count` when the write has no `RETURNING`, or a + /// buffered row stream when it does. Rows are buffered before the caller + /// commits so nothing is tied to the open transaction. + async fn rmw_exec(&mut self, action: &mir::ReadModifyWrite) -> Result { + // Column types of the write's `RETURNING` rows, or `None` when the + // write has no returning (it then reports only a row count). + let output_ty = mir::row_field_types(&action.ty); + + // The probe projects the condition once per matched row. + let ty = Some(vec![stmt::Type::Bool]); + + let mut read_stmt: stmt::Statement = action.read.clone().into(); + let read_params = self.engine.prepare_for_driver(&mut read_stmt); + + let res = self + .connection + .exec( + &self.engine.schema, + operation::QuerySql { + stmt: read_stmt, + params: read_params, + ret: ty, + } + .into(), + ) + .await?; + + let Rows::Stream(rows) = res.values else { + return Err(toasty_core::Error::invalid_result( + "expected Stream, got Count", + )); + }; + + // `matched` counts rows passing the filter; `satisfied` counts those + // that also meet the condition. The write is safe to apply only when the + // two agree — otherwise some matched row was concurrently advanced. + let rows = rows.collect().await?; + let matched = rows.len() as u64; + let mut satisfied = 0; + for row in &rows { + let stmt::Value::Record(record) = row else { + return Err(toasty_core::Error::invalid_result( + "conditional write probe expected Record", + )); + }; + match record.fields.first() { + Some(stmt::Value::Bool(true)) => satisfied += 1, + Some(stmt::Value::Bool(false)) | Some(stmt::Value::Null) => {} + other => { + return Err(toasty_core::Error::invalid_result(format!( + "conditional write probe expected Bool, got {other:?}" + ))); + } + } + } + + // A conditional write targets a row the caller holds an instance of: + // zero matched rows means it has since been deleted. + if matched == 0 { + return Err(toasty_core::Error::record_not_found( + "conditional write matched no rows", + )); + } + + if matched != satisfied { + return Err(toasty_core::Error::condition_failed( + "write condition did not match", + )); + } + + let count = matched; + let mut write_stmt = action.write.clone(); + + // MySQL lacks `RETURNING` on `UPDATE`; strip the returning into a + // follow-up `SELECT` that runs inside this same transaction. After this, + // `write_stmt` carries a native `RETURNING` only on backends that + // support it (SQLite, PostgreSQL). + let mysql_update = self.process_stmt_update_with_returning_on_mysql(&mut write_stmt); + let native_returning = write_stmt.returning().is_some(); + + let write_params = self.engine.prepare_for_driver(&mut write_stmt); + + let res = self + .connection + .exec( + &self.engine.schema, + operation::QuerySql { + stmt: write_stmt, + params: write_params, + ret: if native_returning { + output_ty.clone() + } else { + None + }, + } + .into(), + ) + .await?; + + if let Some(mysql_update) = mysql_update { + // The UPDATE ran without RETURNING; the captured SELECT produces the + // post-update returning values. + let mut res = self + .run_mysql_update_returning_select(mysql_update, output_ty.clone()) + .await?; + res.values.buffer().await?; + return Ok(res.values); + } + + if native_returning { + // Buffer the returned rows before the caller commits so the stream + // is not tied to the open transaction. + let mut values = res.values; + values.buffer().await?; + return Ok(values); + } + + let Rows::Count(actual) = res.values else { + return Err(toasty_core::Error::invalid_result( + "expected Count, got Stream", + )); + }; + + // The write is supposed to touch exactly the probed rows. On backends + // whose probe cannot lock (`select_for_update` is false), a concurrent + // writer may change the matched set between the probe and the write — + // surface a retryable conflict; the caller rolls back. + if actual != count { + return Err(toasty_core::Error::condition_failed(format!( + "write applied to {actual} rows but the probe matched {count}" + ))); + } + + Ok(Rows::Count(actual)) + } +} diff --git a/packages/toasty/src/engine/exec/scan.rs b/packages/toasty/src/engine/exec/scan.rs new file mode 100644 index 00000000000..025a1617f52 --- /dev/null +++ b/packages/toasty/src/engine/exec/scan.rs @@ -0,0 +1,45 @@ +use crate::{ + Result, + engine::{exec::Exec, mir}, +}; +use toasty_core::{ + driver::{ExecResponse, Rows, operation}, + stmt, +}; + +impl Exec<'_> { + pub(super) async fn exec_scan(&mut self, action: &mir::Scan) -> Result { + let mut row_filter = action.row_filter.clone(); + + if let Some(input) = action.input { + let input = self.collect_input([input]).await?; + if let Some(ref mut f) = row_filter { + f.substitute(&input); + } + } + + let res = self + .connection + .exec( + &self.engine.schema, + operation::Scan { + table: action.table, + columns: mir::column_ids(action.table, &action.columns) + .map(|col_id| col_id.index) + .collect(), + filter: row_filter, + limit: action.limit.clone(), + } + .into(), + ) + .await?; + + let rows: Vec = res.values.into_value_stream().collect().await?; + + Ok(ExecResponse { + values: Rows::Stream(stmt::ValueStream::from_vec(rows)), + next_cursor: res.next_cursor, + prev_cursor: None, + }) + } +} diff --git a/packages/toasty/src/engine/exec/update_by_key.rs b/packages/toasty/src/engine/exec/update_by_key.rs new file mode 100644 index 00000000000..1069d59cb9f --- /dev/null +++ b/packages/toasty/src/engine/exec/update_by_key.rs @@ -0,0 +1,81 @@ +use crate::{ + Result, + engine::{exec::Exec, mir}, +}; +use toasty_core::{ + driver::{ExecResponse, Rows, operation}, + schema::db::ColumnId, + stmt::{self, ValueStream}, +}; + +impl Exec<'_> { + pub(super) async fn exec_update_by_key( + &mut self, + action: &mir::UpdateByKey, + ) -> Result { + // The columns to return for each updated row *after* the update. + // `None` means just return the count of updated rows. + let returning = if action.ty.is_unit() { + None + } else { + Some(mir::column_ids(action.table, &action.columns).collect()) + }; + + let keys = self + .vars + .load(action.input) + .await? + .values + .collect_as_value() + .await? + .into_list_unwrap(); + + // Shred a multi-key update into one single-key op per key so each key's + // filter is adjudicated independently — matching SQL's per-row + // semantics, and mirroring how delete fans out. These updates are not + // atomic. + let mut total_count = 0u64; + let mut rows = vec![]; + + for key in keys { + match self.exec_update_one(action, &returning, key).await? { + Rows::Count(n) => total_count += n, + other => rows.extend(other.into_value_stream().collect().await?), + } + } + + // The output shape is a property of the action, not the results: with + // zero keys there is nothing to match on, yet a `returning` update must + // still yield an (empty) stream rather than a count. + let res = if returning.is_some() { + Rows::value_stream(ValueStream::from_vec(rows)) + } else { + Rows::Count(total_count) + }; + + Ok(ExecResponse::from_rows(res)) + } + + /// Execute a single-key `UpdateByKey` op for one resolved key. + async fn exec_update_one( + &mut self, + action: &mir::UpdateByKey, + returning: &Option>, + key: stmt::Value, + ) -> Result { + let op = operation::UpdateByKey { + table: action.table, + keys: vec![key], + assignments: action.assignments.clone(), + filter: action.filter.clone(), + condition: action.condition.clone(), + returning: returning.clone(), + }; + + let res = self.connection.exec(&self.engine.schema, op.into()).await?; + + debug_assert_eq!(!res.values.is_count(), returning.is_some()); + + Ok(res.values) + } +} diff --git a/packages/toasty/src/engine/exec/upsert.rs b/packages/toasty/src/engine/exec/upsert.rs new file mode 100644 index 00000000000..1af31f97e7c --- /dev/null +++ b/packages/toasty/src/engine/exec/upsert.rs @@ -0,0 +1,30 @@ +use toasty_core::{ + driver::{ExecResponse, operation}, + stmt, +}; + +use crate::{ + Result, + engine::{exec::Exec, mir}, +}; + +impl Exec<'_> { + pub(super) async fn exec_upsert(&mut self, action: &mir::Upsert) -> Result { + let mut stmt = stmt::Statement::from(action.stmt.clone()); + + if !action.inputs.is_empty() { + let input = self.collect_input(action.inputs.iter().copied()).await?; + stmt.substitute(&input); + self.engine.simplify_stmt(&mut stmt); + } + + let params = self.engine.prepare_for_driver(&mut stmt); + let op = operation::Upsert { + stmt: stmt.into_insert_unwrap(), + params, + ret: mir::row_field_types(&action.ty), + }; + + self.connection.exec(&self.engine.schema, op.into()).await + } +} diff --git a/packages/toasty/src/engine/exec/var.rs b/packages/toasty/src/engine/exec/var.rs new file mode 100644 index 00000000000..732e2a7bdf4 --- /dev/null +++ b/packages/toasty/src/engine/exec/var.rs @@ -0,0 +1,220 @@ +use std::sync::Arc; + +use index_vec::IndexVec; +use toasty_core::{ + driver::{ExecResponse, Rows}, + schema::Schema, + stmt, +}; + +use crate::engine::mir::NodeId; + +/// Runtime storage for node outputs: one slot per MIR node, keyed by the +/// node's [`NodeId`]. +#[derive(Debug)] +pub(crate) struct VarStore { + slots: IndexVec>, + /// Resolves `Type::Model` (`#[document]`) layouts for the value type-checks. + schema: Arc, +} + +#[derive(Debug)] +struct Entry { + value: EntryValue, + count: usize, +} + +#[derive(Debug)] +enum EntryValue { + Response(ExecResponse), + Skipped, +} + +impl VarStore { + pub(crate) fn new(node_count: usize, schema: Arc) -> Self { + Self { + slots: IndexVec::from_vec((0..node_count).map(|_| None).collect()), + schema, + } + } + + pub(crate) async fn load(&mut self, node: NodeId) -> crate::Result { + let Some(entry) = &mut self.slots[node] else { + panic!("no stream at slot {node:?}; store={self:#?}") + }; + + if matches!(&entry.value, EntryValue::Skipped) { + panic!("load of skipped slot {node:?}") + } + + if entry.count == 1 { + let entry = self.slots[node].take().unwrap(); + let EntryValue::Response(response) = entry.value else { + unreachable!() + }; + return Ok(response); + } + + entry.count -= 1; + let EntryValue::Response(response) = &mut entry.value else { + unreachable!() + }; + Ok(ExecResponse { + values: response.values.dup().await?, + next_cursor: response.next_cursor.clone(), + prev_cursor: response.prev_cursor.clone(), + }) + } + + /// Decrements a slot's use count without observing its value, dropping + /// the entry at zero. Called on paths that decline a load the use + /// counting expects (a skipped `If` arm). + #[track_caller] + pub(crate) fn release(&mut self, node: NodeId) { + let Some(entry) = self.slots[node].as_mut() else { + panic!("release of unset slot {node:?}; store={self:#?}") + }; + + if entry.count == 1 { + self.slots[node] = None; + } else { + entry.count -= 1; + } + } + + /// Returns whether the slot holds at least one row, without consuming a + /// use. A skipped slot is empty, allowing a guard to depend on a node that + /// an earlier guard skipped. A stream-backed slot is buffered in place so + /// the peek does not disturb later loads. + pub(crate) async fn peek_non_empty(&mut self, node: NodeId) -> crate::Result { + let Some(entry) = self.slots[node].as_mut() else { + panic!("no stream at slot {node:?}; store={self:#?}") + }; + + let EntryValue::Response(response) = &mut entry.value else { + return Ok(false); + }; + + response.values.buffer().await?; + + Ok(match &response.values { + Rows::Count(count) => *count > 0, + Rows::Value(stmt::Value::List(items)) => !items.is_empty(), + Rows::Value(stmt::Value::Null) => false, + Rows::Value(_) => true, + Rows::Stream(_) => unreachable!("stream was buffered above"), + }) + } + + /// Debug-asserts every slot has been drained. With exact use counts this + /// holds after the final load of the plan's returning variable, on the + /// success path only — a mid-plan failure legitimately leaves loads + /// unperformed. Undercounting already panics loudly on a load of a freed + /// slot; this converts the silent overcounting direction into a loud one. + pub(crate) fn assert_empty(&self) { + debug_assert!( + self.slots.iter().all(Option::is_none), + "variable slots not drained at plan completion; store={self:#?}" + ); + } + + /// Marks an escaping output from a skipped `If` arm. The slot participates + /// in use counting but has no value and cannot be loaded. + pub(crate) fn store_skipped(&mut self, node: NodeId, count: usize) { + if count == 0 { + return; + } + + self.slots[node] = Some(Entry { + value: EntryValue::Skipped, + count, + }); + } + + #[track_caller] + pub(crate) fn store( + &mut self, + node: NodeId, + ty: &stmt::Type, + count: usize, + response: ExecResponse, + ) { + // A zero-use output is never observed; don't occupy a slot. + if count == 0 { + return; + } + + let values = match response.values { + Rows::Count(_) => { + assert!(ty.is_unit()); + response.values + } + Rows::Value(value) => { + assert!( + value.is_a(&self.schema.app, ty), + "type mismatch: {value:?} is not a {ty:?}", + ); + Rows::Value(value) + } + Rows::Stream(value_stream) => { + let stmt::Type::List(item_tys) = ty else { + todo!("ty={ty:#?}") + }; + let item_ty = (**item_tys).clone(); + + Rows::Stream(value_stream.typed(self.schema.clone(), item_ty)) + } + }; + let response = ExecResponse { + values, + next_cursor: response.next_cursor, + prev_cursor: response.prev_cursor, + }; + + self.slots[node] = Some(Entry { + value: EntryValue::Response(response), + count, + }); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::engine::test_util::test_schema; + + #[tokio::test] + #[should_panic(expected = "load of skipped slot")] + async fn load_skipped_panics() { + let node = NodeId::new(0); + let mut store = VarStore::new(1, Arc::new(test_schema())); + store.store_skipped(node, 1); + + store.load(node).await.unwrap(); + } + + #[tokio::test] + async fn peek_skipped_returns_false_without_consuming_use() { + let node = NodeId::new(0); + let mut store = VarStore::new(1, Arc::new(test_schema())); + store.store_skipped(node, 1); + + assert!(!store.peek_non_empty(node).await.unwrap()); + + store.release(node); + assert!(store.slots[node].is_none()); + } + + #[test] + fn release_drains_skipped_slot() { + let node = NodeId::new(0); + let mut store = VarStore::new(1, Arc::new(test_schema())); + store.store_skipped(node, 2); + + store.release(node); + assert!(store.slots[node].is_some()); + + store.release(node); + assert!(store.slots[node].is_none()); + } +} diff --git a/packages/toasty/src/engine/fold.rs b/packages/toasty/src/engine/fold.rs new file mode 100644 index 00000000000..462856dcea4 --- /dev/null +++ b/packages/toasty/src/engine/fold.rs @@ -0,0 +1,102 @@ +//! Cheap, schema-free, idempotent rewrites. +//! +//! `fold` is the canonicalization pass shared between `lower`, `simplify`, and +//! `exec_statement`. Each rule is local, O(n) per pass, and produces output +//! that satisfies `fold(fold(x)) == fold(x)`. Rules pattern-match on local +//! structure only; no schema lookups. +//! +//! See `docs/dev/design/lower-then-simplify.md` for the full taxonomy. + +mod expr_and; +mod expr_binary_op; +mod expr_cast; +mod expr_in_list; +mod expr_intersects; +mod expr_is_null; +mod expr_is_superset; +mod expr_let; +mod expr_list; +mod expr_match; +mod expr_not; +mod expr_or; +mod expr_record; + +#[cfg(test)] +mod tests; + +use toasty_core::stmt::{self, Expr, Node, VisitMut}; + +/// Folds an expression tree in place: canonicalizes structure, evaluates +/// constants, propagates nulls, drops boolean units. Idempotent. +pub(crate) fn fold_stmt(stmt: &mut T) { + Fold.visit_mut(stmt); +} + +/// Visitor that applies fold rules bottom-up. +struct Fold; + +impl VisitMut for Fold { + fn visit_expr_mut(&mut self, i: &mut Expr) { + // Recurse into children first so parent rules see canonical input. + stmt::visit_mut::visit_expr_mut(self, i); + + // A single rule's output may itself be fold-eligible (e.g. + // `not(eq(x, y))` becomes `ne(x, y)`, which then constant-folds + // to a literal when both sides are values, or De Morgan creates a + // new `Or` of `Not`s that need their own pass). Recurse on the + // replacement so the fixed point is reached in one traversal. + if let Some(mut expr) = fold_one(i) { + self.visit_expr_mut(&mut expr); + *i = expr; + } + } +} + +/// Apply one round of fold rules to `i`. Returns `Some` if a rule fired and +/// produced a replacement expression; returns `None` if no rule applied or +/// the rule mutated `i` in place. +fn fold_one(i: &mut Expr) -> Option { + match i { + Expr::And(expr) => expr_and::fold_expr_and(expr), + Expr::BinaryOp(expr) => { + expr_binary_op::fold_expr_binary_op(expr.op, &mut expr.lhs, &mut expr.rhs) + } + Expr::Cast(expr) => expr_cast::fold_expr_cast(expr), + Expr::InList(expr) => expr_in_list::fold_expr_in_list(expr), + Expr::Intersects(expr) => expr_intersects::fold_expr_intersects(expr), + Expr::IsNull(expr) => expr_is_null::fold_expr_is_null(expr), + Expr::IsSuperset(expr) => expr_is_superset::fold_expr_is_superset(expr), + Expr::Let(expr) => expr_let::fold_expr_let(expr), + Expr::List(expr) => expr_list::fold_expr_list(expr), + Expr::Match(expr) => expr_match::fold_expr_match(expr), + Expr::Not(expr) => expr_not::fold_expr_not(expr), + Expr::Or(expr) => expr_or::fold_expr_or(expr), + Expr::Record(expr) => expr_record::fold_expr_record(expr), + // No fold rules yet — leaves and not-yet-migrated variants pass + // through untouched. Listed explicitly so adding a new `Expr` + // variant forces a decision about whether it has cheap rewrites. + Expr::AllOp(_) + | Expr::Any(_) + | Expr::AnyOp(_) + | Expr::Arg(_) + | Expr::Between(_) + | Expr::Default + | Expr::Error(_) + | Expr::Exists(_) + | Expr::Func(_) + | Expr::Ident(_) + | Expr::InSubquery(_) + | Expr::Incoming(_) + | Expr::IsVariant(_) + | Expr::Length(_) + | Expr::Like(_) + | Expr::Map(_) + | Expr::Project(_) + | Expr::Reference(_) + | Expr::StartsWith(_) + | Expr::Static(_) + | Expr::Stmt(_) + | Expr::Value(_) + | Expr::Variant(_) => None, + } +} diff --git a/packages/toasty/src/engine/fold/expr_and.rs b/packages/toasty/src/engine/fold/expr_and.rs new file mode 100644 index 00000000000..4f673583b16 --- /dev/null +++ b/packages/toasty/src/engine/fold/expr_and.rs @@ -0,0 +1,42 @@ +use std::mem; +use toasty_core::stmt::{self, Expr}; + +/// Cheap canonicalization for AND: flatten nested ANDs, short-circuit on +/// `false`, drop `true`, propagate all-null, collapse single/empty. +/// +/// Heavyweight rules (idempotent law, absorption, complement, range-to-eq, +/// contradiction, OR-branch pruning) live in `simplify/expr_and.rs` and run +/// after this fold pass on canonical input. +pub(super) fn fold_expr_and(expr: &mut stmt::ExprAnd) -> Option { + // Flatten nested ANDs in place. Marker `true` literals are dropped by + // the next step. + for i in 0..expr.operands.len() { + if let Expr::And(and) = &mut expr.operands[i] { + let mut nested = mem::take(&mut and.operands); + expr.operands[i] = true.into(); + expr.operands.append(&mut nested); + } + } + + // `and(..., false, ...)` → `false` + if expr.operands.iter().any(|e| e.is_false()) { + return Some(false.into()); + } + + // `and(..., true, ...)` → `and(..., ...)` + expr.operands.retain(|operand| !operand.is_true()); + + // Null propagation: all remaining operands are null literals. + if !expr.operands.is_empty() && expr.operands.iter().all(|e| e.is_value_null()) { + return Some(Expr::null()); + } + + // Empty/single-operand collapse. + if expr.operands.is_empty() { + Some(true.into()) + } else if expr.operands.len() == 1 { + Some(expr.operands.remove(0)) + } else { + None + } +} diff --git a/packages/toasty/src/engine/fold/expr_binary_op.rs b/packages/toasty/src/engine/fold/expr_binary_op.rs new file mode 100644 index 00000000000..96e3466fbce --- /dev/null +++ b/packages/toasty/src/engine/fold/expr_binary_op.rs @@ -0,0 +1,75 @@ +use std::cmp::PartialOrd; +use toasty_core::stmt::{self, BinaryOp, Expr}; + +/// Cheap canonicalization for binary ops: constant folding, null +/// propagation, boolean-constant simplification, literal-on-right swap. +/// +/// Heavyweight rules (self-comparison with field nullability, tuple +/// decomposition, match elimination, derived-column null check, relation +/// path lifting) live in `simplify/expr_binary_op.rs` and run after this +/// fold pass on canonical input. +pub(super) fn fold_expr_binary_op(op: BinaryOp, lhs: &mut Expr, rhs: &mut Expr) -> Option { + match (&mut *lhs, &mut *rhs) { + // Constant folding and null propagation: + // + // - `5 = 5` → `true` + // - `1 < 5` → `true` + // - `"a" >= "b"` → `false` + // - `null x` → `null` + // - `x null` → `null` + (Expr::Value(lhs_val), Expr::Value(rhs_val)) => { + if lhs_val.is_null() || rhs_val.is_null() { + return Some(Expr::null()); + } + + match op { + BinaryOp::Eq => Some((*lhs_val == *rhs_val).into()), + BinaryOp::Ne => Some((*lhs_val != *rhs_val).into()), + BinaryOp::Lt => { + PartialOrd::partial_cmp(&*lhs_val, &*rhs_val).map(|o| o.is_lt().into()) + } + BinaryOp::Le => { + PartialOrd::partial_cmp(&*lhs_val, &*rhs_val).map(|o| o.is_le().into()) + } + BinaryOp::Gt => { + PartialOrd::partial_cmp(&*lhs_val, &*rhs_val).map(|o| o.is_gt().into()) + } + BinaryOp::Ge => { + PartialOrd::partial_cmp(&*lhs_val, &*rhs_val).map(|o| o.is_ge().into()) + } + BinaryOp::Add => lhs_val.checked_add(rhs_val).map(Expr::from), + BinaryOp::Sub => lhs_val.checked_sub(rhs_val).map(Expr::from), + } + } + // Boolean constant comparisons: + // + // - `x = true` → `x` + // - `x = false` → `not(x)` + // - `x != true` → `not(x)` + // - `x != false` → `x` + (expr, Expr::Value(stmt::Value::Bool(b))) | (Expr::Value(stmt::Value::Bool(b)), expr) + if op.is_eq() || op.is_ne() => + { + let is_eq_true = (op.is_eq() && *b) || (op.is_ne() && !*b); + if is_eq_true { + Some(expr.take()) + } else { + Some(Expr::not(expr.take())) + } + } + // Null propagation: `expr null` → `null` (and symmetric). + // SQL three-valued logic: any comparison with NULL yields NULL. + (_, Expr::Value(stmt::Value::Null)) | (Expr::Value(stmt::Value::Null), _) => { + Some(Expr::null()) + } + // Canonicalization, `literal col` → `col literal`. + // Heavyweight rules can then assume the literal (when present) is on + // the right. Non-commutative ops (e.g. `Sub`) are left in place. + (Expr::Value(_), rhs) if !rhs.is_value() => { + let commuted = op.commute()?; + std::mem::swap(lhs, rhs); + Some(Expr::binary_op(lhs.take(), commuted, rhs.take())) + } + _ => None, + } +} diff --git a/packages/toasty/src/engine/fold/expr_cast.rs b/packages/toasty/src/engine/fold/expr_cast.rs new file mode 100644 index 00000000000..035f12da7da --- /dev/null +++ b/packages/toasty/src/engine/fold/expr_cast.rs @@ -0,0 +1,29 @@ +use toasty_core::stmt::{self, Expr}; + +/// Cheap canonicalization for `CAST`: constant-fold the cast at compile +/// time when the operand is a literal value. +/// +/// Schema-directed casts — a `#[document]` embed's record ↔ object +/// conversions, marked by a source type or a model-typed target — are +/// skipped: fold is schema-free, so they constant-fold in the simplifier +/// instead (`simplify/expr_cast.rs`), which also owns the other heavyweight +/// rewrites (redundant-cast elimination on field references). +pub(super) fn fold_expr_cast(expr: &mut stmt::ExprCast) -> Option { + // `DEFAULT` is a SQL sentinel rather than a typed value. A storage bridge + // may wrap it when an auto-increment column's database type differs from + // its application type, but the cast must not reach SQL serialization. + if matches!(*expr.expr, stmt::Expr::Default) { + return Some(stmt::Expr::Default); + } + + if expr.from.is_some() || expr.ty.contains_model() { + return None; + } + + let stmt::Expr::Value(value) = &mut *expr.expr else { + return None; + }; + + let cast = expr.ty.cast(&(), value.take()).unwrap(); + Some(cast.into()) +} diff --git a/packages/toasty/src/engine/fold/expr_in_list.rs b/packages/toasty/src/engine/fold/expr_in_list.rs new file mode 100644 index 00000000000..de45cef8142 --- /dev/null +++ b/packages/toasty/src/engine/fold/expr_in_list.rs @@ -0,0 +1,58 @@ +use toasty_core::stmt::{self, Expr, Value, ValueSet}; + +/// Cheap canonicalization for `x IN (..)`: empty list → false, null +/// propagation, literal dedup, single-item collapse. +/// +/// Heavyweight rewrites (model-reference → primary-key-reference) live in +/// `simplify/expr_in_list.rs` and run after this fold pass on canonical +/// input. +pub(super) fn fold_expr_in_list(expr: &mut stmt::ExprInList) -> Option { + // `x in ()` → `false` + if expr.list.is_list_empty() { + return Some(Expr::Value(Value::Bool(false))); + } + + // Null propagation, `null in (x, y, z)` → `null` + if expr.expr.is_value_null() { + return Some(Expr::null()); + } + + // Deduplicate literal value lists: `x in (1, 1, 2)` → `x in (1, 2)`. + // Lists carry literal `Value`s only (no symbolic expressions), so value + // equality is safe — there is no non-determinism to preserve. + if let Expr::Value(Value::List(values)) = &mut *expr.list { + let mut seen = ValueSet::new(); + values.retain(|v| seen.insert(v.clone())); + } + + // Rewrite single-item lists into equalities. + rewrite_expr_in_list_with_single_item(expr) +} + +fn rewrite_expr_in_list_with_single_item(expr: &mut stmt::ExprInList) -> Option { + let rhs = match &mut *expr.list { + Expr::Value(value) => { + let values = match value { + Value::List(value) => &value[..], + _ => todo!("{value:#?}"), + }; + + if values.len() != 1 { + return None; + } + + Expr::Value(values[0].clone()) + } + Expr::List(expr_list) => { + if expr_list.items.len() != 1 { + return None; + } + + expr_list.items[0].take() + } + Expr::Record(_) => todo!("should not happen"), + _ => return None, + }; + + Some(Expr::eq(expr.expr.take(), rhs)) +} diff --git a/packages/toasty/src/engine/fold/expr_intersects.rs b/packages/toasty/src/engine/fold/expr_intersects.rs new file mode 100644 index 00000000000..5c12bcd06cd --- /dev/null +++ b/packages/toasty/src/engine/fold/expr_intersects.rs @@ -0,0 +1,13 @@ +use toasty_core::stmt::{self, Expr, Value}; + +/// Cheap canonicalization for `intersects(lhs, rhs)`: an empty rhs makes +/// the predicate vacuously false (no collection intersects the empty +/// set), regardless of `lhs`. The rhs is the user-supplied set literal — +/// `tags.intersects(values)` — so an empty list here means the caller +/// passed `[]`. +pub(super) fn fold_expr_intersects(expr: &mut stmt::ExprIntersects) -> Option { + if expr.rhs.is_list_empty() { + return Some(Expr::Value(Value::Bool(false))); + } + None +} diff --git a/packages/toasty/src/engine/fold/expr_is_null.rs b/packages/toasty/src/engine/fold/expr_is_null.rs new file mode 100644 index 00000000000..92c84186cb0 --- /dev/null +++ b/packages/toasty/src/engine/fold/expr_is_null.rs @@ -0,0 +1,28 @@ +use toasty_core::stmt::{self, Expr}; + +/// Cheap canonicalization for `IS NULL`: constant folding, cast stripping. +/// +/// Heavyweight rewrites (non-nullable field → false) live in +/// `simplify/expr_is_null.rs` and run after this fold pass on canonical +/// input. +pub(super) fn fold_expr_is_null(expr: &mut stmt::ExprIsNull) -> Option { + match &mut *expr.expr { + // Null constant folding: + // - `null is null` → `true` + // - ` is null` → `false` + stmt::Expr::Value(value) => Some((value.is_null() != expr.negated).into()), + // A constructed record or list is structurally non-null, so + // `is_null(record)` → `false`. This fires when an `Option` + // encode guard `is_not_null(field)` sees a `Some` value: the embed + // lowers to an `Expr::Record` of its columns, so the guard subject + // folds to `is_not_null(Record(..))` → `true`. + stmt::Expr::Record(_) | stmt::Expr::List(_) => Some(expr.negated.into()), + // Strip type casts: `is_null(cast(x, T))` → `is_null(x)`. + // Nullity is type-independent so the cast is unnecessary. + stmt::Expr::Cast(expr_cast) => { + *expr.expr = expr_cast.expr.take(); + None + } + _ => None, + } +} diff --git a/packages/toasty/src/engine/fold/expr_is_superset.rs b/packages/toasty/src/engine/fold/expr_is_superset.rs new file mode 100644 index 00000000000..b5680bbec51 --- /dev/null +++ b/packages/toasty/src/engine/fold/expr_is_superset.rs @@ -0,0 +1,13 @@ +use toasty_core::stmt::{self, Expr, Value}; + +/// Cheap canonicalization for `is_superset(lhs, rhs)`: an empty rhs makes +/// the predicate vacuously true (every collection is a superset of the +/// empty set), regardless of `lhs`. The rhs is the user-supplied set +/// literal — `tags.is_superset(values)` — so an empty list here means +/// the caller passed `[]`. +pub(super) fn fold_expr_is_superset(expr: &mut stmt::ExprIsSuperset) -> Option { + if expr.rhs.is_list_empty() { + return Some(Expr::Value(Value::Bool(true))); + } + None +} diff --git a/packages/toasty/src/engine/fold/expr_let.rs b/packages/toasty/src/engine/fold/expr_let.rs new file mode 100644 index 00000000000..2728d927258 --- /dev/null +++ b/packages/toasty/src/engine/fold/expr_let.rs @@ -0,0 +1,45 @@ +use toasty_core::stmt::{self, Expr, visit_mut}; + +/// Cheap canonicalization for `Let`: inline a `Let` whose bindings are all +/// stable by substituting each `Arg(position=i, nesting=0)` in the body +/// with `bindings[i]`. +/// +/// A binding is *stable* when it is fully resolved with no remaining +/// `Stmt` nodes. After lowering converts `Stmt → Arg`, the bindings +/// become stable and the `Let` collapses; the same shape arises at exec +/// time once bind-value substitution has replaced `Arg` with `Value`. +/// +/// Schema-free and idempotent: a `Let` whose bindings are not all stable +/// is left alone, and inlining produces a body that contains no +/// references to the now-removed `Let` scope. +pub(super) fn fold_expr_let(expr_let: &mut stmt::ExprLet) -> Option { + if !expr_let.bindings.iter().all(|b| b.is_stable()) { + return None; + } + + let mut body = *expr_let.body.clone(); + substitute_let_bindings(&mut body, &expr_let.bindings); + Some(body) +} + +/// Substitute `Let` bindings into a body expression. +/// +/// - `Arg(position=i, nesting==depth)` where `i < bindings.len()` is replaced +/// with `bindings[i].clone()`. +/// - `Arg(nesting > depth)` gets `nesting -= 1` (the `Let` scope is removed). +/// +/// Uses `walk_expr_scoped_mut` to track scope depth through `Let`/`Map` +/// scopes automatically. +fn substitute_let_bindings(expr: &mut Expr, bindings: &[Expr]) { + visit_mut::walk_expr_scoped_mut(expr, 0, |expr, scope_depth| match expr { + Expr::Arg(arg) if arg.nesting == scope_depth && arg.position < bindings.len() => { + *expr = bindings[arg.position].clone(); + false + } + Expr::Arg(arg) if arg.nesting > scope_depth => { + arg.nesting -= 1; + false + } + _ => true, + }); +} diff --git a/packages/toasty/src/engine/fold/expr_list.rs b/packages/toasty/src/engine/fold/expr_list.rs new file mode 100644 index 00000000000..5d2097fa48e --- /dev/null +++ b/packages/toasty/src/engine/fold/expr_list.rs @@ -0,0 +1,20 @@ +use toasty_core::stmt::{self, Expr}; + +/// Cheap canonicalization for `List`: when every item is a literal +/// `Value`, collapse the list to a `Value::List`. +/// +/// Heavyweight rewrites (merging single-row Insert statements into a +/// batch Insert) live in `simplify/expr_list.rs` and run after this +/// fold pass on canonical input. +pub(super) fn fold_expr_list(expr: &mut stmt::ExprList) -> Option { + let values = expr + .items + .iter() + .map(|item| match item { + stmt::Expr::Value(value) => Some(value.clone()), + _ => None, + }) + .collect::>>()?; + + Some(stmt::Value::list_from_vec(values).into()) +} diff --git a/packages/toasty/src/engine/fold/expr_match.rs b/packages/toasty/src/engine/fold/expr_match.rs new file mode 100644 index 00000000000..70571b153dd --- /dev/null +++ b/packages/toasty/src/engine/fold/expr_match.rs @@ -0,0 +1,44 @@ +use toasty_core::stmt::{self, Expr}; + +/// Cheap canonicalization for `Match`: constant-subject arm selection and +/// uniform-arms collapse. Both rules are local and schema-free. +/// +/// Heavyweight `Match` rewrites (e.g., distributing projections into arms) live +/// in [`simplify::Simplify::simplify_expr_match`] (none currently). +/// +/// Note: `simplify::Simplify` overrides `visit_expr_match_mut` to skip +/// dead-code arms when the subject is constant. That override prevents +/// downstream rules (notably `simplify_expr_project`) from panicking on +/// invalid sub-expressions in unreachable arms. Fold has no such override +/// because it has no rules that can panic on invalid input — every fold rule +/// matches structurally and bails to `None` on unexpected shape. +pub(super) fn fold_expr_match(expr: &mut stmt::ExprMatch) -> Option { + // Constant-subject arm selection, + // + // - `match v { p1 => e1, ..., pN => eN } else => e0` with `v` constant + // and `v == pI` → `eI` + // - `match v { p1 => e1, ..., pN => eN } else => e0` with `v` constant + // and no `pI == v` → `e0` + if let Expr::Value(value) = expr.subject.as_ref() { + for arm in &expr.arms { + if value == &arm.pattern { + return Some(arm.expr.clone()); + } + } + return Some(expr.else_expr.as_ref().clone()); + } + + // Uniform arms, + // + // - `match v { p1 => e, ..., pN => e } else => e` → `e` + // + // When every arm body and the else branch produce the same expression, + // the match is independent of the subject and reduces to that expression. + let first = expr.arms.first()?; + let all_arms_match = expr.arms.iter().all(|arm| arm.expr == first.expr); + let else_matches = expr.else_expr.as_ref() == &first.expr; + match () { + () if all_arms_match && else_matches => Some(first.expr.clone()), + () => None, + } +} diff --git a/packages/toasty/src/engine/fold/expr_not.rs b/packages/toasty/src/engine/fold/expr_not.rs new file mode 100644 index 00000000000..d8c33cd3a60 --- /dev/null +++ b/packages/toasty/src/engine/fold/expr_not.rs @@ -0,0 +1,65 @@ +use toasty_core::stmt::{self, Expr}; + +/// Cheap canonicalization for `NOT`: double-negation elimination, constant +/// folding, comparison negation, De Morgan's laws, NOT-IN-empty. +/// +/// All `NOT` rules are local and schema-free; there is no heavyweight +/// counterpart in `simplify`. +pub(super) fn fold_expr_not(expr_not: &mut stmt::ExprNot) -> Option { + // Double negation elimination, `not(not(x))` → `x` + if let Expr::Not(inner) = expr_not.expr.as_mut() { + return Some(inner.expr.take()); + } + + // Constant folding, + // + // - `not(true)` → `false` + // - `not(false)` → `true` + // - `not(null)` → `null` + match expr_not.expr.as_ref() { + Expr::Value(stmt::Value::Bool(b)) => { + return Some(Expr::Value(stmt::Value::Bool(!b))); + } + Expr::Value(stmt::Value::Null) => { + return Some(Expr::null()); + } + _ => {} + } + + // Negation of comparisons, `not(x = y)` → `x != y`, etc. + if let Expr::BinaryOp(binary_op) = expr_not.expr.as_mut() + && let Some(negated_op) = binary_op.op.negate() + { + binary_op.op = negated_op; + return Some(expr_not.expr.take()); + } + + // De Morgan's law, `not(a and b)` → `not(a) or not(b)` + if let Expr::And(expr_and) = expr_not.expr.as_mut() { + let negated = expr_and + .operands + .drain(..) + .map(Expr::not) + .collect::>(); + return Some(Expr::or_from_vec(negated)); + } + + // De Morgan's law, `not(a or b)` → `not(a) and not(b)` + if let Expr::Or(expr_or) = expr_not.expr.as_mut() { + let negated = expr_or + .operands + .drain(..) + .map(Expr::not) + .collect::>(); + return Some(Expr::and_from_vec(negated)); + } + + // `not(x in ())` → `true` (x NOT IN empty list is always true) + if let Expr::InList(expr_in_list) = expr_not.expr.as_ref() + && expr_in_list.list.is_list_empty() + { + return Some(true.into()); + } + + None +} diff --git a/packages/toasty/src/engine/fold/expr_or.rs b/packages/toasty/src/engine/fold/expr_or.rs new file mode 100644 index 00000000000..f2b797c8661 --- /dev/null +++ b/packages/toasty/src/engine/fold/expr_or.rs @@ -0,0 +1,42 @@ +use std::mem; +use toasty_core::stmt::{self, Expr}; + +/// Cheap canonicalization for OR: flatten nested ORs, short-circuit on +/// `true`, drop `false`, propagate all-null, collapse single/empty. +/// +/// Heavyweight rules (idempotent law, absorption, factoring, complement, +/// variant tautology, OR-to-IN) live in `simplify/expr_or.rs` and run +/// after this fold pass on canonical input. +pub(super) fn fold_expr_or(expr: &mut stmt::ExprOr) -> Option { + // Flatten nested ORs in place. Marker `false` literals are dropped by + // the next step. + for i in 0..expr.operands.len() { + if let Expr::Or(or) = &mut expr.operands[i] { + let mut nested = mem::take(&mut or.operands); + expr.operands[i] = false.into(); + expr.operands.append(&mut nested); + } + } + + // `or(..., true, ...)` → `true` + if expr.operands.iter().any(|e| e.is_true()) { + return Some(true.into()); + } + + // `or(..., false, ...)` → `or(..., ...)` + expr.operands.retain(|operand| !operand.is_false()); + + // Null propagation: all remaining operands are null literals. + if !expr.operands.is_empty() && expr.operands.iter().all(|e| e.is_value_null()) { + return Some(Expr::null()); + } + + // Empty/single-operand collapse. + if expr.operands.is_empty() { + Some(false.into()) + } else if expr.operands.len() == 1 { + Some(expr.operands.remove(0)) + } else { + None + } +} diff --git a/packages/toasty/src/engine/fold/expr_record.rs b/packages/toasty/src/engine/fold/expr_record.rs new file mode 100644 index 00000000000..c64ee8877a7 --- /dev/null +++ b/packages/toasty/src/engine/fold/expr_record.rs @@ -0,0 +1,19 @@ +use toasty_core::stmt::{self, Expr}; + +/// Cheap canonicalization for `Record`: when every field is a literal +/// `Value`, collapse the record to a `Value::Record`. +/// +/// All `Record` rules are local and schema-free; there is no heavyweight +/// counterpart in `simplify`. +pub(super) fn fold_expr_record(expr: &mut stmt::ExprRecord) -> Option { + let values = expr + .fields + .iter() + .map(|field| match field { + stmt::Expr::Value(value) => Some(value.clone()), + _ => None, + }) + .collect::>>()?; + + Some(stmt::Value::record_from_vec(values).into()) +} diff --git a/packages/toasty/src/engine/fold/tests.rs b/packages/toasty/src/engine/fold/tests.rs new file mode 100644 index 00000000000..4f75a41fc94 --- /dev/null +++ b/packages/toasty/src/engine/fold/tests.rs @@ -0,0 +1,14 @@ +mod expr_and; +mod expr_binary_op; +mod expr_cast; +mod expr_in_list; +mod expr_intersects; +mod expr_is_null; +mod expr_is_superset; +mod expr_let; +mod expr_list; +mod expr_match; +mod expr_not; +mod expr_or; +mod expr_record; +mod prop_idempotence; diff --git a/packages/toasty/src/engine/fold/tests/expr_and.rs b/packages/toasty/src/engine/fold/tests/expr_and.rs new file mode 100644 index 00000000000..1040c88d2b2 --- /dev/null +++ b/packages/toasty/src/engine/fold/tests/expr_and.rs @@ -0,0 +1,285 @@ +use crate::engine::fold::expr_and::fold_expr_and; +use toasty_core::stmt::{Expr, ExprAnd}; + +/// Builds `and(a, and(b, c))`, a nested AND structure for testing flattening. +fn nested_and(a: Expr, b: Expr, c: Expr) -> ExprAnd { + ExprAnd { + operands: vec![ + a, + Expr::And(ExprAnd { + operands: vec![b, c], + }), + ], + } +} + +#[test] +fn flatten_all_symbolic() { + // `and(A, and(B, C)) → and(A, B, C)` + let mut expr = nested_and(Expr::arg(0), Expr::arg(1), Expr::arg(2)); + let result = fold_expr_and(&mut expr); + + assert!(result.is_none()); // Modified in place + assert_eq!(expr.operands.len(), 3); + assert_eq!(expr.operands[0], Expr::arg(0)); + assert_eq!(expr.operands[1], Expr::arg(1)); + assert_eq!(expr.operands[2], Expr::arg(2)); +} + +#[test] +fn flatten_with_true_in_outer() { + // `and(true, and(B, C)) → and(B, C)` + let mut expr = nested_and(true.into(), Expr::arg(1), Expr::arg(2)); + let result = fold_expr_and(&mut expr); + + assert!(result.is_none()); + assert_eq!(expr.operands.len(), 2); + assert_eq!(expr.operands[0], Expr::arg(1)); + assert_eq!(expr.operands[1], Expr::arg(2)); +} + +#[test] +fn flatten_with_true_in_nested_first() { + // `and(A, and(true, C)) → and(A, C)` + let mut expr = nested_and(Expr::arg(0), true.into(), Expr::arg(2)); + let result = fold_expr_and(&mut expr); + + assert!(result.is_none()); + assert_eq!(expr.operands.len(), 2); + assert_eq!(expr.operands[0], Expr::arg(0)); + assert_eq!(expr.operands[1], Expr::arg(2)); +} + +#[test] +fn flatten_with_true_in_nested_second() { + // `and(A, and(B, true)) → and(A, B)` + let mut expr = nested_and(Expr::arg(0), Expr::arg(1), true.into()); + let result = fold_expr_and(&mut expr); + + assert!(result.is_none()); + assert_eq!(expr.operands.len(), 2); + assert_eq!(expr.operands[0], Expr::arg(0)); + assert_eq!(expr.operands[1], Expr::arg(1)); +} + +#[test] +fn flatten_outer_true_nested_one_true() { + // `and(true, and(true, C)) → C` + let mut expr = nested_and(true.into(), true.into(), Expr::arg(2)); + let result = fold_expr_and(&mut expr); + + assert!(result.is_some()); + assert_eq!(result.unwrap(), Expr::arg(2)); +} + +#[test] +fn flatten_outer_symbolic_nested_all_true() { + // `and(A, and(true, true)) → A` + let mut expr = nested_and(Expr::arg(0), true.into(), true.into()); + let result = fold_expr_and(&mut expr); + + assert!(result.is_some()); + assert_eq!(result.unwrap(), Expr::arg(0)); +} + +#[test] +fn flatten_all_true() { + // `and(true, and(true, true)) → true` + let mut expr = nested_and(true.into(), true.into(), true.into()); + let result = fold_expr_and(&mut expr); + + assert!(result.is_some()); + assert!(result.unwrap().is_true()); +} + +#[test] +fn flatten_with_false_in_outer() { + // `and(false, and(B, C)) → false` + let mut expr = nested_and(false.into(), Expr::arg(1), Expr::arg(2)); + let result = fold_expr_and(&mut expr); + + assert!(result.is_some()); + assert!(result.unwrap().is_false()); +} + +#[test] +fn flatten_with_false_in_nested() { + // `and(A, and(false, C)) → false` + let mut expr = nested_and(Expr::arg(0), false.into(), Expr::arg(2)); + let result = fold_expr_and(&mut expr); + + assert!(result.is_some()); + assert!(result.unwrap().is_false()); +} + +#[test] +fn flatten_true_and_false_mixed() { + // `and(true, and(false, true)) → false` + let mut expr = nested_and(true.into(), false.into(), true.into()); + let result = fold_expr_and(&mut expr); + + assert!(result.is_some()); + assert!(result.unwrap().is_false()); +} + +#[test] +fn single_operand_unwrapped() { + // `and(arg(0)) → arg(0)` + let mut expr = ExprAnd { + operands: vec![Expr::arg(0)], + }; + let result = fold_expr_and(&mut expr); + + assert!(result.is_some()); + assert_eq!(result.unwrap(), Expr::arg(0)); +} + +#[test] +fn empty_after_removing_true() { + // `and(true, true) → true` + let mut expr = ExprAnd { + operands: vec![true.into(), true.into()], + }; + let result = fold_expr_and(&mut expr); + + assert!(result.is_some()); + assert!(result.unwrap().is_true()); +} + +// Null propagation tests + +#[test] +fn null_and_null_becomes_null() { + // `null and null` → `null` + let mut expr = ExprAnd { + operands: vec![Expr::null(), Expr::null()], + }; + let result = fold_expr_and(&mut expr); + + assert!(result.is_some()); + assert!(result.unwrap().is_value_null()); +} + +#[test] +fn null_and_true_becomes_null() { + // `null and true` → `null` (true is removed, leaving only null) + let mut expr = ExprAnd { + operands: vec![Expr::null(), true.into()], + }; + let result = fold_expr_and(&mut expr); + + assert!(result.is_some()); + assert!(result.unwrap().is_value_null()); +} + +#[test] +fn true_and_null_becomes_null() { + // `true and null` → `null` (true is removed, leaving only null) + let mut expr = ExprAnd { + operands: vec![true.into(), Expr::null()], + }; + let result = fold_expr_and(&mut expr); + + assert!(result.is_some()); + assert!(result.unwrap().is_value_null()); +} + +#[test] +fn null_and_false_becomes_false() { + // `null and false` → `false` (false short-circuits) + let mut expr = ExprAnd { + operands: vec![Expr::null(), false.into()], + }; + let result = fold_expr_and(&mut expr); + + assert!(result.is_some()); + assert!(result.unwrap().is_false()); +} + +#[test] +fn false_and_null_becomes_false() { + // `false and null` → `false` (false short-circuits) + let mut expr = ExprAnd { + operands: vec![false.into(), Expr::null()], + }; + let result = fold_expr_and(&mut expr); + + assert!(result.is_some()); + assert!(result.unwrap().is_false()); +} + +#[test] +fn null_and_symbolic_not_simplified() { + // `null and a` → no change (symbolic operand present) + let mut expr = ExprAnd { + operands: vec![Expr::null(), Expr::arg(0)], + }; + let result = fold_expr_and(&mut expr); + + assert!(result.is_none()); + assert_eq!(expr.operands.len(), 2); +} + +#[test] +fn multiple_nulls_become_null() { + // `null and null and null` → `null` + let mut expr = ExprAnd { + operands: vec![Expr::null(), Expr::null(), Expr::null()], + }; + let result = fold_expr_and(&mut expr); + + assert!(result.is_some()); + assert!(result.unwrap().is_value_null()); +} + +#[test] +fn multiple_nulls_and_true_becomes_null() { + // `null and true and null and true` → `null` + let mut expr = ExprAnd { + operands: vec![Expr::null(), true.into(), Expr::null(), true.into()], + }; + let result = fold_expr_and(&mut expr); + + assert!(result.is_some()); + assert!(result.unwrap().is_value_null()); +} + +// Error operand tests + +#[test] +fn error_operand_not_simplified_in_and() { + // `and(error("boom"), arg(0))` → no simplification. + // Error represents an unreachable branch; it does not poison the AND. + // In practice, other operands (guards) will drive the AND to false. + let mut expr = ExprAnd { + operands: vec![Expr::error("boom"), Expr::arg(0)], + }; + let result = fold_expr_and(&mut expr); + + assert!(result.is_none()); + assert_eq!(expr.operands.len(), 2); +} + +#[test] +fn error_and_true_keeps_error() { + // `and(error("boom"), true)` → `error("boom")` (true is removed, error remains) + let mut expr = ExprAnd { + operands: vec![Expr::error("boom"), true.into()], + }; + let result = fold_expr_and(&mut expr); + + assert!(result.is_some()); + assert!(matches!(&result.unwrap(), Expr::Error(e) if e.message == "boom")); +} + +#[test] +fn error_and_false_becomes_false() { + // `and(error("boom"), false)` → `false` (false short-circuits AND) + let mut expr = ExprAnd { + operands: vec![Expr::error("boom"), false.into()], + }; + let result = fold_expr_and(&mut expr); + + assert!(result.is_some()); + assert!(result.unwrap().is_false()); +} diff --git a/packages/toasty/src/engine/fold/tests/expr_binary_op.rs b/packages/toasty/src/engine/fold/tests/expr_binary_op.rs new file mode 100644 index 00000000000..8b2d6e44737 --- /dev/null +++ b/packages/toasty/src/engine/fold/tests/expr_binary_op.rs @@ -0,0 +1,335 @@ +use crate::engine::fold::expr_binary_op::fold_expr_binary_op; +use toasty_core::stmt::{BinaryOp, Expr, Value}; + +#[test] +fn constant_eq_same_values_becomes_true() { + // `eq(5, 5)` → `true` + let mut lhs = Expr::Value(Value::from(5i64)); + let mut rhs = Expr::Value(Value::from(5i64)); + + let result = fold_expr_binary_op(BinaryOp::Eq, &mut lhs, &mut rhs); + + assert!(matches!(result, Some(Expr::Value(Value::Bool(true))))); +} + +#[test] +fn constant_eq_different_values_becomes_false() { + // `eq(1, 2)` → `false` + let mut lhs = Expr::Value(Value::from(1i64)); + let mut rhs = Expr::Value(Value::from(2i64)); + + let result = fold_expr_binary_op(BinaryOp::Eq, &mut lhs, &mut rhs); + + assert!(matches!(result, Some(Expr::Value(Value::Bool(false))))); +} + +#[test] +fn constant_ne_same_values_becomes_false() { + // `ne(5, 5)` → `false` + let mut lhs = Expr::Value(Value::from(5i64)); + let mut rhs = Expr::Value(Value::from(5i64)); + + let result = fold_expr_binary_op(BinaryOp::Ne, &mut lhs, &mut rhs); + + assert!(matches!(result, Some(Expr::Value(Value::Bool(false))))); +} + +#[test] +fn constant_ne_different_values_becomes_true() { + // `ne("abc", "def")` → `true` + let mut lhs = Expr::Value(Value::from("abc")); + let mut rhs = Expr::Value(Value::from("def")); + + let result = fold_expr_binary_op(BinaryOp::Ne, &mut lhs, &mut rhs); + + assert!(matches!(result, Some(Expr::Value(Value::Bool(true))))); +} + +#[test] +fn constant_eq_with_null_becomes_null() { + // `eq(null, 5)` → `null` + let mut lhs = Expr::Value(Value::Null); + let mut rhs = Expr::Value(Value::from(5i64)); + + let result = fold_expr_binary_op(BinaryOp::Eq, &mut lhs, &mut rhs); + + assert!(matches!(result, Some(Expr::Value(Value::Null)))); +} + +#[test] +fn constant_eq_null_with_null_becomes_null() { + // `eq(null, null)` → `null` + let mut lhs = Expr::Value(Value::Null); + let mut rhs = Expr::Value(Value::Null); + + let result = fold_expr_binary_op(BinaryOp::Eq, &mut lhs, &mut rhs); + + assert!(matches!(result, Some(Expr::Value(Value::Null)))); +} + +#[test] +fn constant_lt_becomes_true() { + // `1 < 5` → `true` + let mut lhs = Expr::Value(Value::from(1i64)); + let mut rhs = Expr::Value(Value::from(5i64)); + + let result = fold_expr_binary_op(BinaryOp::Lt, &mut lhs, &mut rhs); + + assert!(matches!(result, Some(Expr::Value(Value::Bool(true))))); +} + +#[test] +fn constant_lt_becomes_false() { + // `5 < 1` → `false` + let mut lhs = Expr::Value(Value::from(5i64)); + let mut rhs = Expr::Value(Value::from(1i64)); + + let result = fold_expr_binary_op(BinaryOp::Lt, &mut lhs, &mut rhs); + + assert!(matches!(result, Some(Expr::Value(Value::Bool(false))))); +} + +#[test] +fn constant_le_equal_becomes_true() { + // `5 <= 5` → `true` + let mut lhs = Expr::Value(Value::from(5i64)); + let mut rhs = Expr::Value(Value::from(5i64)); + + let result = fold_expr_binary_op(BinaryOp::Le, &mut lhs, &mut rhs); + + assert!(matches!(result, Some(Expr::Value(Value::Bool(true))))); +} + +#[test] +fn constant_le_becomes_false() { + // `10 <= 5` → `false` + let mut lhs = Expr::Value(Value::from(10i64)); + let mut rhs = Expr::Value(Value::from(5i64)); + + let result = fold_expr_binary_op(BinaryOp::Le, &mut lhs, &mut rhs); + + assert!(matches!(result, Some(Expr::Value(Value::Bool(false))))); +} + +#[test] +fn constant_gt_becomes_true() { + // `10 > 5` → `true` + let mut lhs = Expr::Value(Value::from(10i64)); + let mut rhs = Expr::Value(Value::from(5i64)); + + let result = fold_expr_binary_op(BinaryOp::Gt, &mut lhs, &mut rhs); + + assert!(matches!(result, Some(Expr::Value(Value::Bool(true))))); +} + +#[test] +fn constant_gt_becomes_false() { + // `1 > 5` → `false` + let mut lhs = Expr::Value(Value::from(1i64)); + let mut rhs = Expr::Value(Value::from(5i64)); + + let result = fold_expr_binary_op(BinaryOp::Gt, &mut lhs, &mut rhs); + + assert!(matches!(result, Some(Expr::Value(Value::Bool(false))))); +} + +#[test] +fn constant_ge_equal_becomes_true() { + // `5 >= 5` → `true` + let mut lhs = Expr::Value(Value::from(5i64)); + let mut rhs = Expr::Value(Value::from(5i64)); + + let result = fold_expr_binary_op(BinaryOp::Ge, &mut lhs, &mut rhs); + + assert!(matches!(result, Some(Expr::Value(Value::Bool(true))))); +} + +#[test] +fn constant_ge_becomes_false() { + // `1 >= 5` → `false` + let mut lhs = Expr::Value(Value::from(1i64)); + let mut rhs = Expr::Value(Value::from(5i64)); + + let result = fold_expr_binary_op(BinaryOp::Ge, &mut lhs, &mut rhs); + + assert!(matches!(result, Some(Expr::Value(Value::Bool(false))))); +} + +#[test] +fn constant_lt_string_lexicographic() { + // `"abc" < "def"` → `true` (lexicographic) + let mut lhs = Expr::Value(Value::from("abc")); + let mut rhs = Expr::Value(Value::from("def")); + + let result = fold_expr_binary_op(BinaryOp::Lt, &mut lhs, &mut rhs); + + assert!(matches!(result, Some(Expr::Value(Value::Bool(true))))); +} + +#[test] +fn constant_lt_different_types_not_simplified() { + // `5 < "abc"` is not simplified (incompatible types) + let mut lhs = Expr::Value(Value::from(5i64)); + let mut rhs = Expr::Value(Value::from("abc")); + + let result = fold_expr_binary_op(BinaryOp::Lt, &mut lhs, &mut rhs); + + assert!(result.is_none()); +} + +#[test] +fn lt_with_non_constant_not_simplified() { + // `arg(0) < 5` is not simplified (non-constant lhs, value already on right) + let mut lhs = Expr::arg(0); + let mut rhs = Expr::Value(Value::from(5i64)); + + let result = fold_expr_binary_op(BinaryOp::Lt, &mut lhs, &mut rhs); + + assert!(result.is_none()); +} + +#[test] +fn x_eq_true_becomes_x() { + // `x = true` → `x` + let mut lhs = Expr::arg(0); + let mut rhs = Expr::Value(Value::Bool(true)); + + let result = fold_expr_binary_op(BinaryOp::Eq, &mut lhs, &mut rhs); + + assert!(matches!(result, Some(Expr::Arg(_)))); +} + +#[test] +fn true_eq_x_becomes_x() { + // `true = x` → `x` + let mut lhs = Expr::Value(Value::Bool(true)); + let mut rhs = Expr::arg(0); + + let result = fold_expr_binary_op(BinaryOp::Eq, &mut lhs, &mut rhs); + + assert!(matches!(result, Some(Expr::Arg(_)))); +} + +#[test] +fn x_eq_false_becomes_not_x() { + // `x = false` → `not(x)` + let mut lhs = Expr::arg(0); + let mut rhs = Expr::Value(Value::Bool(false)); + + let result = fold_expr_binary_op(BinaryOp::Eq, &mut lhs, &mut rhs); + + assert!(matches!(result, Some(Expr::Not(_)))); +} + +#[test] +fn x_ne_true_becomes_not_x() { + // `x != true` → `not(x)` + let mut lhs = Expr::arg(0); + let mut rhs = Expr::Value(Value::Bool(true)); + + let result = fold_expr_binary_op(BinaryOp::Ne, &mut lhs, &mut rhs); + + assert!(matches!(result, Some(Expr::Not(_)))); +} + +#[test] +fn x_ne_false_becomes_x() { + // `x != false` → `x` + let mut lhs = Expr::arg(0); + let mut rhs = Expr::Value(Value::Bool(false)); + + let result = fold_expr_binary_op(BinaryOp::Ne, &mut lhs, &mut rhs); + + assert!(matches!(result, Some(Expr::Arg(_)))); +} + +#[test] +fn canonicalize_eq_literal_on_left() { + // `5 = x` → `x = 5` + let mut lhs = Expr::Value(Value::from(5i64)); + let mut rhs = Expr::arg(0); + + let result = fold_expr_binary_op(BinaryOp::Eq, &mut lhs, &mut rhs); + + let Some(Expr::BinaryOp(binary_op)) = result else { + panic!("expected BinaryOp"); + }; + assert_eq!(binary_op.op, BinaryOp::Eq); + assert!(matches!(*binary_op.lhs, Expr::Arg(_))); + assert!(matches!(*binary_op.rhs, Expr::Value(Value::I64(5)))); +} + +#[test] +fn canonicalize_lt_literal_on_left() { + // `5 < x` → `x > 5` + let mut lhs = Expr::Value(Value::from(5i64)); + let mut rhs = Expr::arg(0); + + let result = fold_expr_binary_op(BinaryOp::Lt, &mut lhs, &mut rhs); + + let Some(Expr::BinaryOp(binary_op)) = result else { + panic!("expected BinaryOp"); + }; + assert_eq!(binary_op.op, BinaryOp::Gt); + assert!(matches!(*binary_op.lhs, Expr::Arg(_))); + assert!(matches!(*binary_op.rhs, Expr::Value(Value::I64(5)))); +} + +#[test] +fn canonicalize_gt_literal_on_left() { + // `5 > x` → `x < 5` + let mut lhs = Expr::Value(Value::from(5i64)); + let mut rhs = Expr::arg(0); + + let result = fold_expr_binary_op(BinaryOp::Gt, &mut lhs, &mut rhs); + + let Some(Expr::BinaryOp(binary_op)) = result else { + panic!("expected BinaryOp"); + }; + assert_eq!(binary_op.op, BinaryOp::Lt); + assert!(matches!(*binary_op.lhs, Expr::Arg(_))); + assert!(matches!(*binary_op.rhs, Expr::Value(Value::I64(5)))); +} + +#[test] +fn canonicalize_le_literal_on_left() { + // `5 <= x` → `x >= 5` + let mut lhs = Expr::Value(Value::from(5i64)); + let mut rhs = Expr::arg(0); + + let result = fold_expr_binary_op(BinaryOp::Le, &mut lhs, &mut rhs); + + let Some(Expr::BinaryOp(binary_op)) = result else { + panic!("expected BinaryOp"); + }; + assert_eq!(binary_op.op, BinaryOp::Ge); + assert!(matches!(*binary_op.lhs, Expr::Arg(_))); + assert!(matches!(*binary_op.rhs, Expr::Value(Value::I64(5)))); +} + +#[test] +fn canonicalize_ge_literal_on_left() { + // `5 >= x` → `x <= 5` + let mut lhs = Expr::Value(Value::from(5i64)); + let mut rhs = Expr::arg(0); + + let result = fold_expr_binary_op(BinaryOp::Ge, &mut lhs, &mut rhs); + + let Some(Expr::BinaryOp(binary_op)) = result else { + panic!("expected BinaryOp"); + }; + assert_eq!(binary_op.op, BinaryOp::Le); + assert!(matches!(*binary_op.lhs, Expr::Arg(_))); + assert!(matches!(*binary_op.rhs, Expr::Value(Value::I64(5)))); +} + +#[test] +fn no_canonicalize_when_literal_on_right() { + // `x < 5` is already canonical, no change + let mut lhs = Expr::arg(0); + let mut rhs = Expr::Value(Value::from(5i64)); + + let result = fold_expr_binary_op(BinaryOp::Lt, &mut lhs, &mut rhs); + + assert!(result.is_none()); +} diff --git a/packages/toasty/src/engine/fold/tests/expr_cast.rs b/packages/toasty/src/engine/fold/tests/expr_cast.rs new file mode 100644 index 00000000000..213ba437b6b --- /dev/null +++ b/packages/toasty/src/engine/fold/tests/expr_cast.rs @@ -0,0 +1,131 @@ +use crate::engine::fold::expr_cast::fold_expr_cast; +use toasty_core::{ + schema::app::ModelId, + stmt::{Expr, ExprCast, Type, Value}, +}; +use uuid::Uuid; + +#[test] +fn uuid_cast_to_string() { + // `cast(Uuid("129d7fd7-..."), String) → "129d7fd7-..."` + let uuid = Uuid::parse_str("129d7fd7-cde2-42d1-be4b-5a27b793f93a").unwrap(); + let mut expr = ExprCast { + from: None, + expr: Box::new(Expr::Value(Value::Uuid(uuid))), + ty: Type::String, + }; + let result = fold_expr_cast(&mut expr); + + assert!(result.is_some()); + let Expr::Value(Value::String(s)) = result.unwrap() else { + panic!("expected the result to be a `Value::String`"); + }; + assert_eq!(s, "129d7fd7-cde2-42d1-be4b-5a27b793f93a"); +} + +#[test] +fn string_cast_to_uuid() { + // `cast("129d7fd7-...", Uuid) → Uuid("129d7fd7-...")` + let mut expr = ExprCast { + from: None, + expr: Box::new(Expr::Value(Value::from( + "129d7fd7-cde2-42d1-be4b-5a27b793f93a", + ))), + ty: Type::Uuid, + }; + let result = fold_expr_cast(&mut expr); + + assert!(result.is_some()); + let Expr::Value(Value::Uuid(uuid)) = result.unwrap() else { + panic!("expected result to be a `Value::Uuid`"); + }; + assert_eq!(uuid.to_string(), "129d7fd7-cde2-42d1-be4b-5a27b793f93a"); +} + +#[test] +fn non_const_not_simplified() { + // `cast(arg(0), String)`, non-constant, not simplified + let mut expr = ExprCast { + from: None, + expr: Box::new(Expr::arg(0)), + ty: Type::String, + }; + let result = fold_expr_cast(&mut expr); + + assert!(result.is_none()); +} + +#[test] +fn default_passes_through() { + let mut expr = ExprCast { + from: None, + expr: Box::new(Expr::Default), + ty: Type::I32, + }; + + assert_eq!(fold_expr_cast(&mut expr), Some(Expr::Default)); +} + +#[test] +fn null_passes_through() { + // `cast(null, String) → null` + let mut expr = ExprCast { + from: None, + expr: Box::new(Expr::Value(Value::Null)), + ty: Type::String, + }; + let result = fold_expr_cast(&mut expr); + + assert!(result.is_some()); + let Expr::Value(val) = result.unwrap() else { + panic!("expected result to be a `Value`"); + }; + assert!(val.is_null()); +} + +#[test] +fn document_lowering_cast_skipped() { + // A `#[document]` lowering cast (source type present) is schema-directed; + // fold is schema-free and must leave it for the simplifier. + let mut expr = ExprCast { + from: Some(Type::Model(ModelId(0))), + expr: Box::new(Expr::Value(Value::record_from_vec(vec![Value::from( + "Alice", + )]))), + ty: Type::Object, + }; + let result = fold_expr_cast(&mut expr); + + assert!(result.is_none()); +} + +#[test] +fn document_raising_cast_skipped() { + // A `#[document]` raising cast (model-typed target) is schema-directed; + // fold is schema-free and must leave it for the simplifier. + let mut expr = ExprCast { + from: None, + expr: Box::new(Expr::Value(Value::Null)), + ty: Type::Model(ModelId(0)), + }; + let result = fold_expr_cast(&mut expr); + + assert!(result.is_none()); +} + +#[test] +fn string_identity_cast() { + // `cast("hello", String) → "hello"` + let mut expr = ExprCast { + from: None, + expr: Box::new(Expr::Value(Value::from("hello"))), + ty: Type::String, + }; + let result = fold_expr_cast(&mut expr); + + assert!(result.is_some()); + let Expr::Value(Value::String(s)) = result.unwrap() else { + panic!("expected result to be a `Value::String`"); + }; + assert_eq!(s, "hello"); +} diff --git a/packages/toasty/src/engine/fold/tests/expr_in_list.rs b/packages/toasty/src/engine/fold/tests/expr_in_list.rs new file mode 100644 index 00000000000..c58b5da8bae --- /dev/null +++ b/packages/toasty/src/engine/fold/tests/expr_in_list.rs @@ -0,0 +1,269 @@ +use crate::engine::fold::expr_in_list::fold_expr_in_list; +use toasty_core::stmt::{self, Expr, Value}; + +/// Helper to construct an "in list" expression. +fn in_list(lhs: Expr, list: Expr) -> stmt::ExprInList { + stmt::ExprInList { + expr: Box::new(lhs), + list: Box::new(list), + } +} + +/// Helper for making a value list. +fn value_list(values: Vec) -> Expr { + Expr::Value(Value::List(values)) +} + +/// Helper for making an expression list. +fn expr_list(items: Vec) -> Expr { + Expr::List(stmt::ExprList { items }) +} + +#[test] +fn empty_value_list_becomes_false() { + // `in_list(arg(0), []) → false` + let mut expr = in_list(Expr::arg(0), value_list(vec![])); + let result = fold_expr_in_list(&mut expr); + + assert!(result.is_some()); + assert!(result.unwrap().is_false()); +} + +#[test] +fn empty_expr_list_becomes_false() { + // `in_list(arg(0), list([])) → false` + let mut expr = in_list(Expr::arg(0), expr_list(vec![])); + let result = fold_expr_in_list(&mut expr); + + assert!(result.is_some()); + assert!(result.unwrap().is_false()); +} + +#[test] +fn single_value_becomes_eq() { + // `in_list(arg(0), [42]) → eq(arg(0), 42)` + let mut expr = in_list(Expr::arg(0), value_list(vec![Value::from(42i64)])); + let result = fold_expr_in_list(&mut expr); + + assert!(matches!( + result, + Some(Expr::BinaryOp(stmt::ExprBinaryOp { lhs, op: stmt::BinaryOp::Eq, rhs })) + if *lhs == Expr::arg(0) && *rhs == Expr::Value(Value::from(42i64)) + )); +} + +#[test] +fn single_expr_becomes_eq() { + // `in_list(arg(0), list([arg(1)])) → eq(arg(0), arg(1))` + let mut expr = in_list(Expr::arg(0), expr_list(vec![Expr::arg(1)])); + let result = fold_expr_in_list(&mut expr); + + assert!(matches!( + result, + Some(Expr::BinaryOp(stmt::ExprBinaryOp { lhs, op: stmt::BinaryOp::Eq, rhs })) + if *lhs == Expr::arg(0) && *rhs == Expr::arg(1) + )); +} + +#[test] +fn two_values_unchanged() { + // `in_list(arg(0), [1, 2])`, multiple items, not simplified + let mut expr = in_list( + Expr::arg(0), + value_list(vec![Value::from(1i64), Value::from(2i64)]), + ); + let result = fold_expr_in_list(&mut expr); + + assert!(result.is_none()); +} + +#[test] +fn two_exprs_unchanged() { + // `in_list(arg(0), list([arg(1), arg(2)]))`, multiple items, not simplified + let mut expr = in_list(Expr::arg(0), expr_list(vec![Expr::arg(1), Expr::arg(2)])); + let result = fold_expr_in_list(&mut expr); + + assert!(result.is_none()); +} + +#[test] +fn arg_in_single() { + // `in_list(arg(0), [42]) → eq(arg(0), 42)` + let mut expr = in_list(Expr::arg(0), value_list(vec![Value::from(42i64)])); + let result = fold_expr_in_list(&mut expr); + + assert!(matches!( + result, + Some(Expr::BinaryOp(stmt::ExprBinaryOp { lhs, op: stmt::BinaryOp::Eq, rhs })) + if *lhs == Expr::arg(0) && *rhs == Expr::Value(Value::from(42i64)) + )); +} + +#[test] +fn arg_in_empty() { + // `in_list(arg(0), []) → false` + let mut expr = in_list(Expr::arg(0), value_list(vec![])); + let result = fold_expr_in_list(&mut expr); + + assert!(result.is_some()); + assert!(result.unwrap().is_false()); +} + +#[test] +fn arg_in_multi() { + // `in_list(arg(0), [1, 2])`, multiple items, not simplified + let mut expr = in_list( + Expr::arg(0), + value_list(vec![Value::from(1i64), Value::from(2i64)]), + ); + let result = fold_expr_in_list(&mut expr); + + assert!(result.is_none()); +} + +// Null propagation tests + +#[test] +fn null_in_list_becomes_null() { + // `null in (1, 2, 3)` → `null` + let mut expr = in_list( + Expr::null(), + value_list(vec![ + Value::from(1i64), + Value::from(2i64), + Value::from(3i64), + ]), + ); + let result = fold_expr_in_list(&mut expr); + + assert!(result.is_some()); + assert!(result.unwrap().is_value_null()); +} + +#[test] +fn null_in_single_item_becomes_null() { + // `null in (42)` → `null` + let mut expr = in_list(Expr::null(), value_list(vec![Value::from(42i64)])); + let result = fold_expr_in_list(&mut expr); + + assert!(result.is_some()); + assert!(result.unwrap().is_value_null()); +} + +#[test] +fn null_in_expr_list_becomes_null() { + // `null in list([arg(0), arg(1)])` → `null` + let mut expr = in_list(Expr::null(), expr_list(vec![Expr::arg(0), Expr::arg(1)])); + let result = fold_expr_in_list(&mut expr); + + assert!(result.is_some()); + assert!(result.unwrap().is_value_null()); +} + +// Deduplication tests + +#[test] +fn dedup_all_identical_values() { + // `x in (1, 1, 1)` → `x = 1` + let mut expr = in_list( + Expr::arg(0), + value_list(vec![ + Value::from(1i64), + Value::from(1i64), + Value::from(1i64), + ]), + ); + let result = fold_expr_in_list(&mut expr); + + // Dedup leaves one item, which is then rewritten to equality + assert!(matches!(result, Some(Expr::BinaryOp(_)))); +} + +#[test] +fn dedup_leading_duplicate() { + // `x in (1, 1, 2)` → `x in (1, 2)` + let mut expr = in_list( + Expr::arg(0), + value_list(vec![ + Value::from(1i64), + Value::from(1i64), + Value::from(2i64), + ]), + ); + let result = fold_expr_in_list(&mut expr); + + assert!(result.is_none()); + let Expr::Value(Value::List(values)) = &*expr.list else { + panic!("expected Value::List"); + }; + assert_eq!(values.len(), 2); + assert_eq!(values[0], Value::from(1i64)); + assert_eq!(values[1], Value::from(2i64)); +} + +#[test] +fn dedup_trailing_duplicate() { + // `x in (1, 2, 2)` → `x in (1, 2)` + let mut expr = in_list( + Expr::arg(0), + value_list(vec![ + Value::from(1i64), + Value::from(2i64), + Value::from(2i64), + ]), + ); + let result = fold_expr_in_list(&mut expr); + + assert!(result.is_none()); + let Expr::Value(Value::List(values)) = &*expr.list else { + panic!("expected Value::List"); + }; + assert_eq!(values.len(), 2); + assert_eq!(values[0], Value::from(1i64)); + assert_eq!(values[1], Value::from(2i64)); +} + +#[test] +fn dedup_preserves_order() { + // `x in (3, 1, 2, 1, 3)` → `x in (3, 1, 2)` — first occurrence wins + let mut expr = in_list( + Expr::arg(0), + value_list(vec![ + Value::from(3i64), + Value::from(1i64), + Value::from(2i64), + Value::from(1i64), + Value::from(3i64), + ]), + ); + let result = fold_expr_in_list(&mut expr); + + assert!(result.is_none()); + let Expr::Value(Value::List(values)) = &*expr.list else { + panic!("expected Value::List"); + }; + assert_eq!(values.len(), 3); + assert_eq!(values[0], Value::from(3i64)); + assert_eq!(values[1], Value::from(1i64)); + assert_eq!(values[2], Value::from(2i64)); +} + +#[test] +fn dedup_no_duplicates_unchanged() { + // `x in (1, 2, 3)` — no duplicates, list unchanged + let mut expr = in_list( + Expr::arg(0), + value_list(vec![ + Value::from(1i64), + Value::from(2i64), + Value::from(3i64), + ]), + ); + let result = fold_expr_in_list(&mut expr); + + assert!(result.is_none()); + let Expr::Value(Value::List(values)) = &*expr.list else { + panic!("expected Value::List"); + }; + assert_eq!(values.len(), 3); +} diff --git a/packages/toasty/src/engine/fold/tests/expr_intersects.rs b/packages/toasty/src/engine/fold/tests/expr_intersects.rs new file mode 100644 index 00000000000..3d0651a9b4f --- /dev/null +++ b/packages/toasty/src/engine/fold/tests/expr_intersects.rs @@ -0,0 +1,38 @@ +use crate::engine::fold::expr_intersects::fold_expr_intersects; +use toasty_core::stmt::{self, Expr, Value}; + +fn intersects(lhs: Expr, rhs: Expr) -> stmt::ExprIntersects { + stmt::ExprIntersects { + lhs: Box::new(lhs), + rhs: Box::new(rhs), + } +} + +#[test] +fn empty_value_list_rhs_becomes_false() { + // `intersects(arg(0), []) → false` + let mut expr = intersects(Expr::arg(0), Expr::Value(Value::List(vec![]))); + let result = fold_expr_intersects(&mut expr); + + assert_eq!(result, Some(Expr::Value(Value::Bool(false)))); +} + +#[test] +fn empty_expr_list_rhs_becomes_false() { + // `intersects(arg(0), list([])) → false` + let mut expr = intersects(Expr::arg(0), Expr::List(stmt::ExprList { items: vec![] })); + let result = fold_expr_intersects(&mut expr); + + assert_eq!(result, Some(Expr::Value(Value::Bool(false)))); +} + +#[test] +fn non_empty_rhs_left_alone() { + let mut expr = intersects( + Expr::arg(0), + Expr::Value(Value::List(vec![Value::from("admin".to_string())])), + ); + let result = fold_expr_intersects(&mut expr); + + assert!(result.is_none()); +} diff --git a/packages/toasty/src/engine/fold/tests/expr_is_null.rs b/packages/toasty/src/engine/fold/tests/expr_is_null.rs new file mode 100644 index 00000000000..fdf0511b2dc --- /dev/null +++ b/packages/toasty/src/engine/fold/tests/expr_is_null.rs @@ -0,0 +1,75 @@ +use crate::engine::fold::{expr_is_null::fold_expr_is_null, fold_stmt}; +use toasty_core::stmt::{Expr, ExprArg, ExprCast, ExprIsNull, Type, Value}; + +#[test] +fn cast_is_stripped_from_is_null() { + // `is_null(cast(arg(0), String))` → `is_null(arg(0))` + // Nullity is type-independent, so the cast is stripped. + let mut expr = ExprIsNull { + expr: Box::new(Expr::Cast(ExprCast { + from: None, + expr: Box::new(Expr::arg(0)), + ty: Type::String, + })), + negated: false, + }; + let result = fold_expr_is_null(&mut expr); + + assert!(result.is_none()); + assert!(matches!(*expr.expr, Expr::Arg(ExprArg { position: 0, .. }))); +} + +#[test] +fn non_cast_expr_not_simplified() { + // `is_null(arg(0))`, non-cast, not simplified + let mut expr = ExprIsNull { + expr: Box::new(Expr::arg(0)), + negated: false, + }; + let result = fold_expr_is_null(&mut expr); + + assert!(result.is_none()); + assert!(matches!(*expr.expr, Expr::Arg(ExprArg { position: 0, .. }))); +} + +#[test] +fn null_is_null_becomes_true() { + // `null is null` → `true` + let mut expr = ExprIsNull { + expr: Box::new(Expr::null()), + negated: false, + }; + let result = fold_expr_is_null(&mut expr); + + assert!(matches!(result, Some(Expr::Value(Value::Bool(true))))); +} + +#[test] +fn non_null_const_is_null_becomes_false() { + // `5 is null` → `false` + let mut expr = ExprIsNull { + expr: Box::new(Expr::from(5i64)), + negated: false, + }; + let result = fold_expr_is_null(&mut expr); + + assert!(matches!(result, Some(Expr::Value(Value::Bool(false))))); +} + +#[test] +fn null_is_not_null_becomes_false() { + // `is_not_null(null)` → `false` + let mut expr = Expr::is_not_null(Expr::null()); + fold_stmt(&mut expr); + + assert!(matches!(expr, Expr::Value(Value::Bool(false)))); +} + +#[test] +fn non_null_const_is_not_null_becomes_true() { + // `is_not_null(5)` → `true` + let mut expr = Expr::is_not_null(Expr::from(5i64)); + fold_stmt(&mut expr); + + assert!(matches!(expr, Expr::Value(Value::Bool(true)))); +} diff --git a/packages/toasty/src/engine/fold/tests/expr_is_superset.rs b/packages/toasty/src/engine/fold/tests/expr_is_superset.rs new file mode 100644 index 00000000000..7ec311dc5b8 --- /dev/null +++ b/packages/toasty/src/engine/fold/tests/expr_is_superset.rs @@ -0,0 +1,38 @@ +use crate::engine::fold::expr_is_superset::fold_expr_is_superset; +use toasty_core::stmt::{self, Expr, Value}; + +fn is_superset(lhs: Expr, rhs: Expr) -> stmt::ExprIsSuperset { + stmt::ExprIsSuperset { + lhs: Box::new(lhs), + rhs: Box::new(rhs), + } +} + +#[test] +fn empty_value_list_rhs_becomes_true() { + // `is_superset(arg(0), []) → true` + let mut expr = is_superset(Expr::arg(0), Expr::Value(Value::List(vec![]))); + let result = fold_expr_is_superset(&mut expr); + + assert_eq!(result, Some(Expr::Value(Value::Bool(true)))); +} + +#[test] +fn empty_expr_list_rhs_becomes_true() { + // `is_superset(arg(0), list([])) → true` + let mut expr = is_superset(Expr::arg(0), Expr::List(stmt::ExprList { items: vec![] })); + let result = fold_expr_is_superset(&mut expr); + + assert_eq!(result, Some(Expr::Value(Value::Bool(true)))); +} + +#[test] +fn non_empty_rhs_left_alone() { + let mut expr = is_superset( + Expr::arg(0), + Expr::Value(Value::List(vec![Value::from("admin".to_string())])), + ); + let result = fold_expr_is_superset(&mut expr); + + assert!(result.is_none()); +} diff --git a/packages/toasty/src/engine/fold/tests/expr_let.rs b/packages/toasty/src/engine/fold/tests/expr_let.rs new file mode 100644 index 00000000000..16e1a67af5f --- /dev/null +++ b/packages/toasty/src/engine/fold/tests/expr_let.rs @@ -0,0 +1,152 @@ +use crate::engine::fold::{self, expr_let::fold_expr_let}; +use toasty_core::stmt::{self, Expr, ExprLet, MatchArm, Value}; + +// --- fold_expr_let unit tests --- + +#[test] +fn single_binding_inlined() { + // Let { bindings: [I64(42)], body: Arg(0) } → I64(42) + let mut expr_let = ExprLet { + bindings: vec![Expr::from(42i64)], + body: Box::new(Expr::arg(0)), + }; + + let result = fold_expr_let(&mut expr_let); + assert_eq!(result, Some(Expr::from(42i64))); +} + +#[test] +fn multiple_bindings_inlined() { + // Let { bindings: [I64(1), I64(2)], body: Record([Arg(1), Arg(0)]) } + // → Record([I64(2), I64(1)]) + let mut expr_let = ExprLet { + bindings: vec![Expr::from(1i64), Expr::from(2i64)], + body: Box::new(Expr::record([Expr::arg(1), Expr::arg(0)])), + }; + + let result = fold_expr_let(&mut expr_let); + assert_eq!( + result, + Some(Expr::record([Expr::from(2i64), Expr::from(1i64)])) + ); +} + +#[test] +fn unstable_binding_not_inlined() { + // Binding contains Default (unstable) → no inlining + let mut expr_let = ExprLet { + bindings: vec![Expr::Default], + body: Box::new(Expr::arg(0)), + }; + + let result = fold_expr_let(&mut expr_let); + assert!(result.is_none()); +} + +#[test] +fn body_with_match_inlined() { + // The real-world pattern: nullable relation projection. + // Let { bindings: [I64(5)], body: Match(Arg(0), [Null → Null], Arg(0)) } + // → Match(I64(5), [Null → Null], I64(5)) + let mut expr_let = ExprLet { + bindings: vec![Expr::from(5i64)], + body: Box::new(Expr::match_expr( + Expr::arg(0), + vec![MatchArm { + pattern: Value::Null, + expr: Expr::null(), + }], + Expr::arg(0), + )), + }; + + let result = fold_expr_let(&mut expr_let); + let expected = Expr::match_expr( + Expr::from(5i64), + vec![MatchArm { + pattern: Value::Null, + expr: Expr::null(), + }], + Expr::from(5i64), + ); + assert_eq!(result, Some(expected)); +} + +#[test] +fn outer_arg_nesting_decremented() { + // Body references both the Let binding (nesting=0) and an outer scope + // (nesting=1). After inlining, the outer ref should become nesting=0. + // Let { bindings: [I64(1)], body: Record([Arg(pos=0,nest=0), Arg(pos=0,nest=1)]) } + // → Record([I64(1), Arg(pos=0,nest=0)]) + let mut expr_let = ExprLet { + bindings: vec![Expr::from(1i64)], + body: Box::new(Expr::record([ + Expr::arg(0), + Expr::Arg(stmt::ExprArg { + position: 0, + nesting: 1, + }), + ])), + }; + + let result = fold_expr_let(&mut expr_let); + assert_eq!(result, Some(Expr::record([Expr::from(1i64), Expr::arg(0)]))); +} + +// --- end-to-end tests through fold_stmt --- + +#[test] +fn let_inlined_through_fold_stmt() { + // End-to-end: fold_stmt should inline the Let. + let mut expr = Expr::Let(ExprLet { + bindings: vec![Expr::from(42i64)], + body: Box::new(Expr::arg(0)), + }); + + fold::fold_stmt(&mut expr); + assert_eq!(expr, Expr::from(42i64)); +} + +#[test] +fn nested_let_inlined_bottom_up() { + // Inner Let is inlined first (bottom-up), then the outer Let. + // Outer: Let { bindings: [I64(10)], body: Let { bindings: [Arg(0)], body: Arg(0) } } + // After inner inlining: Let { bindings: [I64(10)], body: Arg(0) } + // After outer inlining: I64(10) + let inner = Expr::Let(ExprLet { + bindings: vec![Expr::arg(0)], + body: Box::new(Expr::arg(0)), + }); + let mut expr = Expr::Let(ExprLet { + bindings: vec![Expr::from(10i64)], + body: Box::new(inner), + }); + + fold::fold_stmt(&mut expr); + assert_eq!(expr, Expr::from(10i64)); +} + +#[test] +fn let_with_match_fully_simplified() { + // Let { bindings: [Null], body: Match(Arg(0), [Null→Null], Arg(0)) } + // Step 1 (Let inlining): Match(Null, [Null→Null], Null) + // Step 2 (Match folding): Null + // + // The fold visitor recurses on the replacement after a rule fires + // (`fold.rs::visit_expr_mut`), so the new Match — now with a constant + // subject — gets a second pass that folds it to Null. + let mut expr = Expr::Let(ExprLet { + bindings: vec![Expr::null()], + body: Box::new(Expr::match_expr( + Expr::arg(0), + vec![MatchArm { + pattern: Value::Null, + expr: Expr::null(), + }], + Expr::arg(0), + )), + }); + + fold::fold_stmt(&mut expr); + assert_eq!(expr, Expr::null()); +} diff --git a/packages/toasty/src/engine/fold/tests/expr_list.rs b/packages/toasty/src/engine/fold/tests/expr_list.rs new file mode 100644 index 00000000000..d3a54634998 --- /dev/null +++ b/packages/toasty/src/engine/fold/tests/expr_list.rs @@ -0,0 +1,81 @@ +use crate::engine::fold::expr_list::fold_expr_list; +use toasty_core::stmt::{Expr, ExprList, Value}; + +#[test] +fn all_const_values_become_value_list() { + // `list([1, 2, 3]) → [1, 2, 3]` + let mut expr = ExprList { + items: vec![ + Expr::Value(Value::from(1i64)), + Expr::Value(Value::from(2i64)), + Expr::Value(Value::from(3i64)), + ], + }; + let result = fold_expr_list(&mut expr); + + assert!(result.is_some()); + let Expr::Value(Value::List(list)) = result.unwrap() else { + panic!("expected result to be a `Value::List`"); + }; + assert_eq!(list.len(), 3); + assert_eq!(list[0], Value::from(1i64)); + assert_eq!(list[1], Value::from(2i64)); + assert_eq!(list[2], Value::from(3i64)); +} + +#[test] +fn mixed_types_in_list() { + // `list(["hello", 42, true]) → ["hello", 42, true]` + let mut expr = ExprList { + items: vec![ + Expr::Value(Value::from("hello")), + Expr::Value(Value::from(42i64)), + Expr::Value(Value::from(true)), + ], + }; + let result = fold_expr_list(&mut expr); + + assert!(result.is_some()); + let Expr::Value(Value::List(list)) = result.unwrap() else { + panic!("expected result to be a `Value::List`"); + }; + assert_eq!(list.len(), 3); + assert_eq!(list[0], Value::from("hello")); + assert_eq!(list[1], Value::from(42i64)); + assert_eq!(list[2], Value::from(true)); +} + +#[test] +fn empty_list_simplified() { + // `list([]) → []` + let mut expr = ExprList { items: vec![] }; + let result = fold_expr_list(&mut expr); + + assert!(result.is_some()); + let Expr::Value(Value::List(list)) = result.unwrap() else { + panic!("expected result to be a `Value::List`"); + }; + assert!(list.is_empty()); +} + +#[test] +fn non_const_not_simplified() { + // `list([1, arg(0)])`, non-constant, not simplified + let mut expr = ExprList { + items: vec![Expr::Value(Value::from(1i64)), Expr::arg(0)], + }; + let result = fold_expr_list(&mut expr); + + assert!(result.is_none()); +} + +#[test] +fn list_with_error_item_not_simplified() { + // `list([1, error("boom")])` — error is not a Value, so not folded + let mut expr = ExprList { + items: vec![Expr::Value(Value::from(1i64)), Expr::error("boom")], + }; + let result = fold_expr_list(&mut expr); + + assert!(result.is_none()); +} diff --git a/packages/toasty/src/engine/fold/tests/expr_match.rs b/packages/toasty/src/engine/fold/tests/expr_match.rs new file mode 100644 index 00000000000..16b587aa9b7 --- /dev/null +++ b/packages/toasty/src/engine/fold/tests/expr_match.rs @@ -0,0 +1,182 @@ +use crate::engine::fold::expr_match::fold_expr_match; +use toasty_core::stmt::{Expr, ExprMatch, MatchArm, Value}; + +#[test] +fn constant_subject_matches_arm() { + // `match I64(1) { 0 => "a", 1 => "b" }` → `"b"` + let mut expr = ExprMatch { + subject: Box::new(Expr::from(1i64)), + arms: vec![ + MatchArm { + pattern: Value::from(0i64), + expr: Expr::from("a"), + }, + MatchArm { + pattern: Value::from(1i64), + expr: Expr::from("b"), + }, + ], + else_expr: Box::new(Expr::null()), + }; + + let result = fold_expr_match(&mut expr); + + assert!(matches!(result, Some(Expr::Value(Value::String(ref s))) if s == "b")); +} + +#[test] +fn constant_subject_no_matching_arm_folds_to_else() { + // `match I64(99) { 0 => "a", 1 => "b" else => null }` — no arm matches, + // folds to else + let mut expr = ExprMatch { + subject: Box::new(Expr::from(99i64)), + arms: vec![ + MatchArm { + pattern: Value::from(0i64), + expr: Expr::from("a"), + }, + MatchArm { + pattern: Value::from(1i64), + expr: Expr::from("b"), + }, + ], + else_expr: Box::new(Expr::null()), + }; + + let result = fold_expr_match(&mut expr); + + assert!(matches!(result, Some(Expr::Value(Value::Null)))); +} + +#[test] +fn non_constant_subject_not_simplified() { + // Non-constant subject → no fold. + let mut expr = ExprMatch { + subject: Box::new(Expr::arg(0)), + arms: vec![ + MatchArm { + pattern: Value::from(0i64), + expr: Expr::from("a"), + }, + MatchArm { + pattern: Value::from(1i64), + expr: Expr::from("b"), + }, + ], + else_expr: Box::new(Expr::null()), + }; + + let result = fold_expr_match(&mut expr); + + assert!(result.is_none()); +} + +#[test] +fn uniform_arms_and_else_folds_to_single_expr() { + // `match arg(0) { 0 => arg(1), 1 => arg(1) } else => arg(1)` → `arg(1)` + let mut expr = ExprMatch { + subject: Box::new(Expr::arg(0)), + arms: vec![ + MatchArm { + pattern: Value::from(0i64), + expr: Expr::arg(1), + }, + MatchArm { + pattern: Value::from(1i64), + expr: Expr::arg(1), + }, + ], + else_expr: Box::new(Expr::arg(1)), + }; + + let result = fold_expr_match(&mut expr); + + assert_eq!(result, Some(Expr::arg(1))); +} + +#[test] +fn uniform_arms_but_different_else_not_simplified() { + // `match arg(0) { 0 => arg(1), 1 => arg(1) } else => null` — arms match + // but else differs, so the match cannot be folded away. + let mut expr = ExprMatch { + subject: Box::new(Expr::arg(0)), + arms: vec![ + MatchArm { + pattern: Value::from(0i64), + expr: Expr::arg(1), + }, + MatchArm { + pattern: Value::from(1i64), + expr: Expr::arg(1), + }, + ], + else_expr: Box::new(Expr::null()), + }; + + let result = fold_expr_match(&mut expr); + + assert!(result.is_none()); +} + +#[test] +fn non_uniform_arms_not_simplified() { + // `match arg(0) { 0 => arg(1), 1 => arg(2) }` — different arm exprs, + // no fold. + let mut expr = ExprMatch { + subject: Box::new(Expr::arg(0)), + arms: vec![ + MatchArm { + pattern: Value::from(0i64), + expr: Expr::arg(1), + }, + MatchArm { + pattern: Value::from(1i64), + expr: Expr::arg(2), + }, + ], + else_expr: Box::new(Expr::null()), + }; + + let result = fold_expr_match(&mut expr); + + assert!(result.is_none()); +} + +#[test] +fn empty_arms_not_simplified() { + // No arms → cannot apply uniform-arms rule (no first arm to compare to). + let mut expr = ExprMatch { + subject: Box::new(Expr::arg(0)), + arms: vec![], + else_expr: Box::new(Expr::null()), + }; + + let result = fold_expr_match(&mut expr); + + assert!(result.is_none()); +} + +#[test] +fn false_arm_expr_not_dropped() { + // `arm.expr = false` is a return *value*, not a filter predicate. + // The arm must be kept so that subj=0 returns `false`, not `null`. + let mut expr = ExprMatch { + subject: Box::new(Expr::arg(0)), + arms: vec![ + MatchArm { + pattern: Value::from(0i64), + expr: Expr::FALSE, + }, + MatchArm { + pattern: Value::from(1i64), + expr: Expr::from("b"), + }, + ], + else_expr: Box::new(Expr::null()), + }; + + let result = fold_expr_match(&mut expr); + + assert!(result.is_none()); + assert_eq!(expr.arms.len(), 2, "false-valued arm must not be dropped"); +} diff --git a/packages/toasty/src/engine/fold/tests/expr_not.rs b/packages/toasty/src/engine/fold/tests/expr_not.rs new file mode 100644 index 00000000000..f0e71323fa9 --- /dev/null +++ b/packages/toasty/src/engine/fold/tests/expr_not.rs @@ -0,0 +1,291 @@ +use crate::engine::fold::expr_not::fold_expr_not; +use toasty_core::stmt::{BinaryOp, Expr, ExprBinaryOp, ExprNot, Value}; + +fn not_expr(expr: Expr) -> ExprNot { + ExprNot { + expr: Box::new(expr), + } +} + +// Double negation elimination tests + +#[test] +fn double_negation_eliminated() { + // `not(not(true))` → `true` + let inner = Expr::not(Expr::Value(Value::Bool(true))); + let mut outer = not_expr(inner); + + let result = fold_expr_not(&mut outer); + + assert!(matches!(result, Some(Expr::Value(Value::Bool(true))))); +} + +#[test] +fn triple_negation_reduces_to_single() { + // `not(not(not(true)))` → `not(true)` + let inner = Expr::not(Expr::not(Expr::Value(Value::Bool(true)))); + let mut outer = not_expr(inner); + + let result = fold_expr_not(&mut outer); + assert!(matches!(result, Some(Expr::Not(_)))); +} + +// Constant folding tests + +#[test] +fn not_true_becomes_false() { + // `not(true)` → `false` + let mut expr = not_expr(Expr::Value(Value::Bool(true))); + + let result = fold_expr_not(&mut expr); + + assert!(matches!(result, Some(Expr::Value(Value::Bool(false))))); +} + +#[test] +fn not_false_becomes_true() { + // `not(false)` → `true` + let mut expr = not_expr(Expr::Value(Value::Bool(false))); + + let result = fold_expr_not(&mut expr); + + assert!(matches!(result, Some(Expr::Value(Value::Bool(true))))); +} + +#[test] +fn not_null_becomes_null() { + // `not(null)` → `null` + let mut expr = not_expr(Expr::null()); + + let result = fold_expr_not(&mut expr); + + assert!(matches!(result, Some(Expr::Value(Value::Null)))); +} + +// Negation of comparison tests + +#[test] +fn not_eq_becomes_ne() { + // `not(1 = 2)` → `1 != 2` + let mut expr = not_expr(Expr::BinaryOp(ExprBinaryOp { + lhs: Box::new(Expr::Value(Value::from(1i64))), + op: BinaryOp::Eq, + rhs: Box::new(Expr::Value(Value::from(2i64))), + })); + + let result = fold_expr_not(&mut expr); + + let Some(Expr::BinaryOp(binary_op)) = result else { + panic!("expected `BinaryOp`"); + }; + assert_eq!(binary_op.op, BinaryOp::Ne); + assert_eq!(*binary_op.lhs, Expr::Value(Value::from(1i64))); + assert_eq!(*binary_op.rhs, Expr::Value(Value::from(2i64))); +} + +#[test] +fn not_ne_becomes_eq() { + // `not(1 != 2)` → `1 = 2` + let mut expr = not_expr(Expr::BinaryOp(ExprBinaryOp { + lhs: Box::new(Expr::Value(Value::from(1i64))), + op: BinaryOp::Ne, + rhs: Box::new(Expr::Value(Value::from(2i64))), + })); + + let result = fold_expr_not(&mut expr); + + let Some(Expr::BinaryOp(binary_op)) = result else { + panic!("expected `BinaryOp`"); + }; + assert_eq!(binary_op.op, BinaryOp::Eq); + assert_eq!(*binary_op.lhs, Expr::Value(Value::from(1i64))); + assert_eq!(*binary_op.rhs, Expr::Value(Value::from(2i64))); +} + +#[test] +fn not_lt_becomes_ge() { + // `not(1 < 2)` → `1 >= 2` + let mut expr = not_expr(Expr::BinaryOp(ExprBinaryOp { + lhs: Box::new(Expr::Value(Value::from(1i64))), + op: BinaryOp::Lt, + rhs: Box::new(Expr::Value(Value::from(2i64))), + })); + + let result = fold_expr_not(&mut expr); + + let Some(Expr::BinaryOp(binary_op)) = result else { + panic!("expected `BinaryOp`"); + }; + assert_eq!(binary_op.op, BinaryOp::Ge); + assert_eq!(*binary_op.lhs, Expr::Value(Value::from(1i64))); + assert_eq!(*binary_op.rhs, Expr::Value(Value::from(2i64))); +} + +#[test] +fn not_ge_becomes_lt() { + // `not(1 >= 2)` → `1 < 2` + let mut expr = not_expr(Expr::BinaryOp(ExprBinaryOp { + lhs: Box::new(Expr::Value(Value::from(1i64))), + op: BinaryOp::Ge, + rhs: Box::new(Expr::Value(Value::from(2i64))), + })); + + let result = fold_expr_not(&mut expr); + + let Some(Expr::BinaryOp(binary_op)) = result else { + panic!("expected `BinaryOp`"); + }; + assert_eq!(binary_op.op, BinaryOp::Lt); + assert_eq!(*binary_op.lhs, Expr::Value(Value::from(1i64))); + assert_eq!(*binary_op.rhs, Expr::Value(Value::from(2i64))); +} + +#[test] +fn not_gt_becomes_le() { + // `not(1 > 2)` → `1 <= 2` + let mut expr = not_expr(Expr::BinaryOp(ExprBinaryOp { + lhs: Box::new(Expr::Value(Value::from(1i64))), + op: BinaryOp::Gt, + rhs: Box::new(Expr::Value(Value::from(2i64))), + })); + + let result = fold_expr_not(&mut expr); + + let Some(Expr::BinaryOp(binary_op)) = result else { + panic!("expected `BinaryOp`"); + }; + assert_eq!(binary_op.op, BinaryOp::Le); + assert_eq!(*binary_op.lhs, Expr::Value(Value::from(1i64))); + assert_eq!(*binary_op.rhs, Expr::Value(Value::from(2i64))); +} + +#[test] +fn not_le_becomes_gt() { + // `not(1 <= 2)` → `1 > 2` + let mut expr = not_expr(Expr::BinaryOp(ExprBinaryOp { + lhs: Box::new(Expr::Value(Value::from(1i64))), + op: BinaryOp::Le, + rhs: Box::new(Expr::Value(Value::from(2i64))), + })); + + let result = fold_expr_not(&mut expr); + + let Some(Expr::BinaryOp(binary_op)) = result else { + panic!("expected `BinaryOp`"); + }; + assert_eq!(binary_op.op, BinaryOp::Gt); + assert_eq!(*binary_op.lhs, Expr::Value(Value::from(1i64))); + assert_eq!(*binary_op.rhs, Expr::Value(Value::from(2i64))); +} + +// De Morgan's law tests + +#[test] +fn not_and_becomes_or_of_nots() { + // `not(a and b)` → `not(a) or not(b)` + let mut expr = not_expr(Expr::and(Expr::arg(0), Expr::arg(1))); + + let result = fold_expr_not(&mut expr); + + let Some(Expr::Or(or_expr)) = result else { + panic!("expected `Or`"); + }; + assert_eq!(or_expr.operands.len(), 2); + assert!(matches!(&or_expr.operands[0], Expr::Not(_))); + assert!(matches!(&or_expr.operands[1], Expr::Not(_))); +} + +#[test] +fn not_or_becomes_and_of_nots() { + // `not(a or b)` → `not(a) and not(b)` + let mut expr = not_expr(Expr::or(Expr::arg(0), Expr::arg(1))); + + let result = fold_expr_not(&mut expr); + + let Some(Expr::And(and_expr)) = result else { + panic!("expected `And`"); + }; + assert_eq!(and_expr.operands.len(), 2); + assert!(matches!(&and_expr.operands[0], Expr::Not(_))); + assert!(matches!(&and_expr.operands[1], Expr::Not(_))); +} + +#[test] +fn not_and_with_three_operands() { + // `not(a and b and c)` → `not(a) or not(b) or not(c)` + let mut expr = not_expr(Expr::and_from_vec(vec![ + Expr::arg(0), + Expr::arg(1), + Expr::arg(2), + ])); + + let result = fold_expr_not(&mut expr); + + let Some(Expr::Or(or_expr)) = result else { + panic!("expected `Or`"); + }; + assert_eq!(or_expr.operands.len(), 3); + for operand in &or_expr.operands { + assert!(matches!(operand, Expr::Not(_))); + } +} + +#[test] +fn not_or_with_three_operands() { + // `not(a or b or c)` → `not(a) and not(b) and not(c)` + let mut expr = not_expr(Expr::or_from_vec(vec![ + Expr::arg(0), + Expr::arg(1), + Expr::arg(2), + ])); + + let result = fold_expr_not(&mut expr); + + let Some(Expr::And(and_expr)) = result else { + panic!("expected `And`"); + }; + assert_eq!(and_expr.operands.len(), 3); + for operand in &and_expr.operands { + assert!(matches!(operand, Expr::Not(_))); + } +} + +// NOT IN tests + +#[test] +fn not_in_empty_list_becomes_true() { + // `not(x in ())` → `true` + let in_list_expr = Expr::in_list(Expr::arg(0), Expr::list::(vec![])); + let mut expr = not_expr(in_list_expr); + + let result = fold_expr_not(&mut expr); + + assert!(result.is_some()); + assert!(result.unwrap().is_true()); +} + +#[test] +fn not_in_non_empty_list_not_simplified() { + // `not(x in (1, 2))` → not simplified directly by fold_expr_not + let in_list_expr = Expr::in_list( + Expr::arg(0), + Expr::list(vec![Value::from(1i64), Value::from(2i64)]), + ); + let mut expr = not_expr(in_list_expr); + + let result = fold_expr_not(&mut expr); + + assert!(result.is_none()); +} + +// Error tests + +#[test] +fn not_error_not_simplified() { + // `not(error("boom"))` → not simplified (error is not a value/binary_op/and/or/in_list) + let mut expr = not_expr(Expr::error("boom")); + + let result = fold_expr_not(&mut expr); + + assert!(result.is_none()); +} diff --git a/packages/toasty/src/engine/fold/tests/expr_or.rs b/packages/toasty/src/engine/fold/tests/expr_or.rs new file mode 100644 index 00000000000..4e77bffb5ed --- /dev/null +++ b/packages/toasty/src/engine/fold/tests/expr_or.rs @@ -0,0 +1,250 @@ +use crate::engine::fold::expr_or::fold_expr_or; +use toasty_core::stmt::{Expr, ExprOr}; + +/// Builds `or(a, or(b, c))`, a nested OR structure for testing flattening. +fn nested_or(a: Expr, b: Expr, c: Expr) -> ExprOr { + ExprOr { + operands: vec![ + a, + Expr::Or(ExprOr { + operands: vec![b, c], + }), + ], + } +} + +#[test] +fn flatten_all_symbolic() { + // `or(A, or(B, C)) → or(A, B, C)` + let mut expr = nested_or(Expr::arg(0), Expr::arg(1), Expr::arg(2)); + let result = fold_expr_or(&mut expr); + + assert!(result.is_none()); // Modified in place + assert_eq!(expr.operands.len(), 3); + assert_eq!(expr.operands[0], Expr::arg(0)); + assert_eq!(expr.operands[1], Expr::arg(1)); + assert_eq!(expr.operands[2], Expr::arg(2)); +} + +#[test] +fn flatten_with_false_in_outer() { + // `or(false, or(B, C)) → or(B, C)` + let mut expr = nested_or(false.into(), Expr::arg(1), Expr::arg(2)); + let result = fold_expr_or(&mut expr); + + assert!(result.is_none()); + assert_eq!(expr.operands.len(), 2); + assert_eq!(expr.operands[0], Expr::arg(1)); + assert_eq!(expr.operands[1], Expr::arg(2)); +} + +#[test] +fn flatten_with_false_in_nested_first() { + // `or(A, or(false, C)) → or(A, C)` + let mut expr = nested_or(Expr::arg(0), false.into(), Expr::arg(2)); + let result = fold_expr_or(&mut expr); + + assert!(result.is_none()); + assert_eq!(expr.operands.len(), 2); + assert_eq!(expr.operands[0], Expr::arg(0)); + assert_eq!(expr.operands[1], Expr::arg(2)); +} + +#[test] +fn flatten_outer_false_nested_one_false() { + // `or(false, or(false, C)) → C` + let mut expr = nested_or(false.into(), false.into(), Expr::arg(2)); + let result = fold_expr_or(&mut expr); + + assert!(result.is_some()); + assert_eq!(result.unwrap(), Expr::arg(2)); +} + +#[test] +fn flatten_outer_symbolic_nested_all_false() { + // `or(A, or(false, false)) → A` + let mut expr = nested_or(Expr::arg(0), false.into(), false.into()); + let result = fold_expr_or(&mut expr); + + assert!(result.is_some()); + assert_eq!(result.unwrap(), Expr::arg(0)); +} + +#[test] +fn flatten_all_false() { + // `or(false, or(false, false)) → false` + let mut expr = nested_or(false.into(), false.into(), false.into()); + let result = fold_expr_or(&mut expr); + + assert!(result.is_some()); + assert!(result.unwrap().is_false()); +} + +#[test] +fn flatten_with_true_in_outer() { + // `or(true, or(B, C)) → true` + let mut expr = nested_or(true.into(), Expr::arg(1), Expr::arg(2)); + let result = fold_expr_or(&mut expr); + + assert!(result.is_some()); + assert!(result.unwrap().is_true()); +} + +#[test] +fn flatten_with_true_in_nested() { + // `or(A, or(true, C)) → true` + let mut expr = nested_or(Expr::arg(0), true.into(), Expr::arg(2)); + let result = fold_expr_or(&mut expr); + + assert!(result.is_some()); + assert!(result.unwrap().is_true()); +} + +#[test] +fn single_operand_unwrapped() { + // `or(arg(0)) → arg(0)` + let mut expr = ExprOr { + operands: vec![Expr::arg(0)], + }; + let result = fold_expr_or(&mut expr); + + assert!(result.is_some()); + assert_eq!(result.unwrap(), Expr::arg(0)); +} + +#[test] +fn empty_after_removing_false() { + // `or(false, false) → false` + let mut expr = ExprOr { + operands: vec![false.into(), false.into()], + }; + let result = fold_expr_or(&mut expr); + + assert!(result.is_some()); + assert!(result.unwrap().is_false()); +} + +// Null propagation tests + +#[test] +fn null_or_null_becomes_null() { + // `null or null` → `null` + let mut expr = ExprOr { + operands: vec![Expr::null(), Expr::null()], + }; + let result = fold_expr_or(&mut expr); + + assert!(result.is_some()); + assert!(result.unwrap().is_value_null()); +} + +#[test] +fn null_or_false_becomes_null() { + // `null or false` → `null` (false is removed, leaving only null) + let mut expr = ExprOr { + operands: vec![Expr::null(), false.into()], + }; + let result = fold_expr_or(&mut expr); + + assert!(result.is_some()); + assert!(result.unwrap().is_value_null()); +} + +#[test] +fn false_or_null_becomes_null() { + // `false or null` → `null` (false is removed, leaving only null) + let mut expr = ExprOr { + operands: vec![false.into(), Expr::null()], + }; + let result = fold_expr_or(&mut expr); + + assert!(result.is_some()); + assert!(result.unwrap().is_value_null()); +} + +#[test] +fn null_or_true_becomes_true() { + // `null or true` → `true` + let mut expr = ExprOr { + operands: vec![Expr::null(), true.into()], + }; + let result = fold_expr_or(&mut expr); + + assert!(result.is_some()); + assert!(result.unwrap().is_true()); +} + +#[test] +fn null_or_symbolic_not_simplified() { + // `null or a` → no change (symbolic operand present) + let mut expr = ExprOr { + operands: vec![Expr::null(), Expr::arg(0)], + }; + let result = fold_expr_or(&mut expr); + + assert!(result.is_none()); + assert_eq!(expr.operands.len(), 2); +} + +#[test] +fn multiple_nulls_become_null() { + // `null or null or null` → `null` + let mut expr = ExprOr { + operands: vec![Expr::null(), Expr::null(), Expr::null()], + }; + let result = fold_expr_or(&mut expr); + + assert!(result.is_some()); + assert!(result.unwrap().is_value_null()); +} + +#[test] +fn multiple_nulls_and_false_becomes_null() { + // `null or false or null or false` → `null` + let mut expr = ExprOr { + operands: vec![Expr::null(), false.into(), Expr::null(), false.into()], + }; + let result = fold_expr_or(&mut expr); + + assert!(result.is_some()); + assert!(result.unwrap().is_value_null()); +} + +// Error operand tests + +#[test] +fn error_operand_preserved_in_or() { + // `or(error("boom"), arg(0))` → no simplification (error is not true/false/null) + let mut expr = ExprOr { + operands: vec![Expr::error("boom"), Expr::arg(0)], + }; + let result = fold_expr_or(&mut expr); + + assert!(result.is_none()); + assert_eq!(expr.operands.len(), 2); + assert!(matches!(&expr.operands[0], Expr::Error(_))); +} + +#[test] +fn error_or_false_keeps_error() { + // `or(error("boom"), false)` → `error("boom")` (false is removed, error remains) + let mut expr = ExprOr { + operands: vec![Expr::error("boom"), false.into()], + }; + let result = fold_expr_or(&mut expr); + + assert!(result.is_some()); + assert!(matches!(&result.unwrap(), Expr::Error(e) if e.message == "boom")); +} + +#[test] +fn error_or_true_becomes_true() { + // `or(error("boom"), true)` → `true` (true short-circuits OR) + let mut expr = ExprOr { + operands: vec![Expr::error("boom"), true.into()], + }; + let result = fold_expr_or(&mut expr); + + assert!(result.is_some()); + assert!(result.unwrap().is_true()); +} diff --git a/packages/toasty/src/engine/fold/tests/expr_record.rs b/packages/toasty/src/engine/fold/tests/expr_record.rs new file mode 100644 index 00000000000..07a1ac00f06 --- /dev/null +++ b/packages/toasty/src/engine/fold/tests/expr_record.rs @@ -0,0 +1,59 @@ +use crate::engine::fold::expr_record::fold_expr_record; +use toasty_core::stmt::{Expr, ExprRecord, Value}; + +#[test] +fn all_const_values_become_value_record() { + // `record([1, "hello", true]) → {1, "hello", true}` + let mut expr = ExprRecord { + fields: vec![ + Expr::Value(Value::from(1i64)), + Expr::Value(Value::from("hello")), + Expr::Value(Value::from(true)), + ], + }; + let result = fold_expr_record(&mut expr); + + assert!(result.is_some()); + let Expr::Value(Value::Record(record)) = result.unwrap() else { + panic!("expected result to be a `Value::Record`"); + }; + assert_eq!(record.len(), 3); + assert_eq!(record[0], Value::from(1i64)); + assert_eq!(record[1], Value::from("hello")); + assert_eq!(record[2], Value::from(true)); +} + +#[test] +fn empty_record_simplified() { + // `record([]) → {}` + let mut expr = ExprRecord { fields: vec![] }; + let result = fold_expr_record(&mut expr); + + assert!(result.is_some()); + let Expr::Value(Value::Record(record)) = result.unwrap() else { + panic!("expected result to be a `Value::Record`"); + }; + assert!(record.is_empty()); +} + +#[test] +fn non_const_not_simplified() { + // `record([1, arg(0)])`, non-constant, not simplified + let mut expr = ExprRecord { + fields: vec![Expr::Value(Value::from(1i64)), Expr::arg(0)], + }; + let result = fold_expr_record(&mut expr); + + assert!(result.is_none()); +} + +#[test] +fn record_with_error_field_not_simplified() { + // `record([1, error("boom")])` — error is not a Value, so not folded + let mut expr = ExprRecord { + fields: vec![Expr::Value(Value::from(1i64)), Expr::error("boom")], + }; + let result = fold_expr_record(&mut expr); + + assert!(result.is_none()); +} diff --git a/packages/toasty/src/engine/fold/tests/prop_idempotence.rs b/packages/toasty/src/engine/fold/tests/prop_idempotence.rs new file mode 100644 index 00000000000..080a3678ee7 --- /dev/null +++ b/packages/toasty/src/engine/fold/tests/prop_idempotence.rs @@ -0,0 +1,86 @@ +use crate::engine::fold; +use proptest::prelude::*; +use toasty_core::stmt::{BinaryOp, Expr, ExprAny, ExprBinaryOp, ExprInList, ExprList, Value}; + +fn arb_i64_value() -> impl Strategy { + (-100i64..=100).prop_map(|n| Expr::Value(Value::I64(n))) +} + +fn arb_i64_raw() -> impl Strategy { + -100i64..=100 +} + +fn arb_binary_op() -> impl Strategy { + prop_oneof![ + Just(BinaryOp::Eq), + Just(BinaryOp::Ne), + Just(BinaryOp::Lt), + Just(BinaryOp::Le), + Just(BinaryOp::Gt), + Just(BinaryOp::Ge), + ] +} + +/// Generates a pure constant boolean expression. Mirrors the generator in +/// `simplify::tests::prop_const` so fold idempotence is exercised on the +/// same shapes that the simplifier already handles. +fn arb_bool_expr() -> impl Strategy { + let leaf = prop_oneof![ + Just(Expr::from(true)), + Just(Expr::from(false)), + (arb_i64_value(), arb_binary_op(), arb_i64_value()).prop_map(|(lhs, op, rhs)| { + Expr::BinaryOp(ExprBinaryOp { + lhs: Box::new(lhs), + op, + rhs: Box::new(rhs), + }) + }), + arb_i64_value().prop_map(Expr::is_null), + Just(Expr::is_null(Expr::null())), + (arb_i64_value(), prop::collection::vec(arb_i64_raw(), 0..=3)).prop_map( + |(expr, values)| { + let list = Expr::Value(Value::List(values.into_iter().map(Value::I64).collect())); + Expr::InList(ExprInList { + expr: Box::new(expr), + list: Box::new(list), + }) + } + ), + ]; + + leaf.prop_recursive(4, 64, 4, |inner| { + prop_oneof![ + inner.clone().prop_map(Expr::not), + prop::collection::vec(inner.clone(), 2..=4).prop_map(Expr::and_from_vec), + prop::collection::vec(inner.clone(), 2..=4).prop_map(Expr::or_from_vec), + prop::collection::vec(inner.clone(), 1..=4).prop_map(|items| { + Expr::Any(ExprAny { + expr: Box::new(Expr::List(ExprList { items })), + }) + }), + (inner.clone(), any::(), any::()).prop_map(|(expr, b, use_eq)| { + let op = if use_eq { BinaryOp::Eq } else { BinaryOp::Ne }; + Expr::binary_op(expr, op, Expr::from(b)) + }), + ] + }) +} + +proptest! { + /// `fold(fold(x)) == fold(x)`. Required for fold to be safely composed + /// from `lower`, `simplify`, and `exec_statement` without surprises. + #[test] + fn fold_is_idempotent(expr in arb_bool_expr()) { + let once = { + let mut e = expr.clone(); + fold::fold_stmt(&mut e); + e + }; + let twice = { + let mut e = once.clone(); + fold::fold_stmt(&mut e); + e + }; + prop_assert!(once.is_equivalent_to(&twice), "once = {once:?}, twice = {twice:?}"); + } +} diff --git a/packages/toasty/src/engine/hir.rs b/packages/toasty/src/engine/hir.rs new file mode 100644 index 00000000000..90281fda6b3 --- /dev/null +++ b/packages/toasty/src/engine/hir.rs @@ -0,0 +1,306 @@ +use std::{ + cell::{Cell, OnceCell}, + ops, +}; + +use hashbrown::HashMap; + +use index_vec::IndexVec; +use indexmap::{IndexMap, IndexSet}; +use toasty_core::stmt; + +use crate::engine::{SelectItems, mir}; + +/// High-level Intermediate Representation of a query. +/// +/// [`HirStatement`] is a collection of related statements with tracked +/// dependencies, produced by the lowering phase. It captures the structure of +/// a query after model-to-table transformation but before operation graph +/// construction. +/// +/// The HIR tracks which statements depend on results from others, enabling the +/// planner to determine execution order and identify opportunities for batching. +/// The dependency graph may contain cycles when preloading associations (e.g., +/// users → todos → users), which the planning phase must break when building +/// the final DAG. +#[derive(Debug)] +pub(super) struct HirStatement { + /// Storage for all statement metadata, indexed by [`StmtId`]. + store: IndexVec, +} + +/// Metadata for a single statement within the HIR. +/// +/// Each [`StatementInfo`] represents one database operation that will execute +/// as part of the query. Not every sub-expression in the original query becomes +/// a [`StatementInfo`]; only those that execute as separate database operations +/// (e.g., the root query and any `include()` subqueries). +/// +/// It tracks the statement's dependencies on other statements, arguments passed +/// from parent or child statements, and references to the MIR nodes created +/// during planning. +#[derive(Debug)] +pub(super) struct StatementInfo { + /// The lowered statement to execute. + /// + /// Initially `None`, populated during lowering after all transformations + /// are complete. Contains the table-level statement ready for planning. + pub(super) stmt: Option>, + + /// Whether cursor pagination resumed this query after an earlier page. + pub(super) has_pagination_cursor: bool, + + /// Ordering edges: statements that must execute, to the degree the + /// [`DepKind`] demands, before this one. + /// + /// These are pure ordering constraints — value flow between statements is + /// carried by [`Self::args`] and [`Self::back_refs`], never by this map. + /// One entry per target: inserting via [`StatementInfo::add_dep`] + /// upgrades an [`Effect`](DepKind::Effect) entry to + /// [`Statement`](DepKind::Statement), which subsumes it. An `IndexMap` + /// keeps iteration — and therefore planning order — deterministic. + pub(super) deps: IndexMap, + + /// Arguments that flow into this statement from other statements. + /// + /// Each [`Arg`] describes data passed from a sub-statement ([`Arg::Sub`]) or + /// referenced from a parent statement ([`Arg::Ref`]). During planning, these + /// arguments become edges in the operation graph. + pub(super) args: Vec, + + /// Column references from child statements that point back to this one. + /// + /// When a child statement references columns from this statement (via + /// [`Arg::Ref`]), those columns must be included in this statement's + /// batch-load query. The key is the child's [`StmtId`], and the value + /// tracks which columns are referenced. + pub(super) back_refs: HashMap, + + /// MIR node that executes this statement's database query. + /// + /// Set during planning when the statement is converted to an operation. + /// Used to wire up dependencies between operations. + pub(super) load_data_statement: Cell>, + + /// Items selected by the `exec_statement` operation. + /// + /// Populated during planning to track which items are fetched from the + /// database. Used to resolve column references in child statements. + pub(super) load_data_select_items: OnceCell, + + /// MIR node representing this statement's final output. + /// + /// May differ from `exec_statement` when post-processing is required + /// (e.g., filtering, projection, or nested merge). Dependent statements + /// reference this node to establish execution order. + pub(super) output: Cell>, + + /// When true, the statement is independent. An independent statement does + /// not depend on any anestors itself nor do any of its sub-dependencies. + pub(super) independent: bool, + + /// True once the statement's planning pass has started. Guards against + /// re-entrant planning when a statement is reached again through a + /// dependency cycle that is acyclic at the operation level. + pub(super) planning: Cell, +} + +index_vec::define_index_type! { + pub(crate) struct StmtId = u32; +} + +/// The kind of ordering edge between two statements. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum DepKind { + /// The target's full result must exist first. During planning the edge + /// anchors on the target's output node, keeping everything the target's + /// returning consumed ordered and reachable. Targets are never + /// ancestors. + Statement, + + /// The target's database write must have happened first. The target may + /// be an ancestor still being planned — e.g. a `belongs_to` + /// returning-load subquery ordered after its enclosing INSERTs. Such an + /// edge is cyclic at statement granularity (the ancestor's output + /// transitively consumes this statement's result) but acyclic at the + /// operation level, so it anchors on the target's data-loading node — + /// reserved up front so the reference resolves regardless of planning + /// order. + Effect, +} + +impl StatementInfo { + /// Creates a new [`StatementInfo`] with the given dependencies. + /// + /// All other fields are initialized to empty or `None` and populated + /// later during lowering and planning. + pub(super) fn new(deps: IndexMap) -> StatementInfo { + StatementInfo { + stmt: None, + has_pagination_cursor: false, + deps, + args: vec![], + back_refs: HashMap::new(), + load_data_statement: Cell::new(None), + load_data_select_items: OnceCell::new(), + output: Cell::new(None), + independent: true, + planning: Cell::new(false), + } + } + + pub(super) fn stmt(&self) -> &stmt::Statement { + self.stmt.as_deref().unwrap() + } + + /// Records an ordering edge on `target`. A [`DepKind::Statement`] edge + /// subsumes an [`DepKind::Effect`] edge to the same target, so inserting + /// the stronger kind upgrades an existing entry and inserting the weaker + /// kind over the stronger is a no-op. + pub(super) fn add_dep(&mut self, target: StmtId, kind: DepKind) { + let entry = self.deps.entry(target).or_insert(kind); + if kind == DepKind::Statement { + *entry = DepKind::Statement; + } + } +} + +/// Tracks columns referenced by a child statement. +/// +/// When a child statement (e.g., an `include()` subquery) references columns +/// from its parent, the parent must include those columns in its result set. +/// [`BackRef`] records which columns are needed and the MIR node that projects +/// them for the child's batch-load operation. +#[derive(Debug, Default)] +pub(super) struct BackRef { + /// Column expressions from this statement that are referenced by a child statement. + /// + /// When a child statement references columns from this statement (via + /// [`Arg::Ref`]), those columns must be included in this statement's + /// batch-load query. This set tracks which columns need to be loaded so they + /// can be used during nested merge. + pub(super) exprs: IndexSet, + + /// Node ID of the projection operation that extracts these back-ref columns. + /// + /// After executing this statement, a projection node is created to extract just + /// the columns needed by child statements. This projection's output is used as + /// input to the child statement's batch-load operation. + pub(super) node_id: Cell>, +} + +/// An argument that flows between statements in the HIR. +/// +/// Arguments represent data dependencies between statements. They describe how +/// results from one statement are used by another, enabling the planner to wire +/// up the operation graph correctly. +#[derive(Debug)] +pub(super) enum Arg { + /// Data from a sub-statement that feeds into this statement. + Sub { + /// The statement ID that provides the data for this argument. + stmt_id: StmtId, + + /// True when the sub-statement is used in the returning clause, false when used in filters. + /// + /// Determines how the sub-statement is handled during planning: + /// - `true`: Data is merged with parent rows via [`NestedMerge`](crate::engine::mir::NestedMerge) + /// - `false`: Data is used as input to filter expressions + returning: bool, + + /// Index in the operation's inputs list. Set during planning. + input: Cell>, + + /// Depending on the statement type, this is used in different ways. For + /// Query types, it is the index of the TableRef that will provide the + /// data for this ref. For Insert statements, it is the offset at which + /// the data should be fetched. + batch_load_index: Cell>, + }, + + /// A reference to a parent statement's columns. + Ref { + /// The expression reference relative to the target statement. + target_expr_ref: stmt::ExprReference, + + /// The parent statement that provides the data for this reference. + stmt_id: StmtId, + + /// Number of nesting levels between this statement and the referenced parent. + /// + /// A value of 1 means the immediate parent, 2 means the grandparent, etc. + nesting: usize, + + /// The MIR node input when the ref is used during the data loading phase + data_load_input: Cell>, + + /// The MIR node input when the ref is used during the returing phase + returning_input: Cell>, + + /// Depending on the statement type, this is used in different ways. For + /// Query types, it is the index of the TableRef that will provide the + /// data for this ref. For Insert statements, it is the offset at which + /// the data should be fetched. + batch_load_index: Cell>, + }, +} + +impl HirStatement { + /// Creates an empty [`HirStatement`] with no statements. + pub(super) fn new() -> HirStatement { + HirStatement { + store: IndexVec::new(), + } + } + + /// Inserts a [`StatementInfo`] and returns its assigned [`StmtId`]. + pub(super) fn insert(&mut self, info: StatementInfo) -> StmtId { + self.store.push(info) + } + + /// Creates and inserts a new [`StatementInfo`] with the given dependencies. + pub(super) fn new_statement_info(&mut self, deps: IndexMap) -> StmtId { + self.insert(StatementInfo::new(deps)) + } + + /// Returns the [`StmtId`] of the root statement. + /// + /// The root statement is always the first one inserted (index 0) and + /// represents the top-level query that produces the final result. + pub(super) fn root_id(&self) -> StmtId { + StmtId::from(0) + } + + /// Returns a reference to the root [`StatementInfo`]. + pub(super) fn root(&self) -> &StatementInfo { + let root_id = self.root_id(); + &self.store[root_id] + } + + /// Iterates all statements. + pub(super) fn statements(&self) -> impl Iterator { + self.store.iter() + } +} + +impl ops::Index for HirStatement { + type Output = StatementInfo; + + fn index(&self, index: StmtId) -> &Self::Output { + self.store.index(index) + } +} + +impl ops::IndexMut for HirStatement { + fn index_mut(&mut self, index: StmtId) -> &mut Self::Output { + self.store.index_mut(index) + } +} + +impl ops::Index<&StmtId> for HirStatement { + type Output = StatementInfo; + + fn index(&self, index: &StmtId) -> &Self::Output { + self.store.index(*index) + } +} diff --git a/packages/toasty/src/engine/index.rs b/packages/toasty/src/engine/index.rs new file mode 100644 index 00000000000..049324029d8 --- /dev/null +++ b/packages/toasty/src/engine/index.rs @@ -0,0 +1,295 @@ +mod index_match; +use index_match::{IndexColumnMatch, IndexMatch}; + +mod or_rewrite; + +mod index_plan; +pub(crate) use index_plan::IndexPlan; + +use crate::{Result, engine::Engine}; +use hashbrown::HashMap; +use toasty_core::{Schema, driver::Capability, schema::db::Index, stmt}; + +impl Engine { + pub(crate) fn plan_index_path<'a>( + &'a self, + stmt: &stmt::Statement, + ) -> Result>> { + plan_index_path(&self.schema, self.capability(), stmt) + } +} + +pub(crate) fn plan_index_path<'a>( + schema: &'a Schema, + capability: &'a Capability, + stmt: &stmt::Statement, +) -> Result>> { + let cx = stmt::ExprContext::new(schema); + let cx = cx.scope(stmt); + // Get a handle to the expression target so it can be passed into the planner + let target = cx.target(); + let stmt::ExprTarget::Table(table) = target else { + todo!("target={target:#?}") + }; + + // Get the statement filter + let filter = stmt.filter_expr_unwrap(); + + // Extract the pre-filter: the part of the filter that depends only on + // args (no table column references) and can be evaluated before issuing + // the database operation. The invariant is: + // filter == AND(pre_filter, remaining_filter) + let (pre_filter, remaining_filter) = extract_pre_filter(filter); + + let index_match = table + .indices + .iter() + .filter_map(|index| { + let mut index_match = IndexMatch { + index, + columns: index + .columns + .iter() + .map(|_| IndexColumnMatch { + exprs: HashMap::new(), + }) + .collect(), + }; + + if !index_match.match_restriction(&cx, &remaining_filter) + || index_match.columns[0].exprs.is_empty() + { + return None; + } + + Some(index_match) + }) + .min_by_key(|index_match| index_match.compute_cost(&remaining_filter)); + + let Some(index_match) = index_match else { + if capability.scan { + return Ok(None); + } + return Err(toasty_core::Error::unsupported_feature(format!( + "{} requires queries to use an index. The current filter cannot be satisfied by \ + any available index. Consider adding an index that matches your query filter, or \ + restructure the query to use indexed fields.", + capability.driver_name + ))); + }; + + let mut partition_cx = PartitionCtx { + capability, + apply_result_filter_on_results: false, + }; + + let (index_filter, result_filter) = + index_match.partition_filter(&mut partition_cx, &remaining_filter); + + // Extract literal key values before OR rewrite, while index_filter is still + // in Expr::Or form. After rewrite it becomes ANY(MAP(...)) and the Or arm + // in try_extract_key_values would no longer fire. + let key_values = try_extract_key_values(&cx, index_match.index, &index_filter); + + // For backends that do not support OR in key conditions (e.g. DynamoDB), rewrite + // any OR in the index filter to canonical ANY(MAP(...)) fan-out form. + let index_filter = if !capability.index_or_predicate { + or_rewrite::index_filter_to_any_map(index_filter) + } else { + index_filter + }; + + let index = schema.db.index(index_match.index.id); + let has_pk_keys = index.primary_key && key_values.is_some(); + + Ok(Some(IndexPlan { + // Reload the index to make lifetimes happy. + index, + index_filter, + result_filter: if result_filter.is_true() { + None + } else { + Some(result_filter) + }, + post_filter: if partition_cx.apply_result_filter_on_results { + Some(remaining_filter.clone()) + } else { + None + }, + pre_filter, + key_values, + has_pk_keys, + })) +} + +struct PartitionCtx<'a> { + capability: &'a Capability, + apply_result_filter_on_results: bool, +} + +/// Try to extract a key expression from `index_filter` for direct `GetByKey` routing. +/// +/// Returns `Some(Expr::Value(Value::List([Value::Record([...]), ...])))` when all key +/// columns have literal equality or IN predicates. Returns `Some(Expr::Arg(i))` for +/// `pk IN (arg[i])` batch-load form. Range predicates and `ANY(MAP(...))` return `None`. +/// +/// Must be called on the `index_filter` produced by `partition_filter` — before the +/// OR-rewrite step converts `Expr::Or` into `ANY(MAP(...))`. +fn try_extract_key_values( + cx: &stmt::ExprContext<'_>, + index: &Index, + index_filter: &stmt::Expr, +) -> Option { + match index_filter { + stmt::Expr::InList(in_list) => match &*in_list.list { + stmt::Expr::Arg(arg) => Some(stmt::Expr::Arg(*arg)), + stmt::Expr::Value(stmt::Value::List(items)) => { + let records = items + .iter() + .map(|item| match item { + record @ stmt::Value::Record(_) => Some(record.clone()), + // Only wrap scalar values as single-field Records for single-column + // indexes. For composite indexes the scalar covers only the partition + // key, so we cannot form a full key record. + value if index.columns.len() == 1 => { + Some(stmt::Value::Record(stmt::ValueRecord::from_vec(vec![ + value.clone(), + ]))) + } + _ => None, + }) + .collect::>>()?; + Some(stmt::Expr::Value(stmt::Value::List(records))) + } + _ => None, + }, + stmt::Expr::Or(or) => { + let mut records = vec![]; + for branch in &or.operands { + records.push(extract_key_record(cx, index, branch)?); + } + Some(stmt::Expr::Value(stmt::Value::List(records))) + } + single => { + let record = extract_key_record(cx, index, single)?; + Some(stmt::Expr::Value(stmt::Value::List(vec![record]))) + } + } +} + +/// Extract a single `Value::Record` from one equality branch of the index filter. +/// +/// - `col = literal` (single-column index) → `Value::Record([literal])` +/// - `col1 = v1 AND col2 = v2 ...` (all key columns, all equality) → `Value::Record([v1, v2, ...])` +/// - Anything else → `None` +fn extract_key_record( + cx: &stmt::ExprContext<'_>, + index: &Index, + expr: &stmt::Expr, +) -> Option { + match expr { + stmt::Expr::BinaryOp(b) if b.op.is_eq() && index.columns.len() == 1 => { + let stmt::Expr::Value(v) = &*b.rhs else { + return None; + }; + Some(stmt::Value::Record(stmt::ValueRecord::from_vec(vec![ + v.clone(), + ]))) + } + stmt::Expr::And(and) if and.operands.len() == index.columns.len() => { + let mut fields = vec![stmt::Value::Null; index.columns.len()]; + + for operand in &and.operands { + let stmt::Expr::BinaryOp(b) = operand else { + return None; + }; + if !b.op.is_eq() { + return None; + } + let stmt::Expr::Reference(expr_ref) = &*b.lhs else { + return None; + }; + let column = cx.resolve_expr_reference(expr_ref).as_column_unwrap(); + let (idx, _) = index + .columns + .iter() + .enumerate() + .find(|(_, c)| c.column == column.id)?; + let stmt::Expr::Value(v) = &*b.rhs else { + return None; + }; + fields[idx] = v.clone(); + } + + if fields.iter().any(|v| matches!(v, stmt::Value::Null)) { + return None; + } + + Some(stmt::Value::Record(stmt::ValueRecord::from_vec(fields))) + } + _ => None, + } +} + +/// Extract the args-only component of a filter expression. +/// +/// Splits the filter into `(pre_filter, remaining_filter)` such that +/// `filter == AND(pre_filter, remaining_filter)`. The `pre_filter` contains +/// only sub-expressions that can be evaluated using args alone (no table +/// column references). +fn extract_pre_filter(expr: &stmt::Expr) -> (Option, stmt::Expr) { + // Only AND nodes can be split into pre_filter and remaining components. + let stmt::Expr::And(and) = expr else { + if references_column(expr) { + return (None, expr.clone()); + } else { + return (Some(expr.clone()), true.into()); + } + }; + + let mut pre = vec![]; + let mut remaining = vec![]; + + for operand in &and.operands { + if references_column(operand) { + remaining.push(operand.clone()); + } else { + pre.push(operand.clone()); + } + } + + // If all operands are args-only, the entire AND is the pre_filter. + if remaining.is_empty() { + return (Some(expr.clone()), true.into()); + } + + let pre_filter = match pre.len() { + 0 => None, + 1 => Some(pre.into_iter().next().unwrap()), + _ => Some(stmt::ExprAnd { operands: pre }.into()), + }; + + let remaining_filter = match remaining.len() { + 1 => remaining.into_iter().next().unwrap(), + _ => stmt::ExprAnd { + operands: remaining, + } + .into(), + }; + + (pre_filter, remaining_filter) +} + +/// Returns `true` if the expression references any table column. +fn references_column(expr: &stmt::Expr) -> bool { + let mut found = false; + stmt::visit::for_each_expr(expr, |e| { + if matches!(e, stmt::Expr::Reference(stmt::ExprReference::Column(_))) { + found = true; + } + }); + found +} + +#[cfg(test)] +mod tests; diff --git a/packages/toasty/src/engine/index/index_match.rs b/packages/toasty/src/engine/index/index_match.rs new file mode 100644 index 00000000000..a037cf039fc --- /dev/null +++ b/packages/toasty/src/engine/index/index_match.rs @@ -0,0 +1,462 @@ +use hashbrown::{HashMap, hash_map}; + +use by_address::ByAddress; +use toasty_core::{schema::db::Index, stmt}; + +use crate::engine::index::PartitionCtx; + +#[derive(Debug)] +pub(super) struct IndexMatch<'stmt> { + /// The index in question + pub(super) index: &'stmt Index, + + /// Restriction matches for each column + pub(super) columns: Vec>, +} + +#[derive(Debug)] +pub(super) struct IndexColumnMatch<'stmt> { + pub(super) exprs: HashMap, ExprMatch>, +} + +#[derive(Debug, Copy, Clone)] +pub(super) struct ExprMatch { + pub(super) eq: bool, +} + +type ExprPair = (stmt::Expr, stmt::Expr); + +impl<'stmt> IndexMatch<'stmt> { + pub(super) fn match_restriction( + &mut self, + cx: &stmt::ExprContext<'_>, + expr: &'stmt stmt::Expr, + ) -> bool { + use stmt::Expr::*; + + match expr { + BinaryOp(e) => match (&*e.lhs, &*e.rhs) { + // A reference-to-reference predicate has no fixed value with + // which to probe an index. Leave it to the scan/result filter + // path instead of treating either operand as an index key. + (stmt::Expr::Reference(_), stmt::Expr::Reference(_)) => false, + (stmt::Expr::Reference(lhs @ stmt::ExprReference::Column(_)), _) => { + self.match_expr_binary_op_column(cx, lhs, expr, e.op) + } + (_, stmt::Expr::Reference(rhs @ stmt::ExprReference::Column(_))) => { + match e.op.commute() { + Some(op) => self.match_expr_binary_op_column(cx, rhs, expr, op), + // Non-commutative op (e.g. `5 - col`) cannot be normalized + // to column-on-left form; it's not an index match. + None => false, + } + } + // Neither operand is a bare column reference (e.g. + // `LENGTH(col) = 0` for `Vec::is_empty()`). This shape is not + // an index match — return false rather than failing. + _ => false, + }, + StartsWith(e) => match &*e.expr { + stmt::Expr::Reference(expr_column @ stmt::ExprReference::Column(_)) => { + // starts_with is a range-like condition: valid on sort key, not partition key + self.match_expr_binary_op_column(cx, expr_column, expr, stmt::BinaryOp::Ge) + } + _ => false, + }, + Between(e) => match &*e.expr { + stmt::Expr::Reference(expr_column @ stmt::ExprReference::Column(_)) => { + // between is a range condition: valid on sort key, not partition key + self.match_expr_binary_op_column(cx, expr_column, expr, stmt::BinaryOp::Ge) + } + _ => false, + }, + InList(e) => self.match_expr_in_list(cx, &e.expr, expr), + // `IS NOT NULL` is a range, not a key value. + IsNull(e) if e.negated => false, + IsNull(e) => match &*e.expr { + stmt::Expr::Reference(expr_column @ stmt::ExprReference::Column(_)) => { + self.match_expr_binary_op_column(cx, expr_column, expr, stmt::BinaryOp::Eq) + } + // Not a bare column reference (e.g. a projection into a + // `#[document]` column). Not an index match — return false + // rather than failing. + _ => false, + }, + And(and_exprs) => { + let matched = self.match_all_restrictions(cx, and_exprs); + + if matched { + // Union all matched columns for each operand + for column in &mut self.columns { + for and_expr in and_exprs { + if let Some(operand_match) = + column.exprs.get(&ByAddress(and_expr)).copied() + { + match column.exprs.entry(ByAddress(expr)) { + hash_map::Entry::Vacant(e) => { + e.insert(ExprMatch { + eq: operand_match.eq, + }); + } + hash_map::Entry::Occupied(mut e) if operand_match.eq => { + e.get_mut().eq = true; + } + _ => continue, + } + + if operand_match.eq { + break; + } + } + } + } + } + + matched + } + Or(or_exprs) => { + let matched_any_expr = self.match_all_restrictions(cx, or_exprs); + let mut matched_operand = false; + + // For OR expressions, we need to ensure that the index match is + // balanced for each column. To do this, if operand matched, then + // we do a deeper search to see if any index column matched with + // *all* or branches. + if matched_any_expr { + 'columns: for column in &mut self.columns { + let mut eq = true; + + for or_expr in or_exprs { + if let Some(operand_match) = column.exprs.get(&ByAddress(or_expr)) { + eq &= operand_match.eq; + } else { + continue 'columns; + } + } + + // At this point, we verified *each* operand matched a + // column, so we can consider the entire expression as + // having matched that column. + column.exprs.insert(ByAddress(expr), ExprMatch { eq }); + matched_operand = true; + } + } + + matched_operand + } + Any(expr_any) => { + // Any operates on a Map expression that evaluates to bools. + // It's similar to a dynamic OR expression. If the map expression + // matches an index, the Any expression matches as well. + // We don't need the "balanced" check like OR because each + // evaluation of the map will have the same structure. + let stmt::Expr::Map(expr_map) = &*expr_any.expr else { + todo!("expr_any.expr={:#?}", expr_any.expr); + }; + + // Try to match the map expression (the predicate applied to each element) + if self.match_restriction(cx, &expr_map.map) { + // If the map expression matched, propagate that match to this Any expression + for column in &mut self.columns { + if let Some(operand_match) = + column.exprs.get(&ByAddress(&*expr_map.map)).copied() + { + column.exprs.insert( + ByAddress(expr), + ExprMatch { + eq: operand_match.eq, + }, + ); + } + } + true + } else { + false + } + } + _ => { + // Unsupported expression type for index matching - return false + // to indicate this expression cannot be matched against an index + false + } + } + } + + /// Returns true if **any** expression in the provided list match with + /// **any** index column. + fn match_all_restrictions( + &mut self, + cx: &stmt::ExprContext<'_>, + exprs: &'stmt [stmt::Expr], + ) -> bool { + let mut matched = false; + + for expr in exprs { + matched |= self.match_restriction(cx, expr); + } + + matched + } + + fn match_expr_in_list( + &mut self, + cx: &stmt::ExprContext<'_>, + lhs: &'stmt stmt::Expr, + expr: &'stmt stmt::Expr, + ) -> bool { + match lhs { + stmt::Expr::Reference(expr_column @ stmt::ExprReference::Column(_)) => { + self.match_expr_binary_op_column(cx, expr_column, expr, stmt::BinaryOp::Eq) + } + stmt::Expr::Record(expr_record) => { + let mut matched = false; + + for sub_expr in expr_record { + let stmt::Expr::Reference(expr_column @ stmt::ExprReference::Column(_)) = + sub_expr + else { + todo!() + }; + matched |= self.match_expr_binary_op_column( + cx, + expr_column, + sub_expr, + stmt::BinaryOp::Eq, + ); + } + + if matched { + for column in &mut self.columns { + for sub_expr in expr_record { + if column.exprs.contains_key(&ByAddress(sub_expr)) { + column.exprs.insert(ByAddress(expr), ExprMatch { eq: true }); + + break; + } + } + } + } + + matched + } + _ => todo!("expr={:#?}", expr), + } + } + + fn match_expr_binary_op_column( + &mut self, + cx: &stmt::ExprContext<'_>, + expr_ref: &stmt::ExprReference, + expr: &'stmt stmt::Expr, + op: stmt::BinaryOp, + ) -> bool { + let mut matched = false; + let resolved_col = cx.resolve_expr_reference(expr_ref).as_column_unwrap(); + + for (i, index_column) in self.index.columns.iter().enumerate() { + // Check that the path matches an index column + if resolved_col.id != index_column.column { + continue; + } + + // If the index column is scoped as a partition key, then only + // equality predicates are supported. + if index_column.scope.is_partition() && !op.is_eq() { + continue; + } + + self.columns[i] + .exprs + .insert(ByAddress(expr), ExprMatch { eq: op.is_eq() }); + + matched = true; + } + + matched + } + + /// Copute the cost of using this index match to execute the query. + pub(super) fn compute_cost(&self, filter: &stmt::Expr) -> usize { + if self.index.unique { + let mut cost = 0; + + for column in &self.columns { + let Some(expr_match) = column.exprs.get(&ByAddress(filter)) else { + break; + }; + + if expr_match.eq { + cost += 1; + } else { + cost += 10; + } + } + + cost + } else { + // Arbitrary + 10 + } + } + + pub(super) fn partition_filter( + &self, + ctx: &mut PartitionCtx<'_>, + expr: &stmt::Expr, + ) -> ExprPair { + use stmt::Expr::*; + + match expr { + StartsWith(_) | Between(_) | InList(_) | IsNull(_) | Like(_) | Not(_) => { + if self + .columns + .iter() + .any(|f| f.exprs.contains_key(&ByAddress(expr))) + { + (expr.clone(), true.into()) + } else { + (true.into(), expr.clone()) + } + } + BinaryOp(binary_op) => { + if self + .columns + .iter() + .any(|f| f.exprs.contains_key(&ByAddress(expr))) + { + if binary_op.op.is_ne() && !ctx.capability.primary_key_ne_predicate { + ctx.apply_result_filter_on_results = true; + return (true.into(), true.into()); + } + + // Normalize the expression to include the column on the LHS + let expr = match (&*binary_op.lhs, &*binary_op.rhs) { + (stmt::Expr::Reference(stmt::ExprReference::Column(_)), _) => expr.clone(), + ( + lhs, + stmt::Expr::Reference(column_ref @ stmt::ExprReference::Column(_)), + ) => stmt::ExprBinaryOp { + lhs: Box::new(stmt::Expr::Reference(*column_ref)), + rhs: Box::new(lhs.clone()), + op: binary_op.op.commute().expect( + "partition filter normalization reached a non-commutative op", + ), + } + .into(), + _ => todo!("binary_op={binary_op:#?}"), + }; + + (expr, true.into()) + } else { + (true.into(), expr.clone()) + } + } + And(expr_and) => { + let mut index_filter_operands = vec![]; + let mut result_filter_operands = vec![]; + + for operand in &expr_and.operands { + let (index_filter, result_filter) = self.partition_filter(ctx, operand); + + if !index_filter.is_true() { + index_filter_operands.push(index_filter); + } + + if !result_filter.is_true() { + result_filter_operands.push(result_filter); + } + } + + let index_filter = match index_filter_operands.len() { + 0 => true.into(), + 1 => index_filter_operands.into_iter().next().unwrap(), + _ => stmt::ExprAnd { + operands: index_filter_operands, + } + .into(), + }; + let result_filter = match result_filter_operands.len() { + 0 => true.into(), + 1 => result_filter_operands.into_iter().next().unwrap(), + _ => stmt::ExprAnd { + operands: result_filter_operands, + } + .into(), + }; + + (index_filter, result_filter) + } + Or(expr_or) => { + let mut index_filter_operands = vec![]; + let mut result_filter_operands = vec![]; + + for operand in &expr_or.operands { + let (index_filter, result_filter) = self.partition_filter(ctx, operand); + + if !index_filter.is_true() && !result_filter.is_true() { + todo!("expr_or={:#?}", expr_or); + } + + if !index_filter.is_true() { + assert!(result_filter_operands.is_empty()); + index_filter_operands.push(index_filter); + } else if !result_filter.is_true() { + assert!(index_filter_operands.is_empty()); + result_filter_operands.push(result_filter); + } else { + todo!() + } + } + + if !index_filter_operands.is_empty() { + ( + stmt::ExprOr { + operands: index_filter_operands, + } + .into(), + true.into(), + ) + } else { + ( + true.into(), + stmt::ExprOr { + operands: result_filter_operands, + } + .into(), + ) + } + } + Any(expr_any) => { + // For Any expressions, partition the inner map expression + let stmt::Expr::Map(expr_map) = &*expr_any.expr else { + todo!("expr_any.expr={:#?}", expr_any.expr); + }; + + let (index_filter, result_filter) = self.partition_filter(ctx, &expr_map.map); + + // If the map expression can be used as an index filter, reconstruct + // the Any with the index filter version + if !index_filter.is_true() { + let index_any = stmt::ExprAny { + expr: Box::new(stmt::Expr::Map(stmt::ExprMap { + base: expr_map.base.clone(), + map: Box::new(index_filter), + })), + }; + (index_any.into(), true.into()) + } else if !result_filter.is_true() { + let result_any = stmt::ExprAny { + expr: Box::new(stmt::Expr::Map(stmt::ExprMap { + base: expr_map.base.clone(), + map: Box::new(result_filter), + })), + }; + (true.into(), result_any.into()) + } else { + (true.into(), true.into()) + } + } + _ => todo!("partition_filter={:#?}", expr), + } + } +} diff --git a/packages/toasty/src/engine/index/index_plan.rs b/packages/toasty/src/engine/index/index_plan.rs new file mode 100644 index 00000000000..5613b337470 --- /dev/null +++ b/packages/toasty/src/engine/index/index_plan.rs @@ -0,0 +1,45 @@ +use toasty_core::{ + schema::db::{Index, TableId}, + stmt, +}; + +#[derive(Debug)] +pub(crate) struct IndexPlan<'a> { + /// The index to use to execute the query + pub(crate) index: &'a Index, + + /// Filter to apply to the index + pub(crate) index_filter: stmt::Expr, + + /// How to filter results after applying the index filter + pub(crate) result_filter: Option, + + /// True if we have to apply the result filter our self + pub(crate) post_filter: Option, + + /// Filter that depends only on args (not on table columns). Evaluated + /// before issuing the database operation; when false the operation is + /// skipped entirely and an empty result is returned. + pub(crate) pre_filter: Option, + + /// Key expression for direct `GetByKey` routing. Populated when every index + /// key column has an exact predicate (equality or IN). Two forms: + /// + /// - `Expr::Value(Value::List([Value::Record([...]), ...]))` — all key values + /// are literals known at plan time; the planner emits a constant `GetByKey`. + /// - `Expr::Arg(0)` — key values come from a runtime input (e.g. `pk IN + /// (arg[0])` batch-load); the planner wires the input node directly. + /// + /// When `Some`, the planner routes to `GetByKey` instead of `QueryPk`. + pub(crate) key_values: Option, + + /// True when this plan targets the primary key and `key_values` was populated. + /// Captured before `key_values` can be consumed via `.take()`. + pub(crate) has_pk_keys: bool, +} + +impl IndexPlan<'_> { + pub(crate) fn table_id(&self) -> TableId { + self.index.on + } +} diff --git a/packages/toasty/src/engine/index/or_rewrite.rs b/packages/toasty/src/engine/index/or_rewrite.rs new file mode 100644 index 00000000000..40db73314b4 --- /dev/null +++ b/packages/toasty/src/engine/index/or_rewrite.rs @@ -0,0 +1,269 @@ +use toasty_core::stmt; + +/// Converts an index filter to canonical `ANY(MAP(...))` form for backends that do +/// not support OR in key conditions (e.g. DynamoDB). +/// +/// Steps: +/// 1. If the expression is already an `InList(col, [v1, v2, ...])`, expand it +/// directly to `ANY(MAP([v1, v2, ...], col = arg(0)))` — equivalent but in +/// the canonical fan-out form. +/// 2. Flatten the expression to Disjunctive Normal Form (DNF) — a flat list of OR +/// branches, each branch being a single predicate or AND of predicates. +/// 3. Group branches by their structural shape (predicate with literal values +/// replaced by `arg(i)`). +/// 4. Unify each same-shape group into `ANY(MAP(Value::List([v1, v2, ...]), shape))`. +pub(super) fn index_filter_to_any_map(expr: stmt::Expr) -> stmt::Expr { + // `col IN [v1, v2, ...]` is equivalent to `v1 = col OR v2 = col OR ...`. + // Expand it directly to the canonical fan-out form so the DNF step doesn't + // need to handle InList as a leaf. + if let stmt::Expr::InList(ref in_list) = expr + && matches!(*in_list.list, stmt::Expr::Value(stmt::Value::List(_))) + { + let shape = stmt::Expr::from(stmt::ExprBinaryOp { + lhs: in_list.expr.clone(), + op: stmt::BinaryOp::Eq, + rhs: Box::new(stmt::Expr::arg(0)), + }); + return stmt::Expr::any(stmt::Expr::map(*in_list.list.clone(), shape)); + } + + let branches = flatten_to_dnf(expr); + unify_dnf_branches(branches) +} + +/// Flatten an expression to a list of OR branches in Disjunctive Normal Form +/// using an iterative work queue. +/// +/// - `Or`: extend the queue with each operand. +/// - `And`: find the first `Or` operand and distribute over its branches, +/// re-queuing each resulting `And` for further processing. When no `Or` +/// operands remain the `And` is in final form. +/// - Leaf predicate: emit directly as a final branch. +fn flatten_to_dnf(expr: stmt::Expr) -> Vec { + let mut branches: Vec = Vec::new(); + let mut queue: Vec = vec![expr]; + + while let Some(expr) = queue.pop() { + match expr { + stmt::Expr::Or(or) => queue.extend(or.operands.into_iter().rev()), + stmt::Expr::And(and) => process_and(and, &mut queue, &mut branches), + leaf => branches.push(leaf), + } + } + + // Validate that no branch contains an Or inside an Any(Map(...)). If it + // does, the distribution logic above has a bug. + for branch in &branches { + assert_no_or_in_any(branch); + } + + branches +} + +/// Process one `And` expression from the DNF work queue. +/// +/// Priority: +/// 1. If any operand is `Or`, distribute AND over it and re-queue each branch. +/// 2. If any operand is `Any(Map(...))`, distribute the remaining operands +/// into the map predicate and re-queue the resulting `Any`. +/// 3. No `Or` or `Any` operands: emit as a final DNF conjunction. +fn process_and(and: stmt::ExprAnd, queue: &mut Vec, branches: &mut Vec) { + if let Some(pos) = and + .operands + .iter() + .position(|op| matches!(op, stmt::Expr::Or(_))) + { + return distribute_over_or(and, pos, queue); + } + + if let Some(pos) = and + .operands + .iter() + .position(|op| matches!(op, stmt::Expr::Any(_))) + { + return distribute_into_any(and, pos, queue); + } + + branches.push(stmt::Expr::And(and)); +} + +/// Distribute AND over an `Or` operand at `pos`, re-queuing one `And` per branch. +/// +/// `(p AND (a OR b) AND q)` → `(p AND a AND q)` and `(p AND b AND q)` on the queue. +fn distribute_over_or(and: stmt::ExprAnd, pos: usize, queue: &mut Vec) { + let mut operands = and.operands; + let stmt::Expr::Or(or) = operands.remove(pos) else { + unreachable!() + }; + + for branch in or.operands.into_iter().rev() { + let mut new_operands = operands.clone(); + new_operands.insert(pos, branch); + queue.push( + stmt::ExprAnd { + operands: new_operands, + } + .into(), + ); + } +} + +/// Distribute the non-`Any` operands of `and` into an `Any(Map(...))` at `pos`. +/// +/// `AND(p, ANY(MAP(base, pred)))` → `ANY(MAP(base, AND(pred, p)))`. +/// +/// This is valid because the non-Any operands do not reference the map's arg variable. +fn distribute_into_any(and: stmt::ExprAnd, pos: usize, queue: &mut Vec) { + let mut operands = and.operands; + let stmt::Expr::Any(any) = operands.remove(pos) else { + unreachable!() + }; + let stmt::Expr::Map(map) = *any.expr else { + todo!("Any with non-Map expr in AND distribution"); + }; + + // Keep the original map predicate first, then the distributed And operands. + let mut inner_operands = vec![*map.map]; + inner_operands.extend(operands); + let inner: stmt::Expr = if inner_operands.len() == 1 { + inner_operands.into_iter().next().unwrap() + } else { + stmt::ExprAnd { + operands: inner_operands, + } + .into() + }; + + queue.push( + stmt::ExprAny { + expr: Box::new(stmt::Expr::Map(stmt::ExprMap { + base: map.base, + map: Box::new(inner), + })), + } + .into(), + ); +} + +/// Group DNF branches by shape; unify each group into `ANY(MAP(...))`. +/// If there is only a single branch (no OR), returns it unchanged. +fn unify_dnf_branches(branches: Vec) -> stmt::Expr { + if branches.len() == 1 { + return branches.into_iter().next().unwrap(); + } + + // Each group: (shape, per-branch scalar-or-record values). + let mut groups: Vec<(stmt::Expr, Vec)> = vec![]; + + for branch in branches { + let (shape, value) = extract_shape(branch); + if let Some((_, values)) = groups.iter_mut().find(|(s, _)| *s == shape) { + values.push(value); + } else { + groups.push((shape, vec![value])); + } + } + + if groups.len() > 1 { + todo!( + "OR index filter with multiple distinct branch shapes is not yet implemented; \ + shapes: {:#?}", + groups.iter().map(|(s, _)| s).collect::>() + ); + } + + let (shape, values) = groups.into_iter().next().unwrap(); + + stmt::Expr::any(stmt::Expr::map( + stmt::Expr::Value(stmt::Value::List(values)), + shape, + )) +} + +/// Extract the per-call predicate template (shape) and single value for one DNF branch. +/// +/// - `col op literal` → shape `col op arg(0)`, value `literal` +/// - `col1 op1 v1 AND col2 op2 v2 AND ...` → shape with `arg(i)` per column, +/// value `Value::Record([v1, v2, ...])` — composite key fan-out (TODO) +fn extract_shape(branch: stmt::Expr) -> (stmt::Expr, stmt::Value) { + match branch { + stmt::Expr::BinaryOp(b) => { + let stmt::Expr::Value(v) = *b.rhs else { + todo!("non-literal value in OR branch rhs: {:#?}", b.rhs); + }; + let shape: stmt::Expr = stmt::ExprBinaryOp { + lhs: b.lhs, + op: b.op, + rhs: Box::new(stmt::Expr::arg(0)), + } + .into(); + (shape, v) + } + // Composite key: (col1 = t1 AND col2 >= s1) OR (col1 = t2 AND col2 >= s2) + // → ANY(MAP([(t1,s1),(t2,s2)], col1=arg(0) AND col2>=arg(1))) + stmt::Expr::And(and) => { + let mut values = vec![]; + let mut shape_operands = vec![]; + + for (i, operand) in and.operands.into_iter().enumerate() { + let stmt::Expr::BinaryOp(b) = operand else { + todo!( + "non-BinaryOp operand in composite AND branch: {:#?}", + operand + ); + }; + let stmt::Expr::Value(v) = *b.rhs else { + todo!( + "non-literal value in composite AND branch rhs: {:#?}", + b.rhs + ); + }; + values.push(v); + shape_operands.push(stmt::Expr::from(stmt::ExprBinaryOp { + lhs: b.lhs, + op: b.op, + rhs: Box::new(stmt::Expr::arg(i)), + })); + } + + let shape = stmt::Expr::from(stmt::ExprAnd { + operands: shape_operands, + }); + let record = stmt::Value::Record(stmt::ValueRecord::from_vec(values)); + (shape, record) + } + _ => todo!("unsupported branch type in OR index filter: {branch:#?}"), + } +} + +/// Asserts that no `Any(Map(...))` in `expr` contains an `Or` anywhere in its +/// predicate sub-tree. This catches bugs where OR distribution was incomplete. +fn assert_no_or_in_any(expr: &stmt::Expr) { + match expr { + stmt::Expr::Any(any) => { + assert!( + !contains_or(&any.expr), + "Any(Map(...)) contains an Or expression after DNF distribution; \ + this is a bug in flatten_to_dnf: {:#?}", + any.expr + ); + } + stmt::Expr::And(and) => and.operands.iter().for_each(assert_no_or_in_any), + _ => {} + } +} + +/// Returns true if `expr` contains an `Expr::Or` anywhere in its sub-tree. +fn contains_or(expr: &stmt::Expr) -> bool { + match expr { + stmt::Expr::Or(_) => true, + stmt::Expr::And(and) => and.operands.iter().any(contains_or), + stmt::Expr::Any(a) => contains_or(&a.expr), + stmt::Expr::Map(m) => contains_or(&m.base) || contains_or(&m.map), + stmt::Expr::BinaryOp(b) => contains_or(&b.lhs) || contains_or(&b.rhs), + stmt::Expr::Not(n) => contains_or(&n.expr), + stmt::Expr::IsNull(n) => contains_or(&n.expr), + // Leaf nodes (Arg, Reference, Value, Default, Type, etc.) contain no sub-expressions. + _ => false, + } +} diff --git a/packages/toasty/src/engine/index/tests.rs b/packages/toasty/src/engine/index/tests.rs new file mode 100644 index 00000000000..eb640032f3c --- /dev/null +++ b/packages/toasty/src/engine/index/tests.rs @@ -0,0 +1,674 @@ +use super::*; +use toasty_core::{ + schema::{Builder, app}, + stmt, +}; + +use crate as toasty; +use crate::schema::Model; + +#[allow(dead_code)] +#[derive(toasty::Model)] +struct User { + #[key] + id: String, + name: String, +} + +/// Composite primary key: user_id (partition) + status (sort). +#[allow(dead_code)] +#[derive(toasty::Model)] +struct Todo { + #[key] + user_id: String, + #[key] + status: String, +} + +/// Composite primary key with a local (sort) column that accepts range predicates. +#[allow(dead_code)] +#[derive(toasty::Model)] +#[key(partition = id, local = rank)] +struct Ranked { + id: String, + rank: i64, +} + +#[test] +fn pk_equality_goes_to_index_filter() -> Result<()> { + let cx = sqlite_test_cx(); + + // col[0] = 1 — pk column equality + let filter = stmt::Expr::eq( + stmt::Expr::Reference(stmt::ExprReference::column(0, 0)), + stmt::Expr::Value(stmt::Value::from(1i64)), + ); + + let plan = cx.plan_basic_query_with_filter(filter.clone())?; + + assert!( + plan.index.primary_key, + "should select the primary key index" + ); + assert_eq!( + plan.index_filter, filter, + "pk equality should be the index filter" + ); + assert!( + plan.result_filter.is_none(), + "no residual result filter expected" + ); + assert!(plan.post_filter.is_none()); + Ok(()) +} + +#[test] +fn pk_equality_column_on_rhs_goes_to_index_filter() -> Result<()> { + let cx = sqlite_test_cx(); + + // 1 = col[0] — pk column on RHS; index matching must commute the operator + let filter = stmt::Expr::eq( + stmt::Expr::Value(stmt::Value::from(1i64)), + stmt::Expr::Reference(stmt::ExprReference::column(0, 0)), + ); + + let plan = cx.plan_basic_query_with_filter(filter)?; + + assert!( + plan.index.primary_key, + "should select the primary key index" + ); + assert!( + plan.result_filter.is_none(), + "no residual result filter expected" + ); + assert!(plan.post_filter.is_none()); + Ok(()) +} + +#[test] +fn pk_lt_column_on_rhs_commutes_to_gt() -> Result<()> { + // Schema: Ranked { id (partition), rank (local) } + // The local-scoped sort column accepts range predicates in the index. + let cx = sqlite_test_cx_ranked(); + + // id = "a" AND 10 < rank — sort column on RHS with Lt + // Index matching must commute the second operand to rank > 10. + let filter = stmt::Expr::And(stmt::ExprAnd { + operands: vec![ + stmt::Expr::eq( + stmt::Expr::Reference(stmt::ExprReference::column(0, 0)), + stmt::Expr::Value(stmt::Value::String("a".to_string())), + ), + stmt::Expr::BinaryOp(stmt::ExprBinaryOp { + lhs: Box::new(stmt::Expr::Value(stmt::Value::from(10i64))), + op: stmt::BinaryOp::Lt, + rhs: Box::new(stmt::Expr::Reference(stmt::ExprReference::column(0, 1))), + }), + ], + }); + + let plan = cx.plan_basic_query_with_filter(filter)?; + + assert!( + plan.index.primary_key, + "should select the primary key index" + ); + + // The index filter should contain both conditions with the sort column + // commuted to col > 10 (not the original 10 < col). + let expected_index_filter = stmt::Expr::And(stmt::ExprAnd { + operands: vec![ + stmt::Expr::eq( + stmt::Expr::Reference(stmt::ExprReference::column(0, 0)), + stmt::Expr::Value(stmt::Value::String("a".to_string())), + ), + stmt::Expr::BinaryOp(stmt::ExprBinaryOp { + lhs: Box::new(stmt::Expr::Reference(stmt::ExprReference::column(0, 1))), + op: stmt::BinaryOp::Gt, + rhs: Box::new(stmt::Expr::Value(stmt::Value::from(10i64))), + }), + ], + }); + assert_eq!(plan.index_filter, expected_index_filter); + assert!( + plan.result_filter.is_none(), + "no residual result filter expected" + ); + Ok(()) +} + +#[test] +fn and_splits_pk_to_index_and_name_to_result() -> Result<()> { + let cx = sqlite_test_cx(); + + // col[0] = 1 AND col[1] = 2 — pk equality AND name equality + let pk_eq = stmt::Expr::eq( + stmt::Expr::Reference(stmt::ExprReference::column(0, 0)), + stmt::Expr::Value(stmt::Value::from(1i64)), + ); + let name_eq = stmt::Expr::eq( + stmt::Expr::Reference(stmt::ExprReference::column(0, 1)), + stmt::Expr::Value(stmt::Value::from(2i64)), + ); + let filter = stmt::Expr::And(stmt::ExprAnd { + operands: vec![pk_eq.clone(), name_eq.clone()], + }); + + let plan = cx.plan_basic_query_with_filter(filter)?; + + assert!(plan.index.primary_key); + assert_eq!( + plan.index_filter, pk_eq, + "only the pk condition goes to index filter" + ); + assert_eq!( + plan.result_filter.as_ref(), + Some(&name_eq), + "non-pk condition goes to result filter" + ); + Ok(()) +} + +#[test] +fn or_on_pk_stays_as_or_for_sql() -> Result<()> { + let cx = sqlite_test_cx(); // SQLite — index_or_predicate = true + + // col[0] = 1 OR col[0] = 2 + let pk_eq_1 = stmt::Expr::eq( + stmt::Expr::Reference(stmt::ExprReference::column(0, 0)), + stmt::Expr::Value(stmt::Value::from(1i64)), + ); + let pk_eq_2 = stmt::Expr::eq( + stmt::Expr::Reference(stmt::ExprReference::column(0, 0)), + stmt::Expr::Value(stmt::Value::from(2i64)), + ); + let filter = stmt::Expr::Or(stmt::ExprOr { + operands: vec![pk_eq_1, pk_eq_2], + }); + + let plan = cx.plan_basic_query_with_filter(filter.clone())?; + + assert!(plan.index.primary_key); + assert_eq!( + plan.index_filter, filter, + "OR should be preserved as-is for SQL backends" + ); + assert!(plan.result_filter.is_none()); + assert!(plan.post_filter.is_none()); + Ok(()) +} + +#[test] +fn or_on_pk_becomes_any_map_for_dynamodb() -> Result<()> { + let cx = ddb_test_cx(); + + // col[0] = 1 OR col[0] = 2 + let pk_eq_1 = stmt::Expr::eq( + stmt::Expr::Reference(stmt::ExprReference::column(0, 0)), + stmt::Expr::Value(stmt::Value::from(1i64)), + ); + let pk_eq_2 = stmt::Expr::eq( + stmt::Expr::Reference(stmt::ExprReference::column(0, 0)), + stmt::Expr::Value(stmt::Value::from(2i64)), + ); + let filter = stmt::Expr::Or(stmt::ExprOr { + operands: vec![pk_eq_1, pk_eq_2], + }); + + let plan = cx.plan_basic_query_with_filter(filter)?; + + // Expected: ANY(MAP([1, 2], col[0][0] = arg(0))) + let expected = stmt::Expr::any(stmt::Expr::map( + stmt::Expr::Value(stmt::Value::List(vec![ + stmt::Value::from(1i64), + stmt::Value::from(2i64), + ])), + stmt::Expr::eq( + stmt::Expr::Reference(stmt::ExprReference::column(0, 0)), + stmt::Expr::arg(0), + ), + )); + + assert!(plan.index.primary_key); + assert_eq!( + plan.index_filter, expected, + "OR should be rewritten to ANY(MAP(...)) for DynamoDB" + ); + assert!(plan.result_filter.is_none()); + assert!(plan.post_filter.is_none()); + Ok(()) +} + +#[test] +fn and_with_any_map_distributes_into_any_map_for_dynamodb() -> Result<()> { + // Schema: Todo { user_id (pk partition), status (pk sort) } + // Both columns are part of the primary key index, so both land in index_filter. + // + // Filter: col[0][1] = "active" AND ANY(MAP(arg[0], col[0][0] = arg[0])) + // col[0][0] = user_id (partition key), col[0][1] = status (sort key) + // + // Expected: ANY(MAP(arg[0], col[0][0] = arg[0] AND col[0][1] = "active")) + let cx = ddb_test_cx_composite(); + + let status_eq = stmt::Expr::eq( + stmt::Expr::Reference(stmt::ExprReference::column(0, 1)), + stmt::Expr::Value(stmt::Value::String("active".to_string())), + ); + let user_id_any = stmt::Expr::any(stmt::Expr::map( + stmt::Expr::arg(0), + stmt::Expr::eq( + stmt::Expr::Reference(stmt::ExprReference::column(0, 0)), + stmt::Expr::arg(0), + ), + )); + let filter = stmt::Expr::And(stmt::ExprAnd { + operands: vec![status_eq.clone(), user_id_any], + }); + + let plan = cx.plan_basic_query_with_filter(filter)?; + + let expected = stmt::Expr::any(stmt::Expr::map( + stmt::Expr::arg(0), + stmt::Expr::And(stmt::ExprAnd { + operands: vec![ + stmt::Expr::eq( + stmt::Expr::Reference(stmt::ExprReference::column(0, 0)), + stmt::Expr::arg(0), + ), + status_eq, + ], + }), + )); + + assert!(plan.index.primary_key); + assert_eq!( + plan.index_filter, expected, + "AND with ANY(MAP) should distribute the non-Any operands into the map predicate" + ); + assert!(plan.result_filter.is_none()); + assert!(plan.post_filter.is_none()); + Ok(()) +} + +// ── key_values extraction ───────────────────────────────────────────────────── + +#[test] +fn pk_equality_sets_key_values() -> Result<()> { + let cx = sqlite_test_cx(); + + // col[0] = 1 — single PK column, literal equality + let filter = stmt::Expr::eq( + stmt::Expr::Reference(stmt::ExprReference::column(0, 0)), + stmt::Expr::Value(stmt::Value::from(1i64)), + ); + + let plan = cx.plan_basic_query_with_filter(filter)?; + + let expected = stmt::Value::List(vec![stmt::Value::Record(stmt::ValueRecord::from_vec( + vec![stmt::Value::from(1i64)], + ))]); + assert_eq!(plan.key_values, Some(stmt::Expr::Value(expected))); + Ok(()) +} + +#[test] +fn pk_or_sets_key_values() -> Result<()> { + let cx = sqlite_test_cx(); + + // col[0] = 1 OR col[0] = 2 — two literal PK values + let filter = stmt::Expr::Or(stmt::ExprOr { + operands: vec![ + stmt::Expr::eq( + stmt::Expr::Reference(stmt::ExprReference::column(0, 0)), + stmt::Expr::Value(stmt::Value::from(1i64)), + ), + stmt::Expr::eq( + stmt::Expr::Reference(stmt::ExprReference::column(0, 0)), + stmt::Expr::Value(stmt::Value::from(2i64)), + ), + ], + }); + + let plan = cx.plan_basic_query_with_filter(filter)?; + + let record = + |v: i64| stmt::Value::Record(stmt::ValueRecord::from_vec(vec![stmt::Value::from(v)])); + let expected = stmt::Value::List(vec![record(1), record(2)]); + assert_eq!(plan.key_values, Some(stmt::Expr::Value(expected))); + Ok(()) +} + +#[test] +fn pk_range_does_not_set_key_values() -> Result<()> { + // Schema: Todo { user_id (pk partition), status (pk sort) } + // Partition key has equality (required to match the index), but the sort key + // has a range predicate — no exact key record can be formed, so key_values = None. + let cx = ddb_test_cx_composite(); + + let filter = stmt::Expr::And(stmt::ExprAnd { + operands: vec![ + stmt::Expr::eq( + stmt::Expr::Reference(stmt::ExprReference::column(0, 0)), + stmt::Expr::Value(stmt::Value::String("u1".to_string())), + ), + stmt::Expr::BinaryOp(stmt::ExprBinaryOp { + lhs: Box::new(stmt::Expr::Reference(stmt::ExprReference::column(0, 1))), + op: stmt::BinaryOp::Ge, + rhs: Box::new(stmt::Expr::Value(stmt::Value::String("s1".to_string()))), + }), + ], + }); + + let plan = cx.plan_basic_query_with_filter(filter)?; + + assert_eq!(plan.key_values, None); + Ok(()) +} + +#[test] +fn composite_pk_full_equality_sets_key_values() -> Result<()> { + // Schema: Todo { user_id (pk partition), status (pk sort) } + // Both key columns have literal equality → key_values is populated. + let cx = ddb_test_cx_composite(); + + let filter = stmt::Expr::And(stmt::ExprAnd { + operands: vec![ + stmt::Expr::eq( + stmt::Expr::Reference(stmt::ExprReference::column(0, 0)), + stmt::Expr::Value(stmt::Value::String("u1".to_string())), + ), + stmt::Expr::eq( + stmt::Expr::Reference(stmt::ExprReference::column(0, 1)), + stmt::Expr::Value(stmt::Value::String("s1".to_string())), + ), + ], + }); + + let plan = cx.plan_basic_query_with_filter(filter)?; + + let expected = stmt::Value::List(vec![stmt::Value::Record(stmt::ValueRecord::from_vec( + vec![ + stmt::Value::String("u1".to_string()), + stmt::Value::String("s1".to_string()), + ], + ))]); + assert_eq!(plan.key_values, Some(stmt::Expr::Value(expected))); + Ok(()) +} + +#[test] +fn composite_pk_partition_key_only_does_not_set_key_values() -> Result<()> { + // Schema: Todo { user_id (pk partition), status (pk sort) } + // Only the partition key is specified — cannot form a full key record. + let cx = ddb_test_cx_composite(); + + let filter = stmt::Expr::eq( + stmt::Expr::Reference(stmt::ExprReference::column(0, 0)), + stmt::Expr::Value(stmt::Value::String("u1".to_string())), + ); + + let plan = cx.plan_basic_query_with_filter(filter)?; + + assert_eq!(plan.key_values, None); + Ok(()) +} + +#[test] +fn any_map_with_arg_base_passes_through_for_dynamodb() { + // ANY(MAP(arg[0], col[0][0] = arg[0])) — the batch-load canonical form produced by + // rewrite_stmt_query_for_batch_load_nosql. It is already in the target form so + // index_filter_to_any_map must return it unchanged. + let filter = stmt::Expr::any(stmt::Expr::map( + stmt::Expr::arg(0), + stmt::Expr::eq( + stmt::Expr::Reference(stmt::ExprReference::column(0, 0)), + stmt::Expr::arg(0), + ), + )); + + let result = or_rewrite::index_filter_to_any_map(filter.clone()); + assert_eq!( + result, filter, + "batch-load ANY(MAP(arg, pred)) should pass through unchanged" + ); +} + +#[test] +fn composite_pk_or_sets_key_values() -> Result<()> { + // Schema: Todo { user_id (pk partition), status (pk sort) } + // + // Filter: (user_id = "u1" AND status = "s1") OR (user_id = "u2" AND status = "s2") + // + // Both branches fully specify all key columns with equality → key_values is populated. + let cx = ddb_test_cx_composite(); + + let branch = |uid: &str, status: &str| { + stmt::Expr::And(stmt::ExprAnd { + operands: vec![ + stmt::Expr::eq( + stmt::Expr::Reference(stmt::ExprReference::column(0, 0)), + stmt::Expr::Value(stmt::Value::String(uid.to_string())), + ), + stmt::Expr::eq( + stmt::Expr::Reference(stmt::ExprReference::column(0, 1)), + stmt::Expr::Value(stmt::Value::String(status.to_string())), + ), + ], + }) + }; + let filter = stmt::Expr::Or(stmt::ExprOr { + operands: vec![branch("u1", "s1"), branch("u2", "s2")], + }); + + let plan = cx.plan_basic_query_with_filter(filter)?; + + let record = |uid: &str, status: &str| { + stmt::Value::Record(stmt::ValueRecord::from_vec(vec![ + stmt::Value::String(uid.to_string()), + stmt::Value::String(status.to_string()), + ])) + }; + let expected = stmt::Value::List(vec![record("u1", "s1"), record("u2", "s2")]); + assert_eq!(plan.key_values, Some(stmt::Expr::Value(expected))); + Ok(()) +} + +#[test] +fn composite_pk_or_becomes_any_map_for_dynamodb() -> Result<()> { + // Schema: Todo { user_id (pk partition), status (pk sort) } + // + // Filter: (user_id = "u1" AND status = "s1") OR (user_id = "u2" AND status = "s2") + // + // Expected: ANY(MAP( + // [(u1,s1), (u2,s2)], + // user_id = arg(0) AND status = arg(1) + // )) + let cx = ddb_test_cx_composite(); + + let branch = |uid: &str, status: &str| { + stmt::Expr::And(stmt::ExprAnd { + operands: vec![ + stmt::Expr::eq( + stmt::Expr::Reference(stmt::ExprReference::column(0, 0)), + stmt::Expr::Value(stmt::Value::String(uid.to_string())), + ), + stmt::Expr::eq( + stmt::Expr::Reference(stmt::ExprReference::column(0, 1)), + stmt::Expr::Value(stmt::Value::String(status.to_string())), + ), + ], + }) + }; + let filter = stmt::Expr::Or(stmt::ExprOr { + operands: vec![branch("u1", "s1"), branch("u2", "s2")], + }); + + let plan = cx.plan_basic_query_with_filter(filter)?; + + let record = |uid: &str, status: &str| { + stmt::Value::Record(stmt::ValueRecord::from_vec(vec![ + stmt::Value::String(uid.to_string()), + stmt::Value::String(status.to_string()), + ])) + }; + let expected = stmt::Expr::any(stmt::Expr::map( + stmt::Expr::Value(stmt::Value::List(vec![ + record("u1", "s1"), + record("u2", "s2"), + ])), + stmt::Expr::And(stmt::ExprAnd { + operands: vec![ + stmt::Expr::eq( + stmt::Expr::Reference(stmt::ExprReference::column(0, 0)), + stmt::Expr::arg(0), + ), + stmt::Expr::eq( + stmt::Expr::Reference(stmt::ExprReference::column(0, 1)), + stmt::Expr::arg(1), + ), + ], + }), + )); + + assert!(plan.index.primary_key); + assert_eq!( + plan.index_filter, expected, + "composite OR should be rewritten to ANY(MAP([records], col0=arg(0) AND col1=arg(1)))" + ); + assert!(plan.result_filter.is_none()); + assert!(plan.post_filter.is_none()); + Ok(()) +} + +#[test] +fn chooses_lowest_cost_index_and_first_on_ties() -> Result<()> { + #[allow(dead_code)] + #[derive(toasty::Model)] + struct Indexed { + #[key] + id: String, + #[unique] + name: String, + } + + let app_schema = app::Schema::from_macro([Indexed::schema()])?; + let cx = TestCx { + schema: Builder::new().build(app_schema, &Capability::SQLITE)?, + capability: &Capability::SQLITE, + }; + + for op in [stmt::BinaryOp::Gt, stmt::BinaryOp::Eq] { + let filter = stmt::Expr::and( + stmt::Expr::binary_op( + stmt::Expr::Reference(stmt::ExprReference::column(0, 0)), + op, + "a", + ), + stmt::Expr::eq( + stmt::Expr::Reference(stmt::ExprReference::column(0, 1)), + "b", + ), + ); + let plan = cx.plan_basic_query_with_filter(filter)?; + assert_eq!(plan.index.primary_key, op.is_eq()); + } + Ok(()) +} + +#[test] +fn no_matching_index_respects_scan_capability() -> Result<()> { + let cx = sqlite_test_cx(); + let stmt = cx.basic_query_with_filter(stmt::Expr::eq( + stmt::Expr::Reference(stmt::ExprReference::column(0, 1)), + "name", + )); + assert!(plan_index_path(&cx.schema, cx.capability, &stmt)?.is_none()); + + let capability = Capability { + scan: false, + ..Capability::SQLITE + }; + assert!( + plan_index_path(&cx.schema, &capability, &stmt) + .unwrap_err() + .is_unsupported_feature() + ); + Ok(()) +} + +struct TestCx { + schema: toasty_core::Schema, + capability: &'static Capability, +} + +fn sqlite_test_cx() -> TestCx { + test_cx_with_capability(&Capability::SQLITE) +} + +fn ddb_test_cx() -> TestCx { + test_cx_with_capability(&Capability::DYNAMODB) +} + +fn ddb_test_cx_composite() -> TestCx { + let app_schema = + app::Schema::from_macro([Todo::schema()]).expect("schema should build from macro"); + let schema = Builder::new() + .build(app_schema, &Capability::DYNAMODB) + .expect("schema should build"); + TestCx { + schema, + capability: &Capability::DYNAMODB, + } +} + +fn sqlite_test_cx_ranked() -> TestCx { + let app_schema = + app::Schema::from_macro([Ranked::schema()]).expect("schema should build from macro"); + let schema = Builder::new() + .build(app_schema, &Capability::SQLITE) + .expect("schema should build"); + TestCx { + schema, + capability: &Capability::SQLITE, + } +} + +fn test_cx_with_capability(capability: &'static Capability) -> TestCx { + let app_schema = + app::Schema::from_macro([User::schema()]).expect("schema should build from macro"); + let schema = Builder::new() + .build(app_schema, capability) + .expect("schema should build"); + TestCx { schema, capability } +} + +impl TestCx { + /// Build a table-targeting `SELECT` statement with the given filter against + /// the first table in the schema. This mirrors the lowered statements that + /// reach `plan_index_path` at runtime. + fn basic_query_with_filter(&self, filter: stmt::Expr) -> stmt::Statement { + let table_id = self.schema.db.tables[0].id; + let source = stmt::SourceTable::new( + vec![stmt::TableRef::Table(table_id)], + stmt::TableWithJoins { + relation: stmt::TableFactor::Table(stmt::SourceTableId(0)), + joins: vec![], + }, + ); + stmt::Statement::Query(stmt::Query { + with: None, + body: stmt::ExprSet::Select(Box::new(stmt::Select::new(source, filter))), + single: false, + order_by: None, + limit: None, + locks: vec![], + }) + } + + fn plan_basic_query_with_filter(&self, filter: stmt::Expr) -> Result> { + let stmt = self.basic_query_with_filter(filter); + plan_index_path(&self.schema, self.capability, &stmt) + .map(|opt| opt.expect("test expects an index plan, not a scan fallback")) + } +} diff --git a/packages/toasty/src/engine/legalize.rs b/packages/toasty/src/engine/legalize.rs new file mode 100644 index 00000000000..92d0210765a --- /dev/null +++ b/packages/toasty/src/engine/legalize.rs @@ -0,0 +1,61 @@ +//! Legalize engine-final statements for the target backend. +//! +//! The engine's internal phases (simplify, lower, plan) produce one canonical +//! statement form regardless of backend, and some constructs in that form +//! have no driver representation. Legalization is the last engine-side +//! transformation before a statement crosses the driver boundary: driven by +//! [`Capability`], it rewrites each such construct into an equivalent form +//! the target backend can represent. +//! +//! Legalization preserves semantics. A rule changes how an operation is +//! expressed, never what it computes — a backend that cannot express an +//! operation at all is rejected earlier, by `verify`. Syntax differences +//! between backends that share a representation (quoting, placeholders) are +//! the serializer's job (`toasty-sql`), not legalization's. +//! +//! One rule module exists today: [`document`], which rewrites `#[document]` +//! path reads into the resolved [`FuncJsonExtract`](stmt::FuncJsonExtract) +//! name paths drivers consume. Future per-backend rewrites join it as sibling +//! modules, invoked from [`Engine::legalize_statement`]. +//! +//! Entry points: [`Engine::prepare_for_driver`] legalizes a full statement +//! and extracts its bind parameters; [`Engine::legalize_table_expr`] +//! legalizes a bare table expression that crosses the boundary inside a +//! key-value operation. + +use super::Engine; +use toasty_core::{driver::operation::TypedValue, stmt}; + +mod document; + +impl Engine { + /// Prepare an engine-final statement to cross the driver boundary: + /// legalize it for the target backend, then (on SQL backends) extract its + /// bind parameters. The returned `Vec` is indexed by the `n` + /// in each `Expr::Arg(n)` placeholder; key-value backends read values + /// directly from the statement, so the vec is empty. + /// + /// Every driver-bound statement passes through this method, after the + /// last engine-side mutation (e.g. the MySQL `RETURNING` rewrites) and + /// immediately before the driver serializes it. + pub(crate) fn prepare_for_driver(&self, stmt: &mut stmt::Statement) -> Vec { + self.legalize_statement(stmt); + + if self.capability().sql() { + super::bind::run(stmt, &self.schema.db, self.capability()) + } else { + vec![] + } + } + + /// Legalize a full statement: run every rule module over it. + fn legalize_statement(&self, stmt: &mut stmt::Statement) { + document::statement(&self.schema, self.capability(), stmt); + } + + /// Legalize a driver-bound expression — a key-value operation's filter or + /// condition, which the driver compiles without an enclosing statement. + pub(crate) fn legalize_table_expr(&self, expr: &mut stmt::Expr) { + document::table_expr(&self.schema, self.capability(), expr); + } +} diff --git a/packages/toasty/src/engine/legalize/document.rs b/packages/toasty/src/engine/legalize/document.rs new file mode 100644 index 00000000000..c8028115013 --- /dev/null +++ b/packages/toasty/src/engine/legalize/document.rs @@ -0,0 +1,246 @@ +//! Legalization rule: rewrite `#[document]` path reads into the shape +//! drivers consume. +//! +//! The engine views a document column at the app level: the mapping's raising +//! cast (`Cast(column, Type::Model)`) turns the driver's named `Value::Object` +//! into the embed's positional `Value::Record`, and a path into the document +//! is a positional projection through that cast — +//! `Project(Cast(column, Model), steps)` — which the in-memory interpreter +//! evaluates directly. Drivers and the database schema see only the +//! structural view: the column is typed `Type::Object`, its value is a named +//! `Value::Object`, and a path is a resolved [`stmt::FuncJsonExtract`] name +//! path. The model identity never crosses the driver boundary. +//! +//! Value conversion between the two views is the mapping's job: the schema +//! builder plants a lowering cast (`Model` → `Object`, carrying the source +//! type — see `stmt::ExprCast::from`) in `model_to_table` and the raising +//! cast in `table_to_model`, and the cast machinery (fold/simplify constant +//! folding, `mir::Eval` evaluation) converts values wherever the casts +//! land. What cannot be expressed as a cast is the *path read*: turning a +//! positional projection into a JSON key path is a per-backend expression +//! rewrite (the leaf's comparison form depends on driver capabilities). That +//! makes it a legalization rule rather than a simplifier rewrite (the +//! simplifier runs identically for every backend) or a serializer concern +//! (the statement must already be representable when the driver receives it). + +use toasty_core::{ + driver::Capability, + schema::{Schema, app}, + stmt::{self, VisitMut}, +}; + +/// Rewrite every projection into a `#[document]` column into the +/// [`FuncJsonExtract`](stmt::FuncJsonExtract) node the SQL serializer +/// renders, and strip any residual raising cast (meaningless driver-side). +pub(super) fn statement(schema: &Schema, capability: &Capability, stmt: &mut stmt::Statement) { + LegalizeDocumentPaths { schema, capability }.visit_mut(stmt); +} + +/// Rewrite a driver-bound expression — a key-value operation's filter or +/// condition — into its driver-consumable shape: document paths become +/// [`stmt::FuncJsonExtract`] name paths, and text-compared document leaves +/// get text operands, exactly as [`statement`] does for full statements. +pub(super) fn table_expr(schema: &Schema, capability: &Capability, expr: &mut stmt::Expr) { + LegalizeDocumentPaths { schema, capability }.visit_expr_mut(expr); +} + +/// Traversal backing [`statement`]. Needs no scope tracking: the rewrite +/// matches the document-path shape structurally and takes the embed identity +/// from the raising cast's type — it never resolves a column reference +/// against the enclosing query's source. +struct LegalizeDocumentPaths<'a> { + schema: &'a Schema, + // Read only by `leaf_compares_as_text`, whose non-trivial arms are gated + // on the features below. + #[cfg_attr( + not(any(feature = "jiff", feature = "rust_decimal", feature = "bigdecimal")), + allow(dead_code) + )] + capability: &'a Capability, +} + +impl LegalizeDocumentPaths<'_> { + /// Rewrite a projection rooted at a `#[document]` column into the + /// [`FuncJsonExtract`](stmt::FuncJsonExtract) the SQL serializer renders. + /// + /// The lowered form of a document path is a positional projection through + /// the column's raising cast — `Project(Cast(column, Model), steps)`. + /// Driver-side there is no positional form: the projection becomes a JSON + /// key path rooted at the bare column reference. + fn lower_project(&self, expr: &mut stmt::Expr) { + let stmt::Expr::Project(project) = expr else { + return; + }; + // Only a projection through a document raising cast rooted at a + // column reference is a document path; anything else is left for + // later phases. + let stmt::Expr::Cast(cast) = &mut *project.base else { + return; + }; + let stmt::Type::Model(embed_id) = cast.ty else { + return; + }; + if !matches!(&*cast.expr, stmt::Expr::Reference(_)) { + return; + } + let Some((path, ty)) = + build_json_path(self.schema, embed_id, project.projection.as_slice()) + else { + return; + }; + let base = Box::new(cast.expr.take()); + *expr = stmt::Expr::from(stmt::FuncJsonExtract { base, path, ty }); + } + + /// Whether a document leaf of type `ty` compares as plain text on this + /// backend. PostgreSQL and MySQL cast the JSON extraction back to the + /// leaf's native SQL type, but a backend without that native type (SQLite + /// has no temporal or decimal column types) compares the extracted text + /// directly — so the bound operand must be the exact text the JSON codec + /// stores. + fn leaf_compares_as_text(&self, ty: &stmt::Type) -> bool { + match ty { + #[cfg(feature = "jiff")] + stmt::Type::Timestamp => !self.capability.native_timestamp, + #[cfg(feature = "jiff")] + stmt::Type::Date => !self.capability.native_date, + #[cfg(feature = "jiff")] + stmt::Type::Time => !self.capability.native_time, + #[cfg(feature = "jiff")] + stmt::Type::DateTime => !self.capability.native_datetime, + #[cfg(feature = "rust_decimal")] + stmt::Type::Decimal => !self.capability.native_decimal, + #[cfg(feature = "bigdecimal")] + stmt::Type::BigDecimal => !self.capability.native_decimal, + #[cfg(feature = "net")] + stmt::Type::Cidr => !self.capability.native_cidr, + #[cfg(feature = "net")] + stmt::Type::Inet => !self.capability.native_inet, + #[cfg(feature = "net")] + stmt::Type::MacAddr => !self.capability.native_macaddr, + #[cfg(feature = "net")] + stmt::Type::MacAddr8 => !self.capability.native_macaddr8, + _ => false, + } + } + + /// If `extract_side` is a JSON extraction whose leaf compares as text on + /// this backend, rewrite the constant `operand` to the leaf's document + /// text form (the exact text the JSON codec stores — see + /// [`document_text`]) and retype the extraction as a text read. + fn textify_comparison(&self, extract_side: &mut stmt::Expr, operand: &mut stmt::Expr) { + let stmt::Expr::Func(stmt::ExprFunc::JsonExtract(func)) = extract_side else { + return; + }; + if !self.leaf_compares_as_text(&func.ty) { + return; + } + let stmt::Expr::Value(value) = operand else { + return; + }; + let Some(text) = document_text(value) else { + return; + }; + *value = stmt::Value::String(text); + func.ty = stmt::Type::String; + } +} + +impl VisitMut for LegalizeDocumentPaths<'_> { + fn visit_expr_mut(&mut self, expr: &mut stmt::Expr) { + // A document path must be lowered before descending into its base: + // the base is the raising cast that names the embed, which the + // residual cast strip below would otherwise remove first (children + // are visited before their parent). + if let stmt::Expr::Project(_) = expr { + self.lower_project(expr); + } + + stmt::visit_mut::visit_expr_mut(self, expr); + + // Children are visited first, so by the time a comparison node is + // reached its document-path side is already a `FuncJsonExtract`. + match expr { + // A raising cast left outside a document path (a whole-document + // reference) is meaningless driver-side: the column already + // carries the named object. Strip it to the bare reference. + stmt::Expr::Cast(expr_cast) if expr_cast.ty.contains_model() => { + *expr = expr_cast.expr.take(); + } + stmt::Expr::BinaryOp(binary) => { + self.textify_comparison(&mut binary.lhs, &mut binary.rhs); + self.textify_comparison(&mut binary.rhs, &mut binary.lhs); + } + stmt::Expr::InList(in_list) => { + let stmt::Expr::Func(stmt::ExprFunc::JsonExtract(func)) = &mut *in_list.expr else { + return; + }; + if !self.leaf_compares_as_text(&func.ty) { + return; + } + match &mut *in_list.list { + stmt::Expr::List(list) => { + for item in &mut list.items { + if let stmt::Expr::Value(value) = item + && let Some(text) = document_text(value) + { + *value = stmt::Value::String(text); + } + } + } + stmt::Expr::Value(stmt::Value::List(items)) => { + for value in items.iter_mut() { + if let Some(text) = document_text(value) { + *value = stmt::Value::String(text); + } + } + } + _ => return, + } + func.ty = stmt::Type::String; + } + _ => {} + } + } +} + +/// Resolves a projection through (possibly nested) document types via the +/// schema's shared [`project_fields`](app::Schema::project_fields) walk, +/// collecting the JSON key path and the leaf field's type. Returns `None` if +/// the projection does not resolve to a document path. +fn build_json_path( + schema: &Schema, + embed_id: app::ModelId, + projection: &[usize], +) -> Option<(Vec, stmt::Type)> { + let mut path = Vec::with_capacity(projection.len()); + let mut leaf_ty = None; + + for field in schema.app.project_fields(embed_id, projection) { + path.push(field.name.app.as_deref()?.to_owned()); + leaf_ty = Some(field.expr_ty().clone()); + } + + // `project_fields` yields fewer fields than asked when the projection does + // not resolve to a document path (an out-of-range step, or a descent past + // a non-document leaf). + (!path.is_empty() && path.len() == projection.len()) + .then(|| (path, leaf_ty.expect("a non-empty path has a leaf type"))) +} + +/// The text form `value` takes inside a stored JSON document +/// ([`stmt::Value::document_storage_text`]), for comparison operands bound +/// against a plain-text extraction — exactly what the codec's `collect_str` +/// writes. `None` for values with no document text form (including `Null`, +/// which comparisons reach via `IsNull` instead). +fn document_text(value: &stmt::Value) -> Option { + #[cfg(any(feature = "jiff", feature = "rust_decimal", feature = "bigdecimal"))] + { + value.document_storage_text().map(|text| text.to_string()) + } + #[cfg(not(any(feature = "jiff", feature = "rust_decimal", feature = "bigdecimal")))] + { + let _ = value; + None + } +} diff --git a/packages/toasty/src/engine/lower.rs b/packages/toasty/src/engine/lower.rs new file mode 100644 index 00000000000..5e4ab607b28 --- /dev/null +++ b/packages/toasty/src/engine/lower.rs @@ -0,0 +1,2465 @@ +mod association; +mod expr_or; +mod include; +mod insert; +mod lift_in_subquery; +mod lift_update_query; +mod paginate; +mod relation; +pub(super) mod relation_expr; +mod relation_path; +mod returning; +mod via_join; + +#[cfg(test)] +mod tests; + +use std::cell::Cell; + +use indexmap::{IndexMap, IndexSet}; + +use index_vec::IndexVec; +use toasty_core::{ + Result, Schema, + driver::Capability, + schema::{ + app::{self, ModelRoot}, + db::ColumnId, + mapping, + }, + stmt::{self, IntoExprTarget, VisitMut, visit_mut}, +}; + +use crate::engine::{Engine, HirStatement, hir, simplify::Simplify, upsert}; + +/// Wrap a nullable single-relation subquery so a missing row passes through as +/// `Null`, while a present row is transformed by `present`. Used when lowering +/// `.include()`/`.select()` of nullable single (`Deferred>` / +/// `Deferred>`) relations that need to project out of the subquery +/// result: +/// +/// ```text +/// Let { +/// binding: subquery, +/// body: Match { subject: arg(0), arms: [Null → Null], else: present }, +/// } +/// ``` +/// +/// The subquery result is bound as `arg(0)`; `present` is the expression for a +/// non-null row and may reference that binding (e.g. project a field out of it). +fn map_nullable_single(subquery: stmt::Expr, present: stmt::Expr) -> stmt::Expr { + stmt::Expr::Let(stmt::ExprLet { + bindings: vec![subquery], + body: Box::new(stmt::Expr::match_expr( + stmt::Expr::arg(0), + vec![stmt::MatchArm { + pattern: stmt::Value::Null, + expr: stmt::Expr::Value(stmt::Value::Null), + }], + present, + )), + }) +} + +impl Engine { + pub(super) fn lower_stmt(&self, stmt: stmt::Statement) -> Result { + let schema = &self.schema; + + let mut state = LoweringState { + hir: HirStatement::new(), + scopes: IndexVec::new(), + engine: self, + relations: vec![], + errors: vec![], + dependencies: IndexSet::new(), + insert_stmts: vec![], + update_returning: IndexMap::new(), + }; + + state.lower_stmt(stmt::ExprContext::new(schema), None, stmt); + + if let Some(err) = state.errors.into_iter().next() { + return Err(err); + } + + Ok(state.hir) + } +} + +impl LoweringState<'_> { + fn lower_stmt( + &mut self, + expr_cx: stmt::ExprContext, + row_index: Option, + mut stmt: stmt::Statement, + ) -> hir::StmtId { + // App-level rewrites that lowering depends on. `Source::Model { via }` + // must be converted to a WHERE filter before the lowering walk + // converts `Source::Model` into `Source::Table`. The IN-subquery + // lift fires here too: code paths outside the lowering walk (e.g. + // `ApplyInsertScope::apply_expr`) see the already-lifted form. + // `UpdateTarget::Query` is lifted into `UpdateTarget::Model` with + // the inner query's filter merged onto the outer update before the + // walk sees the target. The eq/ne operand rewrite (model→PK, + // BelongsTo→FK) fires inside the lowering walk itself via + // `LowerStatement::visit_expr_binary_op_mut`. + association::RewriteVia::new(expr_cx).rewrite(&mut stmt); + lift_in_subquery::LiftInSubquery::new(expr_cx, self.engine.capability.sql()) + .rewrite(&mut stmt); + lift_update_query::LiftUpdateQuery::new().rewrite(&mut stmt); + + // Combine compatible IN subqueries after relation lifting and before + // the lowering walk extracts them into separate NoSQL statements. + Simplify::with_context(expr_cx, self.engine.capability).visit_mut(&mut stmt); + + let stmt_id = self.hir.new_statement_info( + self.dependencies + .iter() + .map(|&dep| (dep, hir::DepKind::Statement)) + .collect(), + ); + let scope_id = self.scopes.push(Scope { stmt_id, row_index }); + let mut collect_dependencies = None; + + // Map the statement + LowerStatement { + state: self, + expr_cx, + scope_id, + cx: LoweringContext::Statement, + collect_dependencies: &mut collect_dependencies, + } + .visit_stmt_mut(&mut stmt); + + self.engine.simplify_stmt(&mut stmt); + + let stmt_info = &mut self.hir[stmt_id]; + stmt_info.stmt = Some(Box::new(stmt)); + + self.scopes.pop(); + + debug_assert!(collect_dependencies.is_none()); + + stmt_id + } +} + +struct LowerStatement<'a, 'b> { + /// Lowering state. This is the state that is constant throughout the entire + /// lowering process. + state: &'a mut LoweringState<'b>, + + /// Expression context in which the statement is being lowered + expr_cx: stmt::ExprContext<'a>, + + /// Identifier to the current scope (stored in `scopes` on LoweringState) + scope_id: ScopeId, + + /// Current lowering context + cx: LoweringContext<'a>, + + /// Track dependencies here. + collect_dependencies: &'a mut Option>, +} + +#[derive(Debug)] +struct LoweringState<'a> { + /// Lowered assignments for updates whose returning expressions are active. + /// Sub-statements that load relations use assigned constants instead of + /// reading them from the update's returned row. + update_returning: IndexMap, + /// Database engine handle + engine: &'a Engine, + + /// Statements to be executed by the database, though they may still be + /// broken down into multiple sub-statements. + hir: HirStatement, + + /// Scope state + scopes: IndexVec, + + /// Planning a query can require walking relations to maintain data + /// consistency. This field tracks the current relation edge being traversed + /// so the planner doesn't walk it backwards. + relations: Vec, + + /// All new statements should include these as part of its dependencies + dependencies: IndexSet, + + /// INSERT statements currently being lowered, outermost first. A + /// `belongs_to` returning-load subquery depends on these so it executes + /// after enclosing inserts (a nested create inserts children before the + /// parent, and the loaded row may be the parent's). + insert_stmts: Vec, + + /// Tracks errors that occurred while lowering the statement + errors: Vec, +} + +#[derive(Debug, Clone, Copy)] +enum LoweringContext<'a> { + /// Lowering an insertion statement + Insert(&'a [ColumnId], Option), + + /// Lowering a value row being inserted + InsertRow(&'a stmt::Expr), + + /// Lowering the returning clause of a statement. Optionally carries the + /// parent INSERT's row index when visiting a per-row returning expression. + Returning(Option), + + /// All other lowering cases + Statement, +} + +#[derive(Debug)] +struct Scope { + /// Identifier of the statement in the lowering state + stmt_id: hir::StmtId, + + /// If the statement is called from an insert's values (i.e. the parent + /// statement is an insert), this tracks the row index + row_index: Option, +} + +index_vec::define_index_type! { + struct ScopeId = u32; +} + +/// A collection operator on a `Vec` field. +/// +/// `Append`, `Remove`, `Pop`, and `RemoveAt` are all collection operators: +/// they target a single-column `Vec` field and carry operator +/// operands (a list of elements, a single element, an index, or nothing). +/// Whether the operand is a list or a scalar is a per-operator encoding +/// detail — the unifying property is that none carries a *field value* +/// that decomposes across columns, so none needs the `model_to_table` +/// substitution that `Set` flows through. +enum CollectionOp<'a> { + Append(&'a mut stmt::Expr), + Remove(&'a mut stmt::Expr), + Pop, + RemoveAt(&'a mut stmt::Expr), +} + +/// Arithmetic assignment operator on a numeric scalar field. +/// +/// `Add` and `Subtract` carry a value to combine atomically with the +/// current column value (`col = col ± expr`). Like collection ops, they +/// target a single-column primitive field and skip the `model_to_table` +/// substitution. +enum ArithmeticOp { + Add, + Subtract, +} + +impl LowerStatement<'_, '_> { + /// Lower a `Set` assignment — the only assignment that carries a whole + /// *field value*. + /// + /// The field may span multiple columns (embedded structs decompose + /// across columns; enums into a discriminant plus variant columns), so + /// this resolves the field mapping and, for each impacted column, + /// substitutes the lowered value into that column's `model_to_table` + /// template before emitting the column-level write. + fn lower_set_assignment( + &mut self, + out: &mut stmt::Assignments, + mapping: &mapping::Model, + projection: &stmt::Projection, + expr: &mut stmt::Expr, + ) { + self.visit_expr_mut(expr); + + let Some(field) = mapping.resolve_field_mapping(projection) else { + self.state + .errors + .push(crate::Error::invalid_statement(format!( + "invalid assignment projection: {projection:?}" + ))); + return; + }; + + for (column, lowering_idx) in field.columns() { + let mut lowering_expr = mapping.model_to_table[lowering_idx].clone(); + + // Substitute the field reference in the column's lowering + // template with the lowered value. + lowering_expr.substitute(AssignmentInput { + assignment_projection: projection.clone(), + value: expr, + }); + + self.visit_expr_mut(&mut lowering_expr); + + out.set(column, lowering_expr); + } + } + + /// Lower a collection operator (`Append` / `Remove` / `Pop` / + /// `RemoveAt`) on a `Vec` field. + /// + /// `Vec` fields always resolve to a single primitive column, so + /// this skips the `model_to_table` substitution that `lower_set_assignment` + /// performs — the operands are operator arguments, not a field value, + /// and for a single-column field `model_to_table` is identity anyway. + /// + /// `Append` is supported on every backend; the removal operators are + /// gated by per-backend capability flags and emit a clear error where + /// the native form is not available. On document collections the + /// removal operators are rejected outright: the per-backend renderings + /// are native-array forms that do not apply to a document column. + fn lower_collection_op( + &mut self, + out: &mut stmt::Assignments, + mapping: &mapping::Model, + projection: &stmt::Projection, + op: CollectionOp, + ) { + let Some(field) = mapping.resolve_field_mapping(projection) else { + self.state + .errors + .push(crate::Error::invalid_statement(format!( + "invalid assignment projection: {projection:?}" + ))); + return; + }; + + let Some(prim) = field.as_primitive() else { + self.state + .errors + .push(crate::Error::invalid_statement(format!( + "collection operator on non-primitive field: {projection:?}" + ))); + return; + }; + + // `Append` is universally supported; the removal operators are + // gated per backend and rejected on document collections, where + // the capability flags speak for the native-array renderings only. + let cap = self.capability(); + let is_document = self + .expr_cx + .schema() + .mapping + .document_column_ty(prim.column) + .is_some(); + let unsupported = match &op { + CollectionOp::Append(_) => None, + CollectionOp::Remove(_) if is_document || !cap.vec_remove => Some("stmt::remove"), + CollectionOp::Pop if is_document || !cap.vec_pop => Some("stmt::pop"), + CollectionOp::RemoveAt(_) if is_document || !cap.vec_remove_at => { + Some("stmt::remove_at") + } + _ => None, + }; + + if let Some(op_name) = unsupported { + let target = if is_document { + "document collections" + } else { + "this backend" + }; + self.state + .errors + .push(crate::Error::invalid_statement(format!( + "{op_name} is not yet supported on {target}" + ))); + return; + } + + match op { + CollectionOp::Append(expr) => { + self.visit_expr_mut(expr); + let mut value = expr.take(); + + // A document collection's append operand (a list of elements, + // matching the column's document type) lowers through the + // same schema-directed cast a `Set` picks up from + // `model_to_table`; `Append` bypasses that template, so the + // cast is applied here. + let schema = self.expr_cx.schema(); + if let Some(doc_ty) = schema.mapping.document_column_ty(prim.column) { + let column_ty = &schema.db.column(prim.column).ty; + value = stmt::Expr::cast_from(value, doc_ty, column_ty); + } + + out.append(prim.column, value); + } + CollectionOp::Remove(expr) => { + self.visit_expr_mut(expr); + out.remove(prim.column, expr.take()); + } + CollectionOp::Pop => { + out.pop(prim.column); + } + CollectionOp::RemoveAt(expr) => { + self.visit_expr_mut(expr); + out.remove_at(prim.column, expr.take()); + } + } + } + + /// Lower an arithmetic assignment operator (`Add` / `Subtract`) on a + /// scalar numeric field. Mirrors `lower_collection_op`: single primitive + /// column, no `model_to_table` substitution needed. + fn lower_arithmetic_op( + &mut self, + out: &mut stmt::Assignments, + mapping: &mapping::Model, + projection: &stmt::Projection, + op: ArithmeticOp, + expr: &mut stmt::Expr, + ) { + let Some(field) = mapping.resolve_field_mapping(projection) else { + self.state + .errors + .push(crate::Error::invalid_statement(format!( + "invalid assignment projection: {projection:?}" + ))); + return; + }; + + let Some(prim) = field.as_primitive() else { + self.state + .errors + .push(crate::Error::invalid_statement(format!( + "arithmetic operator on non-primitive field: {projection:?}" + ))); + return; + }; + + self.visit_expr_mut(expr); + let value = expr.take(); + match op { + ArithmeticOp::Add => out.add(prim.column, value), + ArithmeticOp::Subtract => out.subtract(prim.column, value), + } + } + + /// Dispatch a single owned assignment to the matching per-kind lowering + /// helper. `Batch` first folds via [`fold_batch`] and then recurses on + /// the folded result. A residual `Batch` (from a non-composable pair) + /// is unsupported today; see #717. + fn lower_owned_assignment( + &mut self, + out: &mut stmt::Assignments, + mapping: &mapping::Model, + projection: &stmt::Projection, + assignment: stmt::Assignment, + ) { + match assignment { + stmt::Assignment::Set(mut expr) => { + self.lower_set_assignment(out, mapping, projection, &mut expr); + } + stmt::Assignment::Append(mut expr) => { + self.lower_collection_op(out, mapping, projection, CollectionOp::Append(&mut expr)); + } + stmt::Assignment::Remove(mut expr) => { + self.lower_collection_op(out, mapping, projection, CollectionOp::Remove(&mut expr)); + } + stmt::Assignment::Pop => { + self.lower_collection_op(out, mapping, projection, CollectionOp::Pop); + } + stmt::Assignment::RemoveAt(mut expr) => { + self.lower_collection_op( + out, + mapping, + projection, + CollectionOp::RemoveAt(&mut expr), + ); + } + stmt::Assignment::Add(mut expr) => { + self.lower_arithmetic_op(out, mapping, projection, ArithmeticOp::Add, &mut expr); + } + stmt::Assignment::Subtract(mut expr) => { + self.lower_arithmetic_op( + out, + mapping, + projection, + ArithmeticOp::Subtract, + &mut expr, + ); + } + stmt::Assignment::Batch(entries) => { + let folded = fold_batch(entries); + if matches!(folded, stmt::Assignment::Batch(_)) { + todo!("non-composable batch shape on {projection:?}: {folded:#?}") + } + self.lower_owned_assignment(out, mapping, projection, folded); + } + stmt::Assignment::Insert(_) => { + // `Insert` only applies to has-many relation fields, which + // `plan_stmt_update_relations` consumes before table-level + // lowering runs. + todo!("Insert assignment is not produced for table lowering; got {assignment:#?}") + } + } + } +} + +/// Fold a `Batch` of column-level ops on the same field into a single +/// equivalent assignment, applied left-to-right via [`compose_assignment`]. +/// +/// The result is a non-`Batch` assignment when every pair composes; if any +/// pair is not algebraically reducible (today: `Pop`/`Remove`/`RemoveAt` +/// interleaved with other ops, or non-literal `Append` concat), the +/// residual is returned as a `Batch` so the caller can error with the full +/// shape for diagnostics. +fn fold_batch(entries: Vec) -> stmt::Assignment { + let mut iter = entries.into_iter(); + let mut acc = iter.next().expect("batch is non-empty"); + for next in iter { + acc = compose_assignment(acc, next); + } + acc +} + +/// Compose two assignments on the same field into a single equivalent +/// assignment, in execution order. Returns the un-composed pair as a +/// `Batch` when the combination is not algebraically reducible — the +/// caller treats a residual `Batch` as an unsupported shape. +fn compose_assignment(acc: stmt::Assignment, next: stmt::Assignment) -> stmt::Assignment { + use stmt::Assignment::*; + match (acc, next) { + // `Set` clobbers everything that came before it. + (_, Set(v)) => Set(v), + + // Arithmetic following `Set` rewrites the stored value. + (Set(v), Add(x)) => Set(stmt::Expr::add(v, x)), + (Set(v), Subtract(x)) => Set(stmt::Expr::sub(v, x)), + + // Arithmetic on arithmetic: combine operands. With `Subtract` as + // the outer op, subsequent operand signs flip + // (col - a + b = col - (a - b); col - a - b = col - (a + b)). + (Add(a), Add(x)) => Add(stmt::Expr::add(a, x)), + (Add(a), Subtract(x)) => Add(stmt::Expr::sub(a, x)), + (Subtract(a), Add(x)) => Subtract(stmt::Expr::sub(a, x)), + (Subtract(a), Subtract(x)) => Subtract(stmt::Expr::add(a, x)), + + // Two literal-list `Append`s concatenate. Non-literal Append + // shapes fall through to a residual `Batch`. + (Append(a), Append(b)) => try_concat_list_literals_to_assignment(a, b), + + // A `Batch` accumulator absorbs the next entry as a tail. Arises + // when a previous compose failed to reduce a pair; collecting the + // rest preserves the full shape for the caller's error message. + (Batch(mut tail), next) => { + tail.push(next); + Batch(tail) + } + + // Not composable today — return as a residual `Batch`. + (acc, next) => Batch(vec![acc, next]), + } +} + +/// Concatenate two list-shaped expressions if both are literals (either +/// `Expr::List` or `Expr::Value(Value::List)`). Returns a residual `Batch` +/// containing the original pair if either expression is not a literal. +fn try_concat_list_literals_to_assignment(a: stmt::Expr, b: stmt::Expr) -> stmt::Assignment { + use stmt::Assignment::{Append, Batch}; + if !is_list_literal(&a) || !is_list_literal(&b) { + return Batch(vec![Append(a), Append(b)]); + } + let mut items = take_list_items(a).expect("checked is_list_literal"); + items.extend(take_list_items(b).expect("checked is_list_literal")); + Append(stmt::Expr::list_from_vec(items)) +} + +fn is_list_literal(e: &stmt::Expr) -> bool { + matches!( + e, + stmt::Expr::List(_) | stmt::Expr::Value(stmt::Value::List(_)) + ) +} + +fn take_list_items(e: stmt::Expr) -> Option> { + match e { + stmt::Expr::List(list) => Some(list.items), + stmt::Expr::Value(stmt::Value::List(values)) => { + Some(values.into_iter().map(stmt::Expr::from).collect()) + } + _ => None, + } +} + +impl LowerStatement<'_, '_> { + fn new_dependency(&mut self, stmt: impl Into) -> hir::StmtId { + let row_index = match self.cx { + LoweringContext::Insert(_, row_index) => row_index, + LoweringContext::Returning(row_index) => row_index, + _ => None, + }; + + let stmt_id = self.state.lower_stmt(self.expr_cx, row_index, stmt.into()); + + if let Some(dependencies) = &mut self.collect_dependencies { + dependencies.insert(stmt_id); + } + + self.curr_stmt_info() + .add_dep(stmt_id, hir::DepKind::Statement); + + stmt_id + } + + fn collect_dependencies( + &mut self, + f: impl FnOnce(&mut LowerStatement<'_, '_>), + ) -> IndexSet { + let old = self.collect_dependencies.replace(IndexSet::new()); + f(self); + std::mem::replace(self.collect_dependencies, old).unwrap() + } + + fn track_dependency(&mut self, dependency: hir::StmtId) { + self.curr_stmt_info() + .add_dep(dependency, hir::DepKind::Statement); + } + + fn with_dependencies( + &mut self, + mut dependencies: IndexSet, + f: impl FnOnce(&mut LowerStatement<'_, '_>), + ) { + // Dependencies should stack + dependencies.extend(&self.state.dependencies); + + let old = std::mem::replace(&mut self.state.dependencies, dependencies); + f(self); + self.state.dependencies = old; + } +} + +impl visit_mut::VisitMut for LowerStatement<'_, '_> { + fn visit_stmt_mut(&mut self, stmt: &mut stmt::Statement) { + if let stmt::Statement::Query(query) = stmt + && matches!( + &query.limit, + Some(stmt::Limit::Cursor(cursor)) if cursor.after.is_some() + ) + { + self.curr_stmt_info().has_pagination_cursor = true; + } + + visit_mut::visit_stmt_mut(self, stmt); + } + + fn visit_order_by_expr_mut(&mut self, node: &mut stmt::OrderByExpr) { + // First, run the default visitor to lower sub-expressions + self.visit_expr_mut(&mut node.expr); + + // An embedded newtype field lowers to a single-element record, + // one layer per newtype in the chain. Unwrap them so the ordering + // applies to the underlying column — otherwise the eq synthesis + // below would hit the record-vs-record arm of + // `lower_expr_binary_op`, which drains the operand records and + // would leave an empty record behind. + while let stmt::Expr::Record(rec) = &mut node.expr + && rec.len() == 1 + { + node.expr = rec.fields.pop().unwrap(); + } + + // Reuse binary-op lowering: synthesize `expr == expr` so that + // cast conversions are applied, then keep the LHS result. + let mut lhs = node.expr.clone(); + let mut rhs = node.expr.take(); + self.lower_expr_binary_op(stmt::BinaryOp::Eq, &mut lhs, &mut rhs); + node.expr = lhs; + } + + fn visit_assignments_mut(&mut self, i: &mut stmt::Assignments) { + let mut lowered = stmt::Assignments::default(); + let mapping = self.mapping_unwrap(); + let assignments = std::mem::take(i); + for (projection, assignment) in assignments { + self.lower_owned_assignment(&mut lowered, mapping, &projection, assignment); + } + *i = lowered; + } + + fn visit_expr_set_op_mut(&mut self, i: &mut stmt::ExprSetOp) { + todo!("stmt={i:#?}"); + } + + fn visit_expr_binary_op_mut(&mut self, i: &mut stmt::ExprBinaryOp) { + // App-level operand rewrite for `eq` / `ne` (`Reference::Model` → + // primary-key field, `BelongsTo` → foreign-key field). Must fire + // before the operands are walked, since walking lowers the field + // references into column references and the rewrite has nothing to + // match on. Nested binary ops are handled by the recursive walk + // through `visit_expr_mut`. + if i.op.is_eq() || i.op.is_ne() { + self.rewrite_eq_operand(&mut i.lhs); + self.rewrite_eq_operand(&mut i.rhs); + } + + stmt::visit_mut::visit_expr_binary_op_mut(self, i); + } + + fn visit_expr_in_list_mut(&mut self, i: &mut stmt::ExprInList) { + // App-level operand rewrite: `Reference::Model { nesting } IN list` + // becomes ` IN list`. Must fire before the LHS is walked, + // since walking lowers the model reference into something the rewrite + // can no longer match on. + self.rewrite_in_list_model_operand(i); + + stmt::visit_mut::visit_expr_in_list_mut(self, i); + } + + fn visit_expr_mut(&mut self, expr: &mut stmt::Expr) { + self.plan_typed_record_relations(expr); + match expr { + stmt::Expr::BinaryOp(e) => { + self.visit_expr_binary_op_mut(e); + + if let Some(lowered) = self.lower_expr_binary_op(e.op, &mut e.lhs, &mut e.rhs) { + *expr = lowered; + } + } + // App-level rewrite: an OR covering every variant of an enum + // via `IsVariant` is a tautology. Must fire before the children + // walk lowers the `IsVariant` nodes into discriminant + // comparisons, since the rewrite pattern-matches on + // `Expr::IsVariant`. + stmt::Expr::Or(e) if expr_or::is_variant_tautology_or(self.expr_cx.schema(), e) => { + *expr = true.into(); + } + stmt::Expr::InList(e) => { + self.visit_expr_in_list_mut(e); + + if let Some(lowered) = self.lower_expr_in_list(&mut e.expr, &mut e.list) { + *expr = lowered; + } + + // PostgreSQL-style: `x IN (a, b, c)` → `x = ANY($1)` with the + // list bound as a single array param. + if let stmt::Expr::InList(e) = expr + && self.supports_any_rewrite() + && in_list_is_value_list(e) + { + let stmt::Expr::InList(e) = expr.take() else { + unreachable!() + }; + *expr = stmt::Expr::any_op(*e.expr, stmt::BinaryOp::Eq, *e.list); + } + } + stmt::Expr::Not(e) if matches!(*e.expr, stmt::Expr::InList(_)) => { + // Recurse into inner first so the InList itself is lowered. + self.visit_expr_not_mut(e); + + // If the inner is still an InList (not yet rewritten because + // the gate is off or the list shape doesn't qualify), leave + // `Not(InList)` alone and let the SQL serializer render + // `NOT IN`. Otherwise, rewrite `NOT (x = ANY(arr))` into the + // canonical `x <> ALL(arr)` form. + if self.supports_any_rewrite() + && let stmt::Expr::AnyOp(any) = e.expr.as_mut() + && any.op == stmt::BinaryOp::Eq + { + let stmt::Expr::Not(not) = expr.take() else { + unreachable!() + }; + let stmt::Expr::AnyOp(any) = *not.expr else { + unreachable!() + }; + *expr = stmt::Expr::all_op(*any.lhs, stmt::BinaryOp::Ne, *any.rhs); + } + } + stmt::Expr::InSubquery(e) => { + if self.capability().sql() { + self.visit_expr_in_subquery_mut(e); + + self.lower_in_subquery_operands( + &mut e.expr, + e.query.returning_mut_unwrap().as_project_mut_unwrap(), + ); + + let returning = e.query.returning_mut_unwrap().as_project_mut_unwrap(); + + if !returning.is_record() { + *returning = stmt::Expr::record([returning.take()]); + } + } else { + self.visit_expr_mut(&mut e.expr); + + let source_id = self.scope_stmt_id(); + let target_id = self.scope_statement(|child| { + child.visit_stmt_query_mut(&mut e.query); + }); + + // The subquery must be executable without parent context: + // no `Arg::Ref` (which would reach back into a parent + // scope) and no `back_refs` (columns the parent must + // batch-load on its behalf). `Arg::Sub` is fine — it's a + // forward dependency on the subquery's own child, which + // the planner chains in execution order. + let target_stmt_info = &self.state.hir[target_id]; + debug_assert!( + target_stmt_info + .args + .iter() + .all(|arg| matches!(arg, hir::Arg::Sub { .. })), + "TODO: sub-statement references parent scope" + ); + debug_assert!(target_stmt_info.back_refs.is_empty(), "TODO"); + + self.track_dependency(target_id); + + self.lower_in_subquery_operands( + &mut e.expr, + e.query.returning_mut_unwrap().as_project_mut_unwrap(), + ); + + let stmt::Expr::InSubquery(e) = expr.take() else { + panic!() + }; + + let mut stmt: stmt::Statement = (*e.query).into(); + + // Post-lower simplify. The sub-statement detaches into its + // own HIR entry (`new_sub_statement`), so the parent's + // post-lower simplify never sees it — without this, raw + // lowered shapes (e.g. an embedded-field path, lowered to + // `Project(Record([column]), [i])`) reach the driver. + self.state.engine.simplify_stmt(&mut stmt); + + let arg = self.new_sub_statement(source_id, target_id, Box::new(stmt)); + + let membership: stmt::Expr = stmt::ExprInList { + expr: e.expr, + list: Box::new(arg), + } + .into(); + *expr = if e.negated { + stmt::Expr::not(membership) + } else { + membership + }; + } + } + stmt::Expr::IsVariant(e) => { + // Look up the enum model and variant directly via VariantId + let enum_model = self + .schema() + .app + .model(e.variant.model) + .as_embedded_enum_unwrap(); + let has_data = enum_model.has_data_variants(); + let disc_value = enum_model.variants[e.variant.index].discriminant.clone(); + + // Lower the inner expression + self.visit_expr_mut(&mut e.expr); + + let lowered_expr = collapse_projections(e.expr.take()); + + // Emit the appropriate comparison + *expr = match lowered_expr { + // Data-carrying, lowered to the enum's decode: compare + // the decode's subject, the discriminant column, rather + // than distributing a `project([0])` over the arms — a + // unit variant's arm is the bare discriminant, which + // cannot be projected into. + stmt::Expr::Match(decode) if has_data => { + stmt::Expr::eq(*decode.subject, stmt::Expr::Value(disc_value)) + } + // Data-carrying, in value form: project([0]) to extract + // the discriminant from the record. + lowered_expr if has_data => stmt::Expr::eq( + stmt::Expr::project(lowered_expr, [0usize]), + stmt::Expr::Value(disc_value), + ), + // Unit-only: compare directly + lowered_expr => stmt::Expr::eq(lowered_expr, stmt::Expr::Value(disc_value)), + }; + } + stmt::Expr::Variant(e) => { + // The selected variant's payload. The base lowers to the + // enum's decode — a `Match` on the discriminant column whose + // arms hold `Record([disc, field columns...])` — and the + // selection takes the matching arm's fields, discriminant + // dropped, so the projections above it index the variant's + // fields directly. No check is emitted here: the predicate + // carries its own `is_variant` guard, lowered separately. + let disc_value = self + .schema() + .app + .model(e.variant.model) + .as_embedded_enum_unwrap() + .variants[e.variant.index] + .discriminant + .clone(); + + self.visit_expr_mut(&mut e.base); + + *expr = variant_payload(e.base.take(), &disc_value); + } + stmt::Expr::Project(project) + if let stmt::Expr::Incoming(stmt::ExprIncoming::Model(model)) = + project.base.as_ref() => + { + let (table, column) = { + let mapping = self.schema().mapping_for(*model); + let mapped = mapping + .resolve_field_mapping(&project.projection) + .expect("incoming upsert field must map to a column"); + let mut columns = mapped.columns(); + let (column, _) = columns + .next() + .expect("incoming upsert field has no database column"); + assert!( + columns.next().is_none(), + "incoming() does not yet support column-expanded embedded fields" + ); + (mapping.table, column) + }; + debug_assert_eq!(table, column.table); + *project.base = stmt::ExprIncoming::table(table).into(); + project.projection = stmt::Projection::from_index(column.index); + } + // A null-check on an `Option` field reduces to a null-check on + // the embed's head column instead of distributing the check over + // every flattened column. The head column is `NULL` for `None`, so + // for `Account::fields().contact().is_none()` (an `Option`) + // this emits `contact IS NULL` rather than checking each variant + // column. Normalization expands `.is_some()` to `Not(IsNull(..))`, + // so the surrounding `Not` yields `contact IS NOT NULL`. Scalar + // `Option` and non-nullable fields fall through to the normal + // reference lowering below. + // + // Only fires for WHERE-clause predicates (`Statement` context), + // where a field reference lowers to a column. The `model_to_table` + // encode wraps `is_not_null(field)` over the head column too; in the + // `InsertRow` encode context the field reference must resolve to the + // row value (and `is_null` then folds), so this rewrite must not + // intercept it. (The UPDATE encode substitutes the value before + // re-visiting, so the inner is no longer a field reference there.) + stmt::Expr::IsNull(e) => { + let negated = e.negated; + let head_column = match &*e.expr { + stmt::Expr::Reference(stmt::ExprReference::Field { nesting, index }) + if self.cx.is_statement() => + { + // The head column whose null-ness is the option's + // none-ness: a struct embed's presence column, or an + // enum embed's (nullable) discriminant column. A + // non-nullable embed's `is_null` is already folded to + // `false` in simplify, so reaching here implies nullable. + self.mapping_at_unwrap(*nesting) + .fields + .get(*index) + .and_then(|f| match f { + mapping::Field::Struct(fs) => fs.presence.map(|p| p.index), + mapping::Field::Enum(fe) => Some(fe.discriminant.column.index), + _ => None, + }) + .map(|column| (*nesting, column)) + } + _ => None, + }; + + if let Some((nesting, column)) = head_column { + let mut col_ref = stmt::Expr::column(stmt::ExprColumn { + nesting, + table: 0, + column, + }); + self.visit_expr_mut(&mut col_ref); + *expr = if negated { + stmt::Expr::is_not_null(col_ref) + } else { + stmt::Expr::is_null(col_ref) + }; + } else { + stmt::visit_mut::visit_expr_mut(self, expr); + } + } + stmt::Expr::Reference(expr_reference) => { + match expr_reference { + stmt::ExprReference::Field { nesting, index } => { + *expr = self.lower_expr_field(*nesting, *index); + self.visit_expr_mut(expr); + } + stmt::ExprReference::Model { .. } => todo!(), + stmt::ExprReference::Column(expr_column) => { + if expr_column.nesting > 0 { + let source_id = self.scope_stmt_id(); + let target_id = self.resolve_stmt_id(expr_column.nesting); + + // A relation subquery can use a constant assigned by + // the enclosing update without reading its returned row. + if let Some(update_assignments) = + self.state.update_returning.get(&target_id) + && let Some(stmt::Assignment::Set(assigned_value)) = + update_assignments.get(&[expr_column.column]) + && assigned_value.is_const() + { + *expr = assigned_value.clone(); + return; + } + + // the current scope ID should also be the top of the stack + debug_assert_eq!(self.state.scopes.len(), self.scope_id + 1); + + // The statement is not independent. Walk up the + // scope stack until the referened target statement + // and flag any intermediate statements as also not + // indepdnendent. + for scope in self.state.scopes.iter().rev() { + if scope.stmt_id == target_id { + break; + } + + self.state.hir[scope.stmt_id].independent = false; + } + + let position = self.new_ref(source_id, target_id, *expr_reference); + + // Using ExprArg as a placeholder. It will be rewritten + // later. + *expr = stmt::Expr::arg(position); + } + } + } + } + stmt::Expr::Stmt(_) => { + let stmt::Expr::Stmt(mut expr_stmt) = expr.take() else { + panic!() + }; + + // Expr::Stmt subqueries are valid in returning expressions (e.g., + // INCLUDE preloading) and in VALUES bodies of batch queries. + debug_assert!( + self.cx.is_returning() || matches!(self.cx, LoweringContext::Statement), + "cx={:#?}", + self.cx, + ); + + // For now, we assume nested sub-statements cannot be executed on the + // target database. Eventually, we will need to make this smarter. + let source_id = self.scope_stmt_id(); + let target_id = self.scope_statement(|child| { + visit_mut::visit_expr_stmt_mut(child, &mut expr_stmt); + }); + + // Post-lower simplify. The sub-statement detaches into its own + // HIR entry (`new_sub_statement`), so the parent statement's + // post-lowering simplify never sees it — the heavyweight rules + // (e.g. the schema-directed `#[document]` cast folding) must + // run here. + self.state.engine.simplify_stmt(&mut *expr_stmt.stmt); + + *expr = self.new_sub_statement(source_id, target_id, expr_stmt.stmt); + + if self.state.hir[target_id].independent { + self.curr_stmt_info() + .add_dep(target_id, hir::DepKind::Statement); + } + } + stmt::Expr::Exists(_) if !self.capability().sql() => { + let stmt::Expr::Exists(mut expr_exists) = expr.take() else { + panic!() + }; + + // Extract the EXISTS subquery into a sub-statement so the + // executor can evaluate the condition in memory. + let source_id = self.scope_stmt_id(); + let target_id = self.scope_statement(|child| { + child.visit_stmt_query_mut(&mut expr_exists.subquery); + }); + + let mut stmt = stmt::Statement::Query(*expr_exists.subquery); + // Post-lower simplify. The sub-statement detaches into its own + // HIR entry (`new_sub_statement`), so the parent statement's + // post-lowering simplify never sees it — the heavyweight rules + // must run here. + self.state.engine.simplify_stmt(&mut stmt); + + let arg = self.new_sub_statement(source_id, target_id, Box::new(stmt)); + + if self.state.hir[target_id].independent { + self.curr_stmt_info() + .add_dep(target_id, hir::DepKind::Statement); + } + + // The sub-statement result is a list of rows. Wrap it in + // a single-row VALUES query so the evaluator unwraps the + // outer list, letting EXISTS check the inner row count. + let mut subquery = stmt::Query::values(arg); + subquery.single = true; + *expr = stmt::Expr::Exists(stmt::ExprExists { + subquery: Box::new(subquery), + }); + } + _ => { + // Recurse down the statement tree + stmt::visit_mut::visit_expr_mut(self, expr); + } + } + } + + fn visit_insert_target_mut(&mut self, i: &mut stmt::InsertTarget) { + match i { + stmt::InsertTarget::Scope(_) => todo!("stmt={i:#?}"), + stmt::InsertTarget::Model(model_id) => { + let mapping = self.schema().mapping_for(model_id); + *i = stmt::InsertTable { + table: mapping.table, + columns: mapping.columns.clone(), + } + .into(); + } + _ => todo!(), + } + } + + fn visit_update_target_mut(&mut self, i: &mut stmt::UpdateTarget) { + match i { + stmt::UpdateTarget::Query(_) => todo!("update_target={i:#?}"), + stmt::UpdateTarget::Model(model_id) => { + let table_id = self.schema().table_id_for(model_id); + *i = stmt::UpdateTarget::table(table_id); + } + stmt::UpdateTarget::Table(_) => {} + } + } + + fn visit_expr_stmt_mut(&mut self, i: &mut stmt::ExprStmt) { + // Delegate to the default visitor which dispatches to + // visit_stmt_insert_mut, visit_stmt_query_mut, etc. + stmt::visit_mut::visit_expr_stmt_mut(self, i); + } + + fn visit_returning_mut(&mut self, i: &mut stmt::Returning) { + if let stmt::Returning::Model { include } = i { + // Start from the schema's pre-computed default returning — every + // Deferred fields, top-level or nested, are already `Null`. + // `process_top_level_includes` then splices loaded forms in for + // the fields named by include paths (and for every deferred field + // when this is an `INSERT ... RETURNING`). + let mut returning = self.mapping_unwrap().default_returning.clone(); + let mut include_paths = std::mem::take(include); + let is_insert = self.cx.is_insert(); + + self.prepare_model_returning_for_context(&mut returning, &mut include_paths, is_insert); + self.process_top_level_includes(returning.as_record_mut_unwrap(), &include_paths); + + *i = stmt::Returning::Project(returning); + } + + if self.model().is_some() + && let stmt::Returning::Project(value) = i + { + // Expand relation projections while they still refer to app fields. + // The normal returning pass replaces them with mapped expressions. + self.lower_returning().process_projected_returning(value); + } + + // For multi-row INSERT returning, visit each row with its row index so + // that sub-statements (e.g., child INSERTs for HasOne relations) capture + // the correct parent row index via scope_statement. + if matches!(&self.cx, LoweringContext::Insert(..)) + && let stmt::Returning::Expr(stmt::Expr::List(list)) = i + { + for (index, item) in list.items.iter_mut().enumerate() { + self.lower_returning_for_row(index).visit_expr_mut(item); + } + return; + } + + stmt::visit_mut::visit_returning_mut(&mut self.lower_returning(), i); + } + + fn visit_stmt_delete_mut(&mut self, stmt: &mut stmt::Delete) { + // Create a new expr scope for the statement, and lower all parts + // *except* the source field (since it is borrowed). + let mut lower = self.scope_expr(&stmt.from); + + // Before lowering, handle cascading deletes + lower.plan_stmt_delete_relations(stmt); + + lower.visit_filter_mut(&mut stmt.filter); + + if let Some(expr) = &mut stmt.condition.expr { + lower.visit_expr_mut(expr); + } + + if let Some(returning) = &mut stmt.returning { + lower.visit_returning_mut(returning); + } + + lower.apply_lowering_filter_constraint(&mut stmt.filter); + + self.visit_source_mut(&mut stmt.from); + } + + fn visit_stmt_insert_mut(&mut self, stmt: &mut stmt::Insert) { + // First, if an insertion scope is specified, lower the scope to be just "model" + self.apply_insert_scope(&mut stmt.target, &mut stmt.source); + + let sql = self.state.engine.capability.sql(); + if let Err(err) = upsert::normalize(stmt, !sql) { + self.state.errors.push(err); + return; + } + + self.state.insert_stmts.push(self.scope_stmt_id()); + + // Create a new expr scope for the statement, and lower all parts + // *except* the target field (since it is borrowed). + let mut lower = self.lower_insert(&stmt.target); + + if let Some(upsert) = &mut stmt.upsert { + if let stmt::UpsertTarget::Fields(fields) = &upsert.target { + let mapping = lower.mapping_unwrap(); + let mut columns = Vec::new(); + for field in fields { + let mapped = mapping + .resolve_field_mapping(field) + .expect("upsert target must map to database columns"); + columns.extend(mapped.columns().map(|(column, _)| column)); + } + upsert.target = stmt::UpsertTarget::Columns(columns); + } + if upsert.action == stmt::UpsertAction::Update { + let version = lower.inject_version_increment(&mut upsert.shared); + if !sql && let Some(projection) = version { + upsert.defaults.set(projection, stmt::Value::U64(0)); + } + } + lower.visit_assignments_mut(&mut upsert.shared); + lower.visit_assignments_mut(&mut upsert.defaults); + lower.visit_assignments_mut(&mut upsert.update_defaults); + } + + // Returning lowering rewrites `Model` to `Project`, so preserve the + // caller's original request for insert relation planning. + let returns_model = stmt + .returning + .as_ref() + .is_some_and(stmt::Returning::is_model); + if let Some(returning) = &mut stmt.returning { + lower.visit_returning_mut(returning); + } + + // Preprocess the insertion source (values usually) + // `DO NOTHING RETURNING` legitimately produces zero rows. Keep its + // returning clause as a projection over the database result so the + // normal single-statement boundary can turn an empty list into + // `Value::Null` (and therefore `Option::None`). Converting it to an + // expression would project row zero eagerly and panic on conflict. + let preserve_returning_projection = stmt + .upsert + .as_ref() + .is_some_and(|upsert| upsert.action == stmt::UpsertAction::Ignore); + lower.preprocess_insert_values( + &mut stmt.source, + &mut stmt.returning, + preserve_returning_projection, + returns_model, + ); + + // Lower the insertion source + lower.visit_stmt_query_mut(&mut stmt.source); + + if let Some(returning) = &mut stmt.returning { + lower.visit_returning_mut(returning); + if stmt.upsert.is_none() { + lower.constantize_insert_returning(returning, &stmt.source); + } + + if stmt.source.single + && let stmt::Returning::Expr(expr) = &returning + { + // Not strictly true, but there is nothing that needs to + // return a list at this point for a "single" query. If this + // is ever needed, remove the assertion. + debug_assert!(!expr.is_list()); + } + } + + self.visit_insert_target_mut(&mut stmt.target); + + let popped = self.state.insert_stmts.pop(); + debug_assert_eq!(popped, Some(self.scope_stmt_id())); + } + + fn visit_stmt_query_mut(&mut self, stmt: &mut stmt::Query) { + let mut lower = self.scope_expr(&stmt.body); + + if let Some(with) = &mut stmt.with { + lower.visit_with_mut(with); + } + + if let Some(order_by) = &mut stmt.order_by { + lower.visit_order_by_mut(order_by); + } + + if let Some(limit) = &mut stmt.limit { + lower.visit_limit_mut(limit); + } + + self.visit_expr_set_mut(&mut stmt.body); + + self.rewrite_offset_after_as_filter(stmt); + } + + fn visit_stmt_select_mut(&mut self, stmt: &mut stmt::Select) { + let mut lower = self.scope_expr(&stmt.source); + + lower.visit_filter_mut(&mut stmt.filter); + lower.visit_returning_mut(&mut stmt.returning); + lower.apply_lowering_filter_constraint(&mut stmt.filter); + + self.visit_source_mut(&mut stmt.source); + } + + fn visit_stmt_update_mut(&mut self, stmt: &mut stmt::Update) { + let mut lower = self.scope_expr(&stmt.target); + + let mut returning_changed = false; + + // Before lowering children, convert the "Changed" returning statement + // to an expression referencing changed fields. + if let Some(returning) = &mut stmt.returning + && returning.is_changed() + { + returning_changed = true; + + if let Some(model) = lower.model() { + let mapping = lower.mapping_unwrap(); + + // Step 1 — build a mask of all primitives being changed by + // OR-ing each assigned field's coverage mask together. + let mut changed_bits = stmt::PathFieldSet::new(); + for projection in stmt.assignments.keys() { + if let Some(mf) = mapping.resolve_field_mapping(projection) { + changed_bits |= mf.field_mask(); + } + } + + // Step 2 — build the returning expression. + *returning = stmt::Returning::Project(build_update_returning( + model.id, + None, + &mapping.fields, + &changed_bits, + )); + } + } + + // Plan relations + lower.plan_stmt_update_relations( + &mut stmt.assignments, + &stmt.filter, + &mut stmt.returning, + returning_changed, + ); + + // A versionable model bumps its OCC counter on every update. The + // instance path sets the counter explicitly in generated code, + // alongside the `version == ` condition — only it knows the + // value the caller last loaded. A query-based update leaves the counter + // unset, so the engine injects the relative `version = version + 1` + // here. Upsert conflict updates use the same injection helper from the + // insert lowering path. + // + // Injecting after the `Changed` returning projection is built above + // keeps the engine-managed counter out of the returning clause. A + // relative `+1` is not constantizable; leaving it in the returning + // would force a per-row read-back that the multi-row DynamoDB transact + // path cannot serve. The counter still advances in the database — a + // query update just doesn't read it back (it discards its result). + let _ = lower.inject_version_increment(&mut stmt.assignments); + + lower.visit_assignments_mut(&mut stmt.assignments); + lower.visit_filter_mut(&mut stmt.filter); + + if let Some(expr) = &mut stmt.condition.expr { + lower.visit_expr_mut(expr); + } + + if let Some(returning) = &mut stmt.returning { + let update_stmt_id = lower.scope_stmt_id(); + // Relation subqueries may replace references to this update with + // constants from assignments lowered to column indices. + lower + .state + .update_returning + .insert(update_stmt_id, std::mem::take(&mut stmt.assignments)); + if returning_changed { + lower.process_update_embedded_relations(returning); + } + lower.visit_returning_mut(returning); + // Restore assignments after relation loads finish lowering. + stmt.assignments = lower + .state + .update_returning + .swap_remove(&update_stmt_id) + .expect("update assignments registered while lowering returning"); + // Use the lowered assignments (which are now column-indexed) + returning::constantize_update_returning(lower.expr_cx, returning, &stmt.assignments); + } + + self.visit_update_target_mut(&mut stmt.target); + } + + fn visit_source_mut(&mut self, stmt: &mut stmt::Source) { + if let stmt::Source::Model(source_model) = stmt { + debug_assert!(source_model.via.is_none(), "TODO"); + + let table_id = self.schema().table_id_for(source_model.id); + *stmt = stmt::Source::table(table_id); + } + } + + fn visit_values_mut(&mut self, stmt: &mut stmt::Values) { + if self.cx.is_insert() + && let Some(mapping) = self.mapping() + { + for row in &mut stmt.rows { + let mut lowered = mapping.model_to_table.clone(); + self.lower_insert_row(row) + .visit_expr_record_mut(&mut lowered); + + *row = lowered.into(); + } + + return; + } + + visit_mut::visit_values_mut(self, stmt); + } +} + +impl<'a, 'b> LowerStatement<'a, 'b> { + /// App-level operand rewrite for `eq` and `ne` binary ops. + /// + /// `Reference::Model { nesting }` becomes a reference to the model's + /// primary-key field; a `BelongsTo` field reference becomes a reference + /// to the relation's foreign-key field; a path to a `belongs_to` inside + /// an embedded type becomes the projection of its key field(s). All + /// rewrites only match on app-level shapes; after the surrounding + /// lowering walk converts the references to columns, this method has + /// nothing to rewrite. + /// + /// Must fire before the operand's children are visited, since lowering + /// otherwise replaces the field reference with a column reference and + /// the rewrite has no app-level shape to match on. + fn rewrite_eq_operand(&self, operand: &mut stmt::Expr) { + let stmt::Expr::Reference(expr_reference) = operand else { + self.rewrite_embedded_relation_operand(operand); + return; + }; + + match &*expr_reference { + stmt::ExprReference::Model { nesting } => { + let nesting = *nesting; + let model = self + .expr_cx + .resolve_expr_reference(expr_reference) + .as_model_unwrap(); + + *operand = key_field_refs(nesting, model.primary_key.fields.iter().copied()); + } + stmt::ExprReference::Field { nesting, .. } => { + let nesting = *nesting; + let field = self + .expr_cx + .resolve_expr_reference(expr_reference) + .as_field_unwrap(); + + match &field.ty { + app::FieldTy::Primitive(_) | app::FieldTy::Embedded(_) => {} + app::FieldTy::Has(_) | app::FieldTy::Via(_) => todo!(), + app::FieldTy::BelongsTo(rel) => { + *operand = key_field_refs( + nesting, + rel.foreign_key.fields.iter().map(|fk| fk.source), + ); + } + } + } + _ => {} + } + } + + /// Substitute the projection of the relation's key field(s) for a path + /// ending at a `belongs_to` inside an embedded type, in place. The + /// analogue of `rewrite_eq_operand`'s `BelongsTo` arm for relations + /// inside embedded types; a path continuing into the relation's target + /// is a predicate on the target, lifted by `LiftInSubquery` before + /// lowering. The key expression keeps the path's variant selections, so + /// the `is_variant` guards the typed layer fixed next to the comparison + /// still scope it (see `relation_expr::resolve`). + /// + /// Returns whether the operand was substituted. + fn rewrite_embedded_relation_operand(&self, operand: &mut stmt::Expr) -> bool { + if let Some(relation) = relation_expr::resolve(&self.expr_cx, operand) + && relation.is_endpoint() + && let Some(key_expr) = relation.key_expr() + { + *operand = key_expr; + true + } else { + false + } + } + + /// App-level rewrite for the LHS of an `IN`-list expression: + /// `Reference::Model { nesting } IN list` becomes ` IN list`, + /// and ` IN list` becomes + /// ` IN list`. The list holds model values, which the + /// typed layer already reduced to their keys. + /// + /// Must fire before the LHS is walked, since walking lowers the model + /// reference into a column reference and the rewrite has nothing to + /// match on. + fn rewrite_in_list_model_operand(&self, expr: &mut stmt::ExprInList) { + if self.rewrite_embedded_relation_operand(&mut expr.expr) { + return; + } + + let (nesting, pk_field_id) = { + let stmt::Expr::Reference(expr_ref @ stmt::ExprReference::Model { nesting }) = + &*expr.expr + else { + return; + }; + let nesting = *nesting; + let model = self + .expr_cx + .resolve_expr_reference(expr_ref) + .as_model_unwrap(); + let [pk_field_id] = &model.primary_key.fields[..] else { + todo!() + }; + (nesting, *pk_field_id) + }; + + let schema = self.expr_cx.schema(); + let pk = schema.app.field(pk_field_id); + + // Sanity-check the RHS shape against the PK type. + match &mut *expr.list { + stmt::Expr::List(expr_list) => { + for item in &mut expr_list.items { + match item { + stmt::Expr::Value(value) => { + assert!(value.is_a(&schema.app, &pk.ty.as_primitive_unwrap().ty)); + } + _ => todo!("{item:#?}"), + } + } + } + stmt::Expr::Value(stmt::Value::List(values)) => { + for value in values { + assert!(value.is_a(&schema.app, &pk.ty.as_primitive_unwrap().ty)); + } + } + _ => todo!("expr={expr:#?}"), + } + + *expr.expr = stmt::Expr::ref_field(nesting, pk.id()); + } + + fn lower_expr_binary_op( + &mut self, + op: stmt::BinaryOp, + lhs: &mut stmt::Expr, + rhs: &mut stmt::Expr, + ) -> Option { + match (&mut *lhs, &mut *rhs) { + (stmt::Expr::Value(value), other) | (other, stmt::Expr::Value(value)) + if value.is_null() => + { + let other = other.take(); + + Some(match op { + stmt::BinaryOp::Eq => stmt::Expr::is_null(other), + stmt::BinaryOp::Ne => stmt::Expr::is_not_null(other), + _ => todo!(), + }) + } + // Record-vs-record decomposition for eq/ne. Embedded fields lower + // to record expressions, so a comparison like + // `Record([cast(col, T)]) == Record([val])` only exposes the + // cast (and the cast-stripping rule below) once the record is + // split per-element. Recurse into each pair so cast handling + // fires. + // + // Mirrors the two record shapes handled in `simplify_expr_binary_op`: + // both sides `Expr::Record`, or one side folded to + // `Expr::Value(Value::Record)`. + (stmt::Expr::Record(lhs_rec), stmt::Expr::Record(rhs_rec)) + if (op.is_eq() || op.is_ne()) && lhs_rec.len() == rhs_rec.len() => + { + Some(self.combine_record_op( + op, + std::mem::take(&mut lhs_rec.fields), + std::mem::take(&mut rhs_rec.fields), + )) + } + (stmt::Expr::Record(rec), stmt::Expr::Value(stmt::Value::Record(val_rec))) + | (stmt::Expr::Value(stmt::Value::Record(val_rec)), stmt::Expr::Record(rec)) + if (op.is_eq() || op.is_ne()) && rec.len() == val_rec.len() => + { + let val_exprs = std::mem::take(&mut val_rec.fields) + .into_iter() + .map(stmt::Expr::Value) + .collect(); + Some(self.combine_record_op(op, std::mem::take(&mut rec.fields), val_exprs)) + } + (stmt::Expr::Cast(expr_cast), other) | (other, stmt::Expr::Cast(expr_cast)) => { + // Expand decodes of embedded values into guarded comparisons so + // each cast operand can use the database's stored type. + if expr_is_enum_decode(other) { + let comparison = stmt::Expr::binary_op(lhs.clone(), op, rhs.clone()); + let mut expanded_comparison = comparison.clone(); + Simplify::with_context(self.expr_cx, self.capability()) + .visit_expr_mut(&mut expanded_comparison); + // Simplification also expands optional struct decodes. + // Revisit only after it makes progress to prevent recursion. + if expanded_comparison != comparison { + self.visit_expr_mut(&mut expanded_comparison); + return Some(expanded_comparison); + } + return None; + } + + let target_ty = self.capability().native_type_for(&expr_cast.ty); + self.cast_expr(lhs, &target_ty); + self.cast_expr(rhs, &target_ty); + None + } + _ => None, + } + } + + /// Combines per-element comparisons of two record-shaped operands into a + /// single boolean expression: AND for `eq`, OR for `ne`. Each pair is + /// recursed through `lower_expr_binary_op` so any inner cast handling + /// fires as if the comparison had been written elementwise. + fn combine_record_op( + &mut self, + op: stmt::BinaryOp, + lhs_fields: Vec, + rhs_fields: Vec, + ) -> stmt::Expr { + let comparisons: Vec<_> = lhs_fields + .into_iter() + .zip(rhs_fields) + .map(|(mut l, mut r)| { + self.lower_expr_binary_op(op, &mut l, &mut r) + .unwrap_or_else(|| stmt::Expr::binary_op(l, op, r)) + }) + .collect(); + + if op.is_eq() { + stmt::Expr::and_from_vec(comparisons) + } else { + stmt::Expr::or_from_vec(comparisons) + } + } + + /// Lower the LHS and RHS of an `IN (subquery)` in place. When both sides + /// are records of equal arity (as produced for composite-key FK + /// comparisons) the per-element lowering is applied pair-wise so + /// element-level transforms (NULLs, casts) still fire while the + /// surrounding `IN` is preserved. Otherwise a single binary-op lowering + /// is applied across both sides. + fn lower_in_subquery_operands(&mut self, lhs: &mut stmt::Expr, rhs: &mut stmt::Expr) { + if let (stmt::Expr::Record(lhs_rec), stmt::Expr::Record(rhs_rec)) = (&mut *lhs, &mut *rhs) + && lhs_rec.len() == rhs_rec.len() + { + for (l, r) in lhs_rec.fields.iter_mut().zip(rhs_rec.fields.iter_mut()) { + let maybe_res = self.lower_expr_binary_op(stmt::BinaryOp::Eq, l, r); + assert!(maybe_res.is_none(), "TODO"); + } + } else { + let maybe_res = self.lower_expr_binary_op(stmt::BinaryOp::Eq, lhs, rhs); + assert!(maybe_res.is_none(), "TODO"); + } + } + + fn lower_expr_in_list( + &mut self, + expr: &mut stmt::Expr, + list: &mut stmt::Expr, + ) -> Option { + if in_list_requires_equality_expansion(expr) + && let Some(items) = take_in_list_items(list) + { + return Some(self.expand_in_list_equalities(expr, items)); + } + + if let stmt::Expr::Map(map) = list { + self.lower_mapped_in_list(expr, map); + return None; + } + + if matches!(expr, stmt::Expr::Cast(_)) { + self.lower_cast_in_list(expr, list); + return None; + } + + assert_lowered_in_list(expr, list); + None + } + + fn expand_in_list_equalities( + &mut self, + expr: &stmt::Expr, + items: Vec, + ) -> stmt::Expr { + let equalities = items.into_iter().map(|mut item| { + let mut expr = expr.clone(); + self.lower_expr_binary_op(stmt::BinaryOp::Eq, &mut expr, &mut item) + .unwrap_or_else(|| stmt::Expr::eq(expr, item)) + }); + + stmt::Expr::or_from_vec(equalities.collect()) + } + + fn lower_mapped_in_list(&mut self, expr: &mut stmt::Expr, map: &mut stmt::ExprMap) { + assert!(map.base.is_arg(), "TODO"); + + let lowered = self.lower_expr_binary_op(stmt::BinaryOp::Eq, expr, &mut map.map); + assert!(lowered.is_none(), "TODO"); + } + + fn lower_cast_in_list(&mut self, expr: &mut stmt::Expr, list: &mut stmt::Expr) { + let stmt::Expr::Cast(cast) = expr else { + unreachable!() + }; + let target_ty = self.capability().native_type_for(&cast.ty); + + self.cast_expr(expr, &target_ty); + + match list { + stmt::Expr::List(list) => { + for item in &mut list.items { + self.cast_expr(item, &target_ty); + } + } + stmt::Expr::Value(stmt::Value::List(items)) => { + for item in items { + *item = target_ty + .cast(self.expr_cx.schema(), item.take()) + .expect("failed to cast value"); + } + } + stmt::Expr::Arg(_) => { + let arg = list.take(); + let cast = stmt::Expr::cast(stmt::Expr::arg(0), target_ty); + *list = stmt::Expr::map(arg, cast); + } + _ => todo!("expr={expr:#?}; list={list:#?}"), + } + } + + fn apply_lowering_filter_constraint(&self, _filter: &mut stmt::Filter) {} + + fn lower_expr_field(&self, nesting: usize, index: usize) -> stmt::Expr { + match self.cx { + LoweringContext::Statement + | LoweringContext::Returning(_) + | LoweringContext::Insert(..) => { + // Upsert update assignments are visited in the surrounding + // Insert context. Their field references read the stored row; + // proposed-row values use Expr::Incoming instead. Inserted + // value rows use the separate InsertRow branch below. + let mapping = self.mapping_at_unwrap(nesting); + mapping.table_to_model.lower_expr_reference(nesting, index) + } + LoweringContext::InsertRow(row) => { + // If nesting > 0, this references a parent scope, not the current row + if nesting > 0 { + // Use Statement context to properly handle cross-statement references + let mapping = self.mapping_at_unwrap(nesting); + mapping.table_to_model.lower_expr_reference(nesting, index) + } else { + row.entry(index).unwrap().to_expr() + } + } + } + } + + /// Returns the ArgId for the new reference + fn new_ref( + &mut self, + source_id: hir::StmtId, + target_id: hir::StmtId, + mut expr_reference: stmt::ExprReference, + ) -> usize { + let stmt::ExprReference::Column(expr_column) = &mut expr_reference else { + todo!() + }; + + // First, get the nesting so we can resolve the target statmeent + let nesting = expr_column.nesting; + + // We only track references that point to statements being executed by + // separate operations. References within the same operation are handled + // by the target database. + debug_assert!(nesting != 0, "expr_reference={expr_reference:#?}"); + + // Set the nesting to zero as the stored ExprReference will be used from + // the context of the *target* statement. + expr_column.nesting = 0; + + let target = &mut self.state.hir[target_id]; + + // The `batch_load_index` is the index for this reference in the row + // returned from the target statement's ExecStatement operation. This + // ExecStatement operation batch loads all records needed to execute + // the full root statement. + target + .back_refs + .entry(source_id) + .or_default() + .exprs + .insert_full(expr_reference); + + // Create an argument for inputing the expr reference's value into the statement. + let source = &mut self.state.hir[source_id]; + + // See if an arg already exists + for (i, arg) in source.args.iter().enumerate() { + let hir::Arg::Ref { + target_expr_ref, .. + } = arg + else { + continue; + }; + + if *target_expr_ref == expr_reference { + return i; + } + } + + let arg = source.args.len(); + + source.args.push(hir::Arg::Ref { + target_expr_ref: expr_reference, + stmt_id: target_id, + nesting, + data_load_input: Cell::new(None), + returning_input: Cell::new(None), + batch_load_index: if let Some(row_index) = self.state.scopes[self.scope_id].row_index { + debug_assert_eq!(1, nesting, "TODO"); + Cell::new(Some(row_index)) + } else { + Cell::new(None) + }, + }); + + arg + } + + fn new_statement_info(&mut self) -> hir::StmtId { + // Ambient dependencies and the ones inherited from the enclosing + // statement are completion edges; `Effect` edges are never inherited. + let mut deps: IndexMap = self + .state + .dependencies + .iter() + .map(|&stmt_id| (stmt_id, hir::DepKind::Statement)) + .collect(); + + for (&stmt_id, &kind) in &self.curr_stmt_info().deps { + if kind == hir::DepKind::Statement { + deps.insert(stmt_id, kind); + } + } + + self.state.hir.new_statement_info(deps) + } + + /// Create a new sub-statement. Returns the argument position + fn new_sub_statement( + &mut self, + source_id: hir::StmtId, + target_id: hir::StmtId, + stmt: Box, + ) -> stmt::Expr { + self.state.hir[target_id].stmt = Some(stmt); + self.new_dependency_arg(source_id, target_id) + } + + /// Create a new argument on a dependent statement + fn new_dependency_arg(&mut self, source_id: hir::StmtId, target_id: hir::StmtId) -> stmt::Expr { + let source = &mut self.state.hir[source_id]; + let arg = source.args.len(); + source.args.push(hir::Arg::Sub { + stmt_id: target_id, + returning: self.cx.is_returning(), + input: Cell::new(None), + batch_load_index: Cell::new(None), + }); + + stmt::Expr::arg(arg) + } + + /// Run the canonical pipeline (pre-lower simplify, lowering walk, post-lower + /// simplify) on a synthesized sub-statement and stitch it onto the parent + /// as an `Expr::Arg`. + /// + /// Used at sites where lowering itself synthesizes a new statement to embed + /// in the parent's `Returning` (include subqueries, child inserts for + /// relation planning). Equivalent to a recursive `lower_stmt` call that + /// passes through the `Expr::Stmt` arm in `visit_expr_mut`, but expressed + /// directly so the synthesized statement does not have to round-trip + /// through an `Expr::Stmt` placeholder. + fn lower_sub_stmt(&mut self, stmt: stmt::Statement) -> stmt::Expr { + let source_id = self.scope_stmt_id(); + let mut stmt = Box::new(stmt); + + let target_id = self.scope_statement(|child| { + // Via-association rewrite: `Source::Model { via }` becomes an + // explicit WHERE filter so the lowering walk only sees rewritten + // sources. The IN-subquery lift fires next so non-walk code + // paths see the lifted form. The eq/ne operand rewrite + // (model→PK, BelongsTo→FK) fires inside the lowering walk via + // `LowerStatement::visit_expr_binary_op_mut`. + association::RewriteVia::new(child.expr_cx).rewrite(&mut stmt); + lift_in_subquery::LiftInSubquery::new( + child.expr_cx, + child.state.engine.capability.sql(), + ) + .rewrite(&mut stmt); + // Combine compatible IN subqueries before the lowering walk + // extracts them into separate NoSQL statements. + Simplify::with_context(child.expr_cx, child.state.engine.capability) + .visit_mut(&mut *stmt); + // Lowering walk. + child.visit_stmt_mut(&mut stmt); + // Post-lower simplify: heavyweight rules on the lowered tree. + child.state.engine.simplify_stmt(&mut *stmt); + }); + + // Sub-statements built via this helper always live in the parent's + // Returning clause (include subqueries, child inserts for relation + // planning). Force the new `Arg::Sub`'s `returning` flag accordingly: + // `plan_nested_merge` keys off `returning: true` to discover include + // subqueries, and the line-399 `Expr::Stmt` arm gets the same flag + // because it fires during the parent's Returning walk. + let saved_cx = std::mem::replace(&mut self.cx, LoweringContext::Returning(None)); + let arg = self.new_sub_statement(source_id, target_id, stmt); + self.cx = saved_cx; + + if self.state.hir[target_id].independent { + self.curr_stmt_info() + .add_dep(target_id, hir::DepKind::Statement); + } + + arg + } + + fn schema(&self) -> &'b Schema { + &self.state.engine.schema + } + + fn capability(&self) -> &Capability { + self.state.engine.capability() + } + + /// Both flags must be true to rewrite `IN (...)` into `= ANY()`: + /// the dialect must accept the predicate, and the bind layer must accept + /// a single array-valued parameter. + fn supports_any_rewrite(&self) -> bool { + let cap = self.capability(); + cap.bind_list_param && cap.predicate_match_any + } + + fn field(&self, id: impl Into) -> &'b app::Field { + self.schema().app.field(id.into()) + } + + fn model(&self) -> Option<&'a ModelRoot> { + self.expr_cx.target().as_model() + } + + #[track_caller] + fn model_unwrap(&self) -> &'a ModelRoot { + self.expr_cx.target().as_model_unwrap() + } + + fn mapping(&self) -> Option<&'b mapping::Model> { + self.model() + .map(|model| self.state.engine.schema.mapping_for(model)) + } + + #[track_caller] + fn mapping_unwrap(&self) -> &'b mapping::Model { + self.state.engine.schema.mapping_for(self.model_unwrap()) + } + + #[track_caller] + fn mapping_at_unwrap(&self, nesting: usize) -> &'b mapping::Model { + let model = self.expr_cx.target_at(nesting).as_model_unwrap(); + self.state.engine.schema.mapping_for(model) + } + + fn curr_stmt_info(&mut self) -> &mut hir::StatementInfo { + let stmt_id = self.scope_stmt_id(); + &mut self.state.hir[stmt_id] + } + + /// Returns the `StmtId` for the Statement at the **current** scope. + fn scope_stmt_id(&self) -> hir::StmtId { + self.state.scopes[self.scope_id].stmt_id + } + + /// Get the StmtId for the specified nesting level + fn resolve_stmt_id(&self, nesting: usize) -> hir::StmtId { + debug_assert!( + self.scope_id >= nesting, + "invalid nesting; nesting={nesting:#?}; scopes={:#?}", + self.state.scopes + ); + self.state.scopes[self.scope_id - nesting].stmt_id + } + + /// Plan a sub-statement that is able to reference the parent statement + fn scope_statement(&mut self, f: impl FnOnce(&mut LowerStatement<'_, '_>)) -> hir::StmtId { + let stmt_id = self.new_statement_info(); + let row_index = match &self.cx { + LoweringContext::Insert(_, row_index) => *row_index, + LoweringContext::Returning(row_index) => *row_index, + _ => None, + }; + let scope_id = self.state.scopes.push(Scope { stmt_id, row_index }); + let mut dependencies = None; + + let mut lower = LowerStatement { + state: self.state, + expr_cx: self.expr_cx, + scope_id, + cx: LoweringContext::Statement, + collect_dependencies: &mut dependencies, + }; + + f(&mut lower); + + debug_assert!(dependencies.is_none()); + self.state.scopes.pop(); + stmt_id + } + + fn scope_expr<'child>( + &'child mut self, + target: impl IntoExprTarget<'child>, + ) -> LowerStatement<'child, 'b> { + LowerStatement { + state: self.state, + expr_cx: self.expr_cx.scope(target), + scope_id: self.scope_id, + cx: self.cx, + collect_dependencies: self.collect_dependencies, + } + } + + fn lower_insert<'child>( + &'child mut self, + target: &'child stmt::InsertTarget, + ) -> LowerStatement<'child, 'b> { + let columns = match target { + stmt::InsertTarget::Scope(_) => { + panic!("InsertTarget::Scope should already have been lowered by this point") + } + stmt::InsertTarget::Model(model_id) => &self.schema().mapping_for(model_id).columns, + stmt::InsertTarget::Table(insert_table) => &insert_table.columns, + }; + + LowerStatement { + state: self.state, + expr_cx: self.expr_cx.scope(target), + scope_id: self.scope_id, + cx: LoweringContext::Insert(columns, None), + collect_dependencies: self.collect_dependencies, + } + } + + fn lower_insert_with_row(&mut self, row: usize, f: impl FnOnce(&mut Self)) { + let LoweringContext::Insert(_, maybe_row) = &mut self.cx else { + todo!() + }; + debug_assert!(maybe_row.is_none()); + *maybe_row = Some(row); + f(self); + + let LoweringContext::Insert(_, maybe_row) = &mut self.cx else { + todo!() + }; + debug_assert_eq!(Some(row), *maybe_row); + *maybe_row = None; + } + + fn lower_insert_row<'child>( + &'child mut self, + row: &'child stmt::Expr, + ) -> LowerStatement<'child, 'b> { + LowerStatement { + state: self.state, + expr_cx: self.expr_cx, + scope_id: self.scope_id, + cx: LoweringContext::InsertRow(row), + collect_dependencies: self.collect_dependencies, + } + } + + fn lower_returning(&mut self) -> LowerStatement<'_, 'b> { + LowerStatement { + state: self.state, + expr_cx: self.expr_cx, + scope_id: self.scope_id, + cx: LoweringContext::Returning(None), + collect_dependencies: self.collect_dependencies, + } + } + + fn lower_returning_for_row<'child>( + &'child mut self, + row_index: usize, + ) -> LowerStatement<'child, 'b> { + LowerStatement { + state: self.state, + expr_cx: self.expr_cx, + scope_id: self.scope_id, + cx: LoweringContext::Returning(Some(row_index)), + collect_dependencies: self.collect_dependencies, + } + } + + fn cast_expr(&mut self, expr: &mut stmt::Expr, target_ty: &stmt::Type) { + assert!(!target_ty.is_list(), "TODO"); + match expr { + stmt::Expr::Cast(expr_cast) => { + // TODO: verify that this is actually a correct cast. + // Remove the cast - the inner expression is already the right type + *expr = expr_cast.expr.take(); + } + stmt::Expr::Value(value) => { + // Cast the value to target_ty using existing cast method + let casted = target_ty + .cast(self.expr_cx.schema(), value.take()) + .expect("failed to cast value"); + *value = casted; + } + stmt::Expr::Project(expr_project) => { + // A projection into an embedded field's lowered record — + // e.g. the key of a relation stored in an embedded type, as + // the LHS of a lifted FK IN-subquery. Collapse the + // projection and recurse; the collapsed entry is typically + // the column's decode cast, which the Cast arm strips. + let Some(entry) = expr_project.base.entry(&expr_project.projection) else { + todo!("cast_expr: cannot collapse projection: {expr_project:#?}") + }; + *expr = entry.to_expr(); + self.cast_expr(expr, target_ty); + } + stmt::Expr::Arg(_) => { + // Create a cast expression for the arg + let base = expr.take(); + *expr = stmt::Expr::cast(base, target_ty.clone()); + } + _ => todo!("cast_expr: cannot cast {expr:#?} to {target_ty:?}"), + } + } +} + +impl LoweringContext<'_> { + fn is_insert(&self) -> bool { + matches!(self, LoweringContext::Insert { .. }) + } + + fn is_insert_without_row(&self) -> bool { + matches!(self, Self::Insert(_, None)) + } + + fn is_insert_with_row(&self) -> bool { + matches!(self, Self::Insert(_, Some(_))) + } + + fn is_returning(&self) -> bool { + matches!(self, LoweringContext::Returning(_)) + } + + fn is_statement(&self) -> bool { + matches!(self, LoweringContext::Statement) + } +} + +/// Build a scalar-or-record expression of field references for a key +/// (primary or foreign): a single `ref_field` for a single-column key, a +/// `Record` of `ref_field`s for a composite key. Used both for the +/// eq-operand rewrite (where the surrounding `==` decomposes pair-wise via +/// `lower_expr_binary_op`'s `Record == Record` handler) and for composite +/// FK IN-subquery comparisons (where the tuple LHS pairs with a tuple +/// projection on the RHS). +/// Whether an expression is an embedded-enum decode — a `Match` over the +/// discriminant, possibly under a projection selecting a record slot. +fn expr_is_enum_decode(expr: &stmt::Expr) -> bool { + match expr { + stmt::Expr::Match(_) => true, + stmt::Expr::Project(project) => expr_is_enum_decode(&project.base), + _ => false, + } +} + +/// Collapse projections into a lowered record — an embed lowers to +/// `Record([..])` of its fields' expressions — so the expression at the +/// projected position is exposed. Stops at a projection that cannot be +/// collapsed (a base that is not a record). +fn collapse_projections(mut expr: stmt::Expr) -> stmt::Expr { + while let stmt::Expr::Project(project) = &expr { + let Some(entry) = project.base.entry(&project.projection) else { + break; + }; + expr = entry.to_expr(); + } + expr +} + +/// The payload of a lowered enum value as the variant with discriminant +/// `disc_value`: the variant's field expressions, without the discriminant. +/// +/// A lowered enum is its decode `Match` — possibly under projections into +/// the lowered record of an enclosing embed, which collapse to expose it — +/// or, for a value row, the encoded record itself. Slot 0 of an arm's record +/// (and of a value record) is the discriminant; a unit variant's arm is the +/// bare discriminant and has an empty payload. +fn variant_payload(lowered: stmt::Expr, disc_value: &stmt::Value) -> stmt::Expr { + match collapse_projections(lowered) { + stmt::Expr::Match(match_expr) => { + let arm = match_expr + .arms + .into_iter() + .find(|arm| arm.pattern == *disc_value) + .expect("enum decode has an arm per variant"); + + match arm.expr { + stmt::Expr::Record(record) => stmt::Expr::record(record.fields.into_iter().skip(1)), + _ => stmt::Expr::record(Vec::::new()), + } + } + stmt::Expr::Value(stmt::Value::Record(record)) + if record.fields.first() == Some(disc_value) => + { + stmt::Expr::record(record.fields.into_iter().skip(1).map(stmt::Expr::Value)) + } + stmt::Expr::Value(_) => stmt::Expr::error("value is not the selected variant"), + lowered => todo!("variant selection on lowered expression {lowered:#?}"), + } +} + +pub(super) fn key_field_refs( + nesting: usize, + fields: impl ExactSizeIterator, +) -> stmt::Expr { + scalar_or_record(fields.map(|field| stmt::Expr::ref_field(nesting, field))) +} + +fn scalar_or_record(mut fields: impl ExactSizeIterator) -> stmt::Expr { + if fields.len() == 1 { + fields.next().unwrap() + } else { + stmt::Expr::record(fields) + } +} + +/// Input implementation for assignment substitution. +/// +/// Provides assignment values when substituting field references in `model_to_table` +/// expressions. Handles projections automatically for embedded fields. +struct AssignmentInput<'a> { + assignment_projection: stmt::Projection, + value: &'a stmt::Expr, +} + +impl stmt::Input for AssignmentInput<'_> { + fn resolve_ref( + &mut self, + expr_reference: &stmt::ExprReference, + expr_projection: &stmt::Projection, + ) -> Option { + let stmt::ExprReference::Field { nesting: 0, index } = expr_reference else { + return None; + }; + + let assignment_steps = self.assignment_projection.as_slice(); + + if *index != assignment_steps[0] { + return None; + } + + let remaining_steps = &assignment_steps[1..]; + + // The assignment value sits at `remaining_steps` below the field; + // a template projection outside that subtree refers to a sibling + // this assignment does not cover. + let inner_steps = expr_projection.as_slice().strip_prefix(remaining_steps)?; + + if inner_steps.is_empty() { + Some(self.value.clone()) + } else { + // The column's encode template projects into variant-field + // positions of the assignment value. When a mixed enum is updated + // to a unit (or otherwise shorter) variant, the value record is + // narrower than a sibling variant's column expects, so the + // projection falls out of bounds and `entry` returns `None`. Those + // references live inside a discriminant-guarded match arm that is + // unreachable for this value, so resolving them to `null` is safe — + // the simplifier drops the dead arm. + Some( + self.value + .entry(&stmt::Projection::from(inner_steps)) + .map(|e| e.to_expr()) + .unwrap_or_else(stmt::Expr::null), + ) + } + } +} + +/// Builds the returning expression for a `Returning::Changed` update. +/// +/// Iterates `mapping_fields` using `changed_bits` to determine what to include: +/// +/// - Relation fields → null placeholder (populated during relation planning) +/// - Field whose `field_mask` is fully covered by `changed_bits` → emit +/// `project(ref_self_field, sub_projection)` (constantized later; project +/// omitted when `sub_projection` is identity) +/// - Field only partially covered → recurse to build a nested `SparseRecord` +/// +/// `root_field_id` controls how the self-field reference is constructed: +/// - `None` at the top level: derived per-iteration as `{ model_id, i }`, since +/// each model field is its own root. +/// - `Some(id)` when recursing into an embedded type: fixed throughout, because +/// all sub-field expressions project from the same top-level ancestor field. +fn build_update_returning( + model_id: app::ModelId, + root_field_id: Option, + mapping_fields: &[mapping::Field], + changed_bits: &stmt::PathFieldSet, +) -> stmt::Expr { + let mut exprs = vec![]; + let mut field_set = stmt::PathFieldSet::new(); + + for (i, mf) in mapping_fields.iter().enumerate() { + let intersection = changed_bits.clone() & mf.field_mask(); + + if intersection.is_empty() { + continue; + } + + field_set.insert(i); + + if mf.is_relation() { + // Relation field: null placeholder for the relation planner to fill + // in via set_returning_field. + exprs.push(stmt::Expr::null()); + } else { + let root_field_id = root_field_id.unwrap_or(app::FieldId { + model: model_id, + index: i, + }); + + if intersection == mf.field_mask() { + // Full coverage: all primitives in this field are being updated. + // Emit a projected field reference; the lowering + constantize + // pipeline will substitute the assignment value. + let base = stmt::Expr::ref_self_field(root_field_id); + let expr = if mf.sub_projection().is_identity() { + base + } else { + stmt::Expr::project(base, mf.sub_projection().clone()) + }; + exprs.push(expr); + } else { + // Partial embedded update: only some sub-fields are changing. + // Recurse to build a nested SparseRecord. The lowering pipeline + // resolves each reference to the correct column, and the simplifier + // folds project(record([...]), [i]) → column_ref. + let emb_mapping = mf.as_struct().unwrap(); + exprs.push(build_update_returning( + model_id, + Some(root_field_id), + &emb_mapping.fields, + &intersection, + )); + } + } + } + + stmt::Expr::cast( + stmt::ExprRecord::from_vec(exprs), + stmt::Type::SparseRecord(field_set), + ) +} + +/// Returns whether `IN` must use the same per-value lowering as equality. +fn in_list_requires_equality_expansion(expr: &stmt::Expr) -> bool { + match expr { + stmt::Expr::Match(_) => true, + stmt::Expr::Record(record) => record.fields.iter().any(|field| !field.is_column()), + _ => false, + } +} + +/// Takes the expressions from a literal `IN` list. +fn take_in_list_items(list: &mut stmt::Expr) -> Option> { + match list { + stmt::Expr::List(list) => Some(std::mem::take(&mut list.items)), + stmt::Expr::Value(stmt::Value::List(items)) => Some( + std::mem::take(items) + .into_iter() + .map(stmt::Expr::Value) + .collect(), + ), + _ => None, + } +} + +/// Asserts that both operands have reached a driver-level `IN` form. +fn assert_lowered_in_list(expr: &stmt::Expr, list: &stmt::Expr) { + match expr { + stmt::Expr::Record(record) => { + assert!(record.fields.iter().all(stmt::Expr::is_column)); + } + stmt::Expr::Reference(reference) => assert!(reference.is_column()), + stmt::Expr::Project(_) => {} + _ => todo!("expr={expr:#?}; list={list:#?}"), + } + + match list { + stmt::Expr::Value(stmt::Value::List(_)) => {} + stmt::Expr::List(list) => { + assert!(list.items.iter().all(stmt::Expr::is_value)); + } + _ => panic!("invalid; should have been caught earlier"), + } +} + +/// True when an `IN` list is a candidate for the `= ANY($1)` rewrite: +/// +/// - The lhs is scalar (not a `Record`). Composite-key `IN` would need a PG +/// row-array bind which the current driver doesn't support. +/// - The list is a constant collection of scalar values. Lists containing +/// references, sub-statements, or record values stay as expanded `IN`. +fn in_list_is_value_list(e: &stmt::ExprInList) -> bool { + if matches!(*e.expr, stmt::Expr::Record(_)) { + return false; + } + let scalar = |v: &stmt::Value| !matches!(v, stmt::Value::Record(_) | stmt::Value::List(_)); + match &*e.list { + stmt::Expr::Value(stmt::Value::List(items)) => items.iter().all(scalar), + stmt::Expr::List(list) => list + .items + .iter() + .all(|i| matches!(i, stmt::Expr::Value(v) if scalar(v))), + _ => false, + } +} diff --git a/packages/toasty/src/engine/lower/association.rs b/packages/toasty/src/engine/lower/association.rs new file mode 100644 index 00000000000..a9f51b1aea2 --- /dev/null +++ b/packages/toasty/src/engine/lower/association.rs @@ -0,0 +1,238 @@ +use toasty_core::{ + schema::app, + stmt::{self, ExprContext, IntoExprTarget, VisitMut}, +}; + +/// Pre-lowering pass that rewrites `Source::Model { via: Some(_) }` into +/// an explicit WHERE filter on the surrounding statement. +/// +/// `via` associations are an app-level construct. They appear when a +/// query is built from a relation traversal, e.g. +/// `user.todos().delete(...)`, and the lowering walk converts +/// `Source::Model` into `Source::Table` once the association has been +/// rewritten as a filter. This pass runs before the lowering walk so +/// that the walk only sees the rewritten form. +pub(super) struct RewriteVia<'a> { + cx: ExprContext<'a>, +} + +impl<'a> RewriteVia<'a> { + pub(super) fn new(cx: ExprContext<'a>) -> Self { + Self { cx } + } + + /// Walk a statement and apply the via-association rewrite to every + /// Delete, Insert, and Query node it contains. + pub(super) fn rewrite(&mut self, stmt: &mut stmt::Statement) { + self.visit_mut(stmt); + } + + fn schema(&self) -> &'a toasty_core::Schema { + self.cx.schema() + } + + fn scope<'scope>(&'scope self, target: impl IntoExprTarget<'scope>) -> RewriteVia<'scope> { + RewriteVia { + cx: self.cx.scope(target), + } + } + + pub(super) fn rewrite_via_for_delete(&mut self, stmt: &mut stmt::Delete) { + if let stmt::Source::Model(model) = &mut stmt.from + && let Some(via) = model.via.take() + { + // Create a new scope to indicate we are operating in the + // context of stmt.from + let mut s = self.scope(&stmt.from); + + let filter = s.rewrite_association_as_filter(via); + stmt.filter = stmt::Filter::and(stmt.filter.take(), filter); + } + } + + pub(super) fn rewrite_via_for_insert(&mut self, stmt: &mut stmt::Insert) { + if let stmt::InsertTarget::Scope(scope) = &mut stmt.target { + self.rewrite_via_for_query(scope); + } + } + + pub(super) fn rewrite_via_for_query(&mut self, stmt: &mut stmt::Query) { + if let stmt::ExprSet::Select(select) = &mut stmt.body + && let stmt::Source::Model(model) = &mut select.source + && let Some(via) = model.via.take() + { + // Complete a scalar-terminal via used as a query source. For + // `#[has_many(via = todos.tags.name)]`, `user.tag_names()` selects + // the `name` column from `Tag` (the model the chain reaches), not + // whole `Tag` records: + // + // SELECT DISTINCT tag.name -- returning: project Tag.name + // FROM tag -- source model: Tag (the via target) + // WHERE -- the chain, unfolded below + // + // The navigation method can't build this: it runs in user code with + // no linked schema, so it can't name the target model and emits a + // placeholder source id with no projection. The resolved `Via` is + // available here, so set the source model (the FROM table) and the + // terminal projection (the RETURNING) before unfolding the chain + // into the WHERE filter. + // + // A model-terminal via has `terminal == None` — its source already + // selects whole target records — so this is a no-op there. + let scalar_terminal = + self.schema() + .app + .resolve_field_path(&via.path) + .and_then(|field| match &field.ty { + app::FieldTy::Via(v) => v.terminal.map(|terminal| (v.target, terminal)), + _ => None, + }); + + if let Some((target, _)) = scalar_terminal { + model.id = target; + } + if let Some((target, terminal)) = scalar_terminal { + select.returning = + stmt::Returning::Project(stmt::Path::field(target, terminal).into_stmt()); + } + + // Create a new scope to indicate we are operating in the + // context of stmt.target + let mut s = self.scope(&select.source); + + let filter = s.rewrite_association_as_filter(via); + select.filter = stmt::Filter::and(select.filter.take(), filter); + } + } + + pub(super) fn rewrite_association_as_filter( + &mut self, + association: stmt::Association, + ) -> stmt::Filter { + assert!( + !association.path.projection.is_empty(), + "via path must have at least one step" + ); + + // Resolve every via in the path and unfold the chain into nested + // single-step `Source::Model { via }` wrappers. After this the path + // is one step and the terminal field is guaranteed not to be a via. + let mut association = self.unfold_path(association); + + // Run the visitor's overridden `visit_stmt_query_mut` on the source + // so any `Source::Model { via: Some(_) }` introduced by unfolding is + // rewritten on its own merits before the outer single-step filter is + // built. The free-function walker would skip the override on the + // source query itself. + self.visit_stmt_query_mut(&mut association.source); + + let Some(field) = self.schema().app.resolve_field_path(&association.path) else { + todo!() + }; + + match &field.ty { + app::FieldTy::BelongsTo(rel) => { + self.rewrite_association_belongs_to_as_filter(rel, association) + } + // Direct has-one / has-many: filter the target by its paired + // `BelongsTo` against the source query. Via relations were + // already unfolded, so only direct kinds reach this arm. + app::FieldTy::Has(has) => stmt::Expr::in_subquery( + stmt::Expr::ref_self_field(has.pair_id), + *association.source, + ) + .into(), + _ => todo!("field={field:#?}"), + } + } + + /// Resolve the association path into direct relation fields, then wrap + /// each intermediate field in a nested `Source::Model { via }` query. + /// Returns an association whose path contains one direct relation step. + fn unfold_path(&self, association: stmt::Association) -> stmt::Association { + let stmt::Association { source, path } = association; + let source_model_id = source.body.as_select_unwrap().source.model_id_unwrap(); + let fields = super::relation_path::flatten_relation_path( + self.schema(), + source_model_id, + path.projection.as_slice(), + ); + + self.unfold_fields(source, &fields) + } + + fn unfold_fields( + &self, + source: Box, + fields: &[app::FieldId], + ) -> stmt::Association { + let [field_id, rest @ ..] = fields else { + unreachable!("unfold_fields called with empty fields") + }; + let field = self.schema().app.field(*field_id); + + // Base case: a single direct relation step stays on the outer + // association. + if rest.is_empty() { + return stmt::Association { + source, + path: stmt::Path::from_index(field_id.model, field_id.index), + }; + } + + let next_model_id = field + .relation_target_id() + .expect("unfolded association path field is not a relation"); + + let inner = stmt::Association { + source, + path: stmt::Path::from_index(field_id.model, field_id.index), + }; + let new_source = Box::new(stmt::Query::new_select( + stmt::Source::Model(stmt::SourceModel { + id: next_model_id, + via: Some(inner), + }), + stmt::Expr::Value(stmt::Value::Bool(true)), + )); + + debug_assert_eq!(rest[0].model, next_model_id); + self.unfold_fields(new_source, rest) + } + + fn rewrite_association_belongs_to_as_filter( + &mut self, + rel: &app::BelongsTo, + association: stmt::Association, + ) -> stmt::Filter { + // The FK lives on the source model; the target model carries the + // referenced fields. Filter is ` IN (SELECT + // FROM )` — a single field reference on each + // side for single-column FKs, a record of references for composite + // FKs (lowered to a tuple-style IN by the SQL serializer). + let target = super::key_field_refs(0, rel.foreign_key.fields.iter().map(|fk| fk.target)); + let returning = super::key_field_refs(0, rel.foreign_key.fields.iter().map(|fk| fk.source)); + + let mut source = *association.source; + source.body.as_select_mut_unwrap().returning = stmt::Returning::Project(returning); + + stmt::Expr::in_subquery(target, source).into() + } +} + +impl VisitMut for RewriteVia<'_> { + fn visit_stmt_delete_mut(&mut self, i: &mut stmt::Delete) { + self.rewrite_via_for_delete(i); + stmt::visit_mut::visit_stmt_delete_mut(self, i); + } + + fn visit_stmt_insert_mut(&mut self, i: &mut stmt::Insert) { + self.rewrite_via_for_insert(i); + stmt::visit_mut::visit_stmt_insert_mut(self, i); + } + + fn visit_stmt_query_mut(&mut self, i: &mut stmt::Query) { + self.rewrite_via_for_query(i); + stmt::visit_mut::visit_stmt_query_mut(self, i); + } +} diff --git a/packages/toasty/src/engine/lower/expr_or.rs b/packages/toasty/src/engine/lower/expr_or.rs new file mode 100644 index 00000000000..ce0e15ecd08 --- /dev/null +++ b/packages/toasty/src/engine/lower/expr_or.rs @@ -0,0 +1,47 @@ +use toasty_core::{Schema, stmt}; + +/// True when `expr` is an OR whose `IsVariant` operands cover every variant +/// of the same enum applied to the same anchor expression: the disjunction +/// is a tautology and the whole OR folds to `true`. +/// +/// Pure on `(schema, expr)`; pulled out as a free function so the +/// lowering visitor and the unit tests can both call it without +/// constructing a full `LowerStatement`. +pub(super) fn is_variant_tautology_or(schema: &Schema, expr: &stmt::ExprOr) -> bool { + // Find the first IsVariant to use as anchor. + let Some(first) = expr.operands.iter().find_map(|op| match op { + stmt::Expr::IsVariant(iv) => Some(iv), + _ => None, + }) else { + return false; + }; + + let anchor_expr = &first.expr; + let model_id = first.variant.model; + let num_variants = schema + .app + .model(model_id) + .as_embedded_enum_unwrap() + .variants + .len(); + + let mut seen = bit_set::BitSet::with_capacity(num_variants); + + for operand in &expr.operands { + let stmt::Expr::IsVariant(iv) = operand else { + continue; + }; + + // Every `IsVariant` subject must be equivalent to the anchor: + // two syntactically different (or non-deterministic) subjects + // could disagree at runtime, so covering all variants of the + // anchor tells us nothing about them. + if !iv.expr.is_equivalent_to(anchor_expr) || iv.variant.model != model_id { + return false; + } + + seen.insert(iv.variant.index); + } + + seen.count() == num_variants +} diff --git a/packages/toasty/src/engine/lower/include.rs b/packages/toasty/src/engine/lower/include.rs new file mode 100644 index 00000000000..888f8d9c97f --- /dev/null +++ b/packages/toasty/src/engine/lower/include.rs @@ -0,0 +1,742 @@ +//! Lowering for `Returning::Model` includes and deferred-field masking. +//! +//! `mapping::Model::default_returning` is computed at schema-build time with +//! every deferred field — top-level or nested inside an embedded type — +//! pre-masked to `Null`. Lowering starts from a clone of the default +//! expression and splices loaded forms in for fields named by `.include()` +//! paths or, for an `INSERT … RETURNING`, for every deferred field. +//! +//! The recursion is mapping-driven: each `mapping::Field` variant decides how +//! to descend into its corresponding expression. Driving off the mapping +//! tree (rather than the expression's shape) is what lets us reach a +//! deferred sub-field of an embed struct nested inside an enum variant — +//! the masked `Null` lives inside a `Match` expression, not a `Record`. +//! +//! Include entries arrive as [`stmt::Include`] values. The first thing we do +//! is flatten any `PathRoot::Variant` chain +//! into a plain projection of the form `[…parent_steps, variant_idx, +//! …local_var_field_steps]`. This LOCAL form mirrors the IR's `Match` arm +//! record (where each arm is `[disc, field_0, field_1, …]`), so descent +//! through enum variants is just two index steps: one selecting the arm by +//! variant index, the next addressing a local variant field. +//! +//! The flow: +//! +//! ```text +//! process_top_level_includes +//! └─► process_fields ──┬──► relation field: build_relation_subquery_inner +//! ▲ └──► non-relation: process_field +//! │ ├── deferred: splice + process_embed +//! │ └── eager embed: process_embed +//! │ ├── struct → process_fields +//! └────────────────────────────────── └── enum → process_enum_arms +//! ``` + +use toasty_core::{ + schema::{app, mapping}, + stmt, +}; + +use crate::engine::lower::LowerStatement; +use crate::schema::lazy_slot; + +struct FlatInclude { + projection: stmt::Projection, + query: Option, +} + +#[derive(Default)] +struct IncludeQuery { + filter: Option, + order_by: Option, +} + +/// The include entries that target a single field, partitioned by whether +/// they name the field itself or a sub-path within it. +/// +/// Either kind activates the field. Sub-paths only matter when the field +/// is an embed — they drive the recursion into nested fields. +struct FieldIncludes { + /// At least one include path names this field or one of its descendants. + included: bool, + /// Merged query modifiers for includes ending at this field. + top_query: IncludeQuery, + /// Tails of every `[i, …]` include path, with the leading index stripped. + sub_paths: Vec, +} + +impl LowerStatement<'_, '_> { + /// Load relations in a query projection. + pub(super) fn process_projected_returning(&mut self, expr: &mut stmt::Expr) { + if self.process_projected_field(expr) { + // A resolved field is terminal. Recursing into a scalar's base + // would load relations from its containing embed. + return; + } + + struct ProjectionVisitor<'a, 'b, 'c>(&'a mut LowerStatement<'b, 'c>); + + impl stmt::VisitMut for ProjectionVisitor<'_, '_, '_> { + fn visit_expr_mut(&mut self, expr: &mut stmt::Expr) { + self.0.process_projected_returning(expr); + } + + // Each nested statement is lowered in its own model scope. + fn visit_stmt_mut(&mut self, _: &mut stmt::Statement) {} + fn visit_stmt_query_mut(&mut self, _: &mut stmt::Query) {} + } + + // Use the general visitor so fields inside projections, lists, and + // other expression wrappers follow the same path. + stmt::visit_mut::visit_expr_mut(&mut ProjectionVisitor(self), expr); + } + + /// Load relations for one expression that resolves to an app field. + /// + /// Returns `true` when the expression is a complete field projection. + fn process_projected_field(&mut self, expr: &mut stmt::Expr) -> bool { + let Some(projected) = self.expr_cx.resolve_projected_field(expr) else { + return false; + }; + // Relation loading belongs to the statement that owns the reference. + if projected.nesting != 0 { + return false; + } + + match &projected.field.ty { + app::FieldTy::Embedded(embedded) => { + use stmt::VisitMut; + self.visit_expr_mut(expr); + super::Simplify::with_context(self.expr_cx, self.capability()).visit_expr_mut(expr); + self.process_embed( + expr, + embedded.target, + projected.mapping, + &[], + &projected.path, + ); + } + _ if projected.field.ty.is_relation() && expr.is_self_field() => { + *expr = self.build_relation_subquery(projected.field.id.index); + } + _ => {} + } + + true + } + + /// Load complete returned embeds and refresh relations whose keys changed. + pub(super) fn process_update_embedded_relations(&mut self, returning: &mut stmt::Returning) { + let stmt::Returning::Project(expr) = returning else { + return; + }; + let Some(model) = self.model() else { + return; + }; + let mapping = self.mapping_unwrap(); + self.lower_returning().process_sparse_embeds_for_update( + expr, + &model.fields, + &mapping.fields, + &stmt::Path::model(model.id), + ); + } + + fn process_sparse_embeds_for_update( + &mut self, + expr: &mut stmt::Expr, + app_fields: &[app::Field], + mapping_fields: &[mapping::Field], + record_path: &stmt::Path, + ) { + let stmt::Expr::Cast(cast) = expr else { + return; + }; + let stmt::Type::SparseRecord(returned_fields) = &mut cast.ty else { + return; + }; + let stmt::Expr::Record(record) = &mut *cast.expr else { + return; + }; + + if !record_path.projection.is_empty() { + self.refresh_changed_relations( + returned_fields, + &mut record.fields, + app_fields, + record_path, + ); + } + + for (field_index, field_expr) in returned_fields.iter().zip(&mut record.fields) { + let app::FieldTy::Embedded(embedded) = &app_fields[field_index].ty else { + continue; + }; + let embed_path = field_path(record_path, field_index); + + if matches!( + field_expr, + stmt::Expr::Cast(cast) if matches!(cast.ty, stmt::Type::SparseRecord(_)) + ) { + let app::Model::EmbeddedStruct(model) = self.schema().app.model(embedded.target) + else { + unreachable!("only structs support partial updates") + }; + self.process_sparse_embeds_for_update( + field_expr, + &model.fields, + &mapping_fields[field_index].as_struct().unwrap().fields, + &embed_path, + ); + } else { + // A non-sparse embed is returned as a complete value. + use stmt::VisitMut; + self.visit_expr_mut(field_expr); + self.process_embed( + field_expr, + embedded.target, + &mapping_fields[field_index], + &[], + &embed_path, + ); + } + } + } + + /// Add eager belongs-to fields whose foreign keys occur in a sparse result. + fn refresh_changed_relations( + &mut self, + returned_fields: &mut stmt::PathFieldSet, + returned_values: &mut Vec, + app_fields: &[app::Field], + record_path: &stmt::Path, + ) { + for (field_index, field) in app_fields.iter().enumerate() { + let app::FieldTy::BelongsTo(relation) = &field.ty else { + continue; + }; + if field.deferred { + continue; + } + + let has_returned_foreign_key_field = relation + .foreign_key + .fields + .iter() + .any(|fk| returned_fields.contains(fk.source.index)); + + if !has_returned_foreign_key_field { + continue; + } + + let relation_load = self.build_relation_subquery_inner( + field, + record_path, + &[], + IncludeQuery::default(), + ); + + // Sparse values follow ascending field indexes. Preserve that + // alignment when adding or replacing the relation field. + let value_index = returned_fields + .iter() + .take_while(|index| *index < field_index) + .count(); + if returned_fields.contains(field_index) { + returned_values[value_index] = relation_load; + } else { + returned_fields.insert(field_index); + returned_values.insert(value_index, relation_load); + } + } + } + + /// Process a model's returning record, including per-row insert results. + /// Flattens each include to its projection (folding any + /// `PathRoot::Variant` chain into discriminant-index steps), then runs + /// the recursion against the model's fields. + pub(super) fn process_top_level_includes( + &mut self, + record: &mut stmt::ExprRecord, + includes: &[stmt::Include], + ) { + let flat: Vec = includes.iter().map(flatten_include).collect(); + let app_fields = &self.model_unwrap().fields; + let mapping_fields = &self.mapping_unwrap().fields; + self.process_fields( + &mut record.fields, + app_fields, + mapping_fields, + &flat, + &stmt::Path::model(self.model_unwrap().id), + ); + } + + /// Process the fields of a struct-shaped record (top-level model or + /// embedded struct). For each field, partition matching include paths via + /// [`FieldIncludes`] and dispatch: + /// + /// - Relation → splice a subquery via [`Self::build_relation_subquery_inner`]. + /// - Anything else → [`process_field`]. + fn process_fields( + &mut self, + returning: &mut [stmt::Expr], + app_fields: &[app::Field], + mapping_fields: &[mapping::Field], + includes: &[FlatInclude], + host: &stmt::Path, + ) { + for (i, (field, mapping)) in app_fields.iter().zip(mapping_fields).enumerate() { + let field_includes = partition_includes(includes, i); + + if field.ty.is_relation() { + // Relation loads need a specific insert row to read its keys. + // We add them later, when processing each row. + if self.cx.is_insert_without_row() { + continue; + } + + // Insert planning already filled has-one and has-many results. + if self.cx.is_insert_with_row() && !field.ty.is_belongs_to() { + continue; + } + + if field_includes.included || !field.deferred { + let value = self.build_relation_subquery_inner( + field, + host, + &field_includes.sub_paths, + field_includes.top_query, + ); + returning[i] = if field.deferred { + lazy_slot::loaded_expr(value) + } else { + value + }; + } + continue; + } + + self.process_field( + &mut returning[i], + field, + mapping, + &field_includes, + &field_path(host, i), + ); + } + } + + /// Process one non-relation field by `mapping::Field` kind. + /// + /// - **Deferred field** — when activated, replaces the masked `Null` + /// with `Record([loaded])`. `loaded` is the column reference for + /// primitives or the embed's pre-computed `default_returning` for + /// embed targets. For an embed, `default_returning` has its own + /// deferred sub-fields pre-masked, so recurse to splice loaded forms + /// in for those too. + /// - **Eager embed** — recurses into the embed's expression. (Eager + /// primitives are a no-op; the column reference already sits in + /// `default_returning`.) + fn process_field( + &mut self, + returning: &mut stmt::Expr, + field: &app::Field, + mapping: &mapping::Field, + matches: &FieldIncludes, + path: &stmt::Path, + ) { + if field.deferred { + if !self.cx.is_insert() && !matches.included { + return; + } + if !self.cx.is_insert_with_row() { + *returning = lazy_slot::loaded_expr(loaded_form(field, mapping)); + } + } + + if let app::FieldTy::Embedded(embedded) = &field.ty { + let returning = if field.deferred { + let stmt::Expr::Record(outer) = returning else { + unreachable!("just-wrapped record"); + }; + &mut outer[0] + } else { + returning + }; + self.process_embed( + returning, + embedded.target, + mapping, + &matches.sub_paths, + path, + ); + } + } + + /// Process an embed's expression. Struct embeds expose a `Record`; enum + /// embeds expose a `Match` (or a bare column ref for unit-only enums, + /// which has nothing nested to splice). + fn process_embed( + &mut self, + returning: &mut stmt::Expr, + target: app::ModelId, + mapping: &mapping::Field, + sub_includes: &[FlatInclude], + path: &stmt::Path, + ) { + match (self.schema().app.model(target), mapping) { + (app::Model::EmbeddedStruct(em), mapping::Field::Struct(fs)) => { + // A nullable struct embed (`Option`) wraps its record in + // a presence `Match`; descend into the `Some` arm's record so + // its deferred sub-fields are still processed (e.g. loaded on + // `INSERT … RETURNING`). A non-nullable embed is a bare record. + let record = match returning { + stmt::Expr::Record(record) => record, + stmt::Expr::Match(match_expr) => { + let Some(stmt::Expr::Record(record)) = + match_expr.arms.first_mut().map(|arm| &mut arm.expr) + else { + return; + }; + record + } + _ => return, + }; + self.process_fields( + &mut record.fields, + em.fields.as_slice(), + fs.fields.as_slice(), + sub_includes, + path, + ); + } + (app::Model::EmbeddedEnum(em), mapping::Field::Enum(fe)) => { + self.process_enum_arms(returning, em, fe, sub_includes, path); + } + _ => {} + } + } + + /// Process the arms of an embedded enum's `Match`, handling variant + /// fields the same way as a struct's record fields. + /// + /// Each data-arm record has the discriminant at position 0 and variant + /// fields at positions `1..`. `sub_paths` are the path remainders that + /// have already had the parent field index stripped off — within them, + /// the leading step is a variant index and the next is a local variant + /// field index. We partition by variant index per arm and then by local + /// field index per variant field. Insert contexts also activate deferred + /// non-relation fields for `INSERT … RETURNING`. + fn process_enum_arms( + &mut self, + returning: &mut stmt::Expr, + app_enum: &app::EmbeddedEnum, + mapping: &mapping::FieldEnum, + sub_includes: &[FlatInclude], + path: &stmt::Path, + ) { + let stmt::Expr::Match(match_expr) = returning else { + return; + }; + + for (variant_idx, arm) in match_expr.arms.iter_mut().enumerate() { + let variant_fields = app_enum.variant_fields(variant_idx); + if variant_fields.is_empty() { + continue; + } + let stmt::Expr::Record(arm_record) = &mut arm.expr else { + continue; + }; + let variant_mapping = &mapping.variants[variant_idx]; + let host = stmt::Path::from_variant( + path.clone(), + app::VariantId { + model: app_enum.id, + index: variant_idx, + }, + ); + + let matches = partition_includes(sub_includes, variant_idx); + + self.process_fields( + &mut arm_record.fields[1..], + variant_fields, + &variant_mapping.fields, + &matches.sub_paths, + &host, + ); + } + } + + /// Build a subquery that loads the related model(s) for a + /// `BelongsTo`/`HasOne`/`HasMany` field, run the canonical lowering + /// pipeline on it, and return it stitched onto the parent statement as an + /// `Expr::Arg`. This is the `.select(rel_field)` entry point; + /// `.include(...)` goes through [`build_relation_subquery_inner`] so it + /// can pass its nested includes and filter down. + pub(super) fn build_relation_subquery(&mut self, field_index: usize) -> stmt::Expr { + self.build_relation_subquery_inner( + &self.model_unwrap().fields[field_index], + &stmt::Path::model(self.model_unwrap().id), + &[], + IncludeQuery::default(), + ) + } + + fn build_relation_subquery_inner( + &mut self, + field: &app::Field, + host: &stmt::Path, + nested: &[FlatInclude], + top_query: IncludeQuery, + ) -> stmt::Expr { + let field_index = field.id.index; + + // A multi-step (`via`) relation reaches its target through a path of + // existing relations. Build the child query as a single JOIN through + // the via chain so the engine can issue one query (per include) and + // group the children with the parent in `NestedMerge`. This relies on + // the database executing the join, so it is SQL-only — a key-value + // backend would need a cascade of per-step queries instead. + let via = match &field.ty { + app::FieldTy::Via(via) => Some(via), + _ => None, + }; + if let Some(via) = via { + if !self.capability().sql() { + todo!( + "`.include()` / `.select()` of a multi-step `via` relation is only \ + supported on SQL backends; query the relation directly instead" + ); + } + // `via` lowering does not thread per-relation filters through its + // JOIN chain yet; reject rather than silently drop them. + if top_query.filter.is_some() + || top_query.order_by.is_some() + || nested.iter().any(|fi| query_has_modifiers(&fi.query)) + { + todo!( + "include query modifiers on a multi-step `via` relation are not yet supported" + ); + } + let nested_projections: Vec = + nested.iter().map(|fi| fi.projection.clone()).collect(); + return self.build_via_include_subquery(field_index, via, &nested_projections); + } + + let (mut stmt, target_model_id) = match &field.ty { + app::FieldTy::Has(rel) => { + let mut query = stmt::Query::new_select( + rel.target, + stmt::Expr::eq( + stmt::Expr::ref_parent_model(), + stmt::Expr::ref_self_field(rel.pair_id), + ), + ); + if rel.is_one() { + // To handle single relations, we need a new query modifier that + // returns a single record and not a list. This matters for the + // type system. + query.single = true; + } + (query, rel.target) + } + // To handle single relations, we need a new query modifier that + // returns a single record and not a list. This matters for the + // type system. + app::FieldTy::BelongsTo(rel) => { + let source_fk = super::scalar_or_record( + rel.foreign_key + .fields + .iter() + .map(|fk| self.relation_source_field(host, fk.source)), + ); + let target_pk = + super::key_field_refs(0, rel.foreign_key.fields.iter().map(|fk| fk.target)); + + let mut query = + stmt::Query::new_select(rel.target, stmt::Expr::eq(source_fk, target_pk)); + query.single = true; + (query, rel.target) + } + _ => unreachable!("build_include_subquery called on non-relation field"), + }; + + // AND the user-supplied filter (if any) onto the join predicate; the + // pipeline below lowers it like any other filter on the target. + if let Some(filter) = top_query.filter { + stmt.add_filter(filter); + } + stmt.order_by = top_query.order_by; + + // Attach each non-empty remainder as a nested include on the + // subquery, carrying any deeper-level filter forward. Empty remainders + // (from a bare `.include(posts())`) need no nested include — the + // subquery itself satisfies them. The lowering pipeline will + // recursively group and process the nested includes when it encounters + // `Returning::Model` on this subquery. + for fi in nested { + if !fi.projection.is_empty() { + stmt.include(stmt::Include { + path: stmt::Path { + root: stmt::PathRoot::Model(target_model_id), + projection: fi.projection.clone(), + }, + query: fi.query.clone(), + }); + } + } + + // Run the canonical pipeline (pre-lower simplify, lowering walk, + // post-lower simplify) on the synthesized subquery, stitching it onto + // the parent as an `Expr::Arg`. + let mut statement = stmt::Statement::Query(stmt); + + self.state + .engine + .normalize_stmt(&mut statement) + .expect("valid include subquery"); + + let relation_load = self.lower_sub_stmt(statement); + + if self.cx.is_insert_with_row() && field.ty.is_belongs_to() { + // The relation query reads keys written by the enclosing inserts. + self.order_relation_load_after_enclosing_inserts(&relation_load); + Self::single_relation_from_load(relation_load) + } else { + relation_load + } + } + + fn relation_source_field( + &self, + record_path: &stmt::Path, + source_field: app::FieldId, + ) -> stmt::Expr { + let local_index = match &record_path.root { + stmt::PathRoot::Variant { variant_id, .. } if record_path.projection.is_empty() => { + // Field IDs are model-wide, but an enum arm uses indexes local + // to its variant. + let app::Model::EmbeddedEnum(model) = self.schema().app.model(variant_id.model) + else { + unreachable!() + }; + model + .variant_fields(variant_id.index) + .iter() + .position(|field| field.id == source_field) + .unwrap() + } + _ => source_field.index, + }; + + // This expression runs in the relation query and reads its parent. + field_path(record_path, local_index).into_stmt_with_nesting(1) + } +} + +fn field_path(host: &stmt::Path, index: usize) -> stmt::Path { + let mut path = host.clone(); + path.projection.push(index); + path +} + +/// Partitions includes for a field and merges modifiers on the field itself. +fn partition_includes(includes: &[FlatInclude], i: usize) -> FieldIncludes { + let mut included = false; + let mut unfiltered_self = false; + let mut top_filter: Option = None; + let mut top_order_by = None; + let mut sub_paths = Vec::new(); + for fi in includes { + if let Some((first, rest)) = fi.projection.as_slice().split_first() + && *first == i + { + included = true; + if rest.is_empty() { + top_order_by = fi.query.as_ref().and_then(|query| query.order_by.clone()); + match query_filter_expr(&fi.query) { + Some(f) if !unfiltered_self => { + let f = f.clone(); + top_filter = Some(match top_filter.take() { + Some(prev) => stmt::Expr::or(prev, f), + None => f, + }); + } + Some(_) => {} + None => { + unfiltered_self = true; + top_filter = None; + } + } + } else { + sub_paths.push(FlatInclude { + projection: stmt::Projection::from(rest), + query: fi.query.clone(), + }); + } + } + } + FieldIncludes { + included, + top_query: IncludeQuery { + filter: top_filter, + order_by: top_order_by, + }, + sub_paths, + } +} + +fn flatten_include(include: &stmt::Include) -> FlatInclude { + FlatInclude { + projection: flatten_path(&include.path), + query: include.query.clone(), + } +} + +fn query_filter_expr(query: &Option) -> Option<&stmt::Expr> { + match &query.as_ref()?.body { + stmt::ExprSet::Select(select) => select.filter.expr.as_ref(), + _ => None, + } +} + +fn query_has_modifiers(query: &Option) -> bool { + query_filter_expr(query).is_some() + || query.as_ref().is_some_and(|query| query.order_by.is_some()) +} + +/// Flatten an include [`stmt::Path`] into a single projection, folding any +/// `PathRoot::Variant` chain into a discriminant-index step. +/// +/// The result uses LOCAL field indices for variant fields (matching the IR's +/// `Match` arm record convention and the local convention also used by +/// `Schema::resolve`). Include lowering walks the IR shape, +/// not the schema, so LOCAL is what `process_enum_arms` needs. +fn flatten_path(path: &stmt::Path) -> stmt::Projection { + let mut acc = if let stmt::PathRoot::Variant { parent, variant_id } = &path.root { + let mut acc = flatten_path(parent); + acc.push(variant_id.index); + acc + } else { + stmt::Projection::identity() + }; + for step in path.projection.as_slice() { + acc.push(*step); + } + acc +} + +/// Build the loaded-form inner expression for a deferred field. +/// +/// - Primitive — the cached column reference (with any storage-type cast). +/// - Embed (struct or enum) — the embed's pre-computed `default_returning`. +fn loaded_form(field: &app::Field, mapping: &mapping::Field) -> stmt::Expr { + match (&field.ty, mapping) { + (app::FieldTy::Primitive(_), mapping::Field::Primitive(p)) => p.column_expr.clone(), + (app::FieldTy::Embedded(_), mapping::Field::Struct(s)) => s.default_returning.clone(), + (app::FieldTy::Embedded(_), mapping::Field::Enum(e)) => e.default_returning.clone(), + _ => unreachable!("deferred field has unexpected mapping shape"), + } +} diff --git a/packages/toasty/src/engine/lower/insert.rs b/packages/toasty/src/engine/lower/insert.rs new file mode 100644 index 00000000000..cadd2178aff --- /dev/null +++ b/packages/toasty/src/engine/lower/insert.rs @@ -0,0 +1,544 @@ +use bit_set::BitSet; +use toasty_core::{schema::app, stmt}; + +use crate::engine::lower::LowerStatement; + +/// Process the scope component of an insert statement. +struct ApplyInsertScope<'a> { + expr: &'a mut stmt::Expr, +} + +/// How to encode an auto-generated value for a given field. Each newtype +/// embed layer adds one record wrapping around the leaf primitive value. +struct AutoTarget<'a> { + ty: &'a stmt::Type, + wrap_depth: usize, +} + +impl AutoTarget<'_> { + fn wrap(&self, mut value: stmt::Value) -> stmt::Value { + for _ in 0..self.wrap_depth { + value = stmt::Value::record_from_vec(vec![value]); + } + value + } + + fn wrap_expr(&self, mut expr: stmt::Expr) -> stmt::Expr { + for _ in 0..self.wrap_depth { + expr = stmt::Expr::record_from_vec(vec![expr]); + } + expr + } +} + +impl LowerStatement<'_, '_> { + // First, apply the insertion scope to the insertion values + pub(super) fn apply_insert_scope( + &mut self, + target: &mut stmt::InsertTarget, + source: &mut stmt::Query, + ) { + let stmt::InsertTarget::Scope(scope) = target else { + // Insertion is not targetting a scope + return; + }; + + let stmt::ExprSet::Values(values) = &mut source.body else { + todo!() + }; + + let scope = &scope.body.as_select_unwrap(); + + if let Some(filter) = &scope.filter.expr { + for expr in &mut values.rows { + ApplyInsertScope { expr }.apply_expr(filter) + } + } + + *target = stmt::InsertTarget::Model(scope.source.model_id_unwrap()); + } + + pub(super) fn preprocess_insert_values( + &mut self, + source: &mut stmt::Query, + returning: &mut Option, + preserve_returning_projection: bool, + returning_model: bool, + ) { + let stmt::ExprSet::Values(values) = &mut source.body else { + todo!() + }; + + let Some(model) = self.expr_cx.target_as_model() else { + return; + }; + + let mut set_fields: BitSet = BitSet::default(); + + // First, apply any defaults while also tracking all the fields that are set. + for (index, row) in values.rows.iter_mut().enumerate() { + self.lower_insert_with_row(index, |lower| { + lower.apply_app_level_insertion_defaults(model, row, &mut set_fields); + }); + } + + // `preserve_returning_projection` is true for `INSERT ... DO NOTHING`. + // Such an insert can return no rows after a conflict, so its returning + // clause must stay a projection over the database result. For example, + // converting `RETURNING id` to an expression for input row zero would + // try to project a row that does not exist. Other inserts return a row + // for every input row, so convert their returning projection into + // per-row expressions for relation planning. + if !preserve_returning_projection { + self.convert_returning_for_insert(values, returning, source.single); + } + + for (index, row) in values.rows.iter_mut().enumerate() { + self.lower_insert_with_row(index, |lower| { + lower.plan_stmt_insert_relations(row, returning, index); + if returning_model { + lower.plan_insert_returning_relations( + row, + returning, + index, + preserve_returning_projection, + ); + } + lower.verify_field_constraints(model, row); + }); + } + } + + /// Load eager relations after insert planning has resolved their foreign keys. + fn plan_insert_returning_relations( + &mut self, + row: &stmt::Expr, + returning: &mut Option, + index: usize, + preserve_returning_projection: bool, + ) { + let Some(record) = + Self::insert_returning_record_mut(returning, index, preserve_returning_projection) + else { + return; + }; + + // A preserved projection is shared across rows, so it cannot hold + // a different relation load for each row. + if preserve_returning_projection && index > 0 { + for field in &self.model_unwrap().fields { + if let app::FieldTy::BelongsTo(rel) = &field.ty { + assert!( + field.deferred || Self::belongs_to_fk_is_unset(row, rel), + "eager belongs_to in a multi-row insert with a preserved returning projection" + ); + } + } + } + + self.process_top_level_includes(record, &[]); + } + + /// Return whether the row has no usable foreign key for this relation. + fn belongs_to_fk_is_unset(row: &stmt::Expr, rel: &app::BelongsTo) -> bool { + rel.foreign_key.fields.iter().any(|fk_field| { + row.entry(fk_field.source.index).is_none_or(|entry| { + let expr = entry.to_expr(); + expr.is_value_null() || expr.is_default() + }) + }) + } + + /// Return the model record for one row of an INSERT's returning value. + fn insert_returning_record_mut( + returning: &mut Option, + index: usize, + preserve_returning_projection: bool, + ) -> Option<&mut stmt::ExprRecord> { + match returning { + Some(stmt::Returning::Expr(stmt::Expr::List(rows))) + if !preserve_returning_projection => + { + Some(rows.items[index].as_record_mut_unwrap()) + } + Some(stmt::Returning::Expr(stmt::Expr::Record(record))) + if !preserve_returning_projection => + { + Some(record) + } + Some(stmt::Returning::Project(stmt::Expr::Record(record))) + if preserve_returning_projection => + { + Some(record) + } + _ => None, + } + } + + /// Make a relation load wait for the database writes that can create its row. + pub(super) fn order_relation_load_after_enclosing_inserts(&mut self, load: &stmt::Expr) { + let stmt::Expr::Arg(expr_arg) = load else { + unreachable!("belongs_to subquery lowers to a sub-statement arg"); + }; + let crate::engine::hir::Arg::Sub { + stmt_id: sub_id, .. + } = self.curr_stmt_info().args[expr_arg.position] + else { + unreachable!("subquery arg refers to a sub-statement"); + }; + + let state = &mut *self.state; + let (insert_stmts, hir) = (&state.insert_stmts, &mut state.hir); + for &target in insert_stmts { + hir[sub_id].add_dep(target, crate::engine::hir::DepKind::Effect); + } + } + + /// Convert a relation query's row list into one nullable record. + pub(super) fn single_relation_from_load(load: stmt::Expr) -> stmt::Expr { + stmt::Expr::match_expr( + load.clone(), + vec![stmt::MatchArm { + pattern: stmt::Value::List(vec![]), + expr: stmt::Expr::null(), + }], + stmt::Expr::project(load, [0usize]), + ) + } + + // Checks all fields of a record and handles nulls + fn apply_app_level_insertion_defaults( + &mut self, + model: &app::ModelRoot, + expr: &mut stmt::Expr, + set_fields: &mut BitSet, + ) { + // First, we pad the record to account for all fields + if let stmt::Expr::Record(expr_record) = expr { + // TODO: get rid of this + assert_eq!(expr_record.len(), model.fields.len()); + } + + // Next, we have to find all belongs-to fields and normalize them to FK + // values + for field in &model.fields { + if let app::FieldTy::BelongsTo(rel) = &field.ty { + let mut field_expr = expr.entry_mut(field.id.index); + + if field_expr.is_value_null() { + continue; + } + + if !field_expr.is_value() && !field_expr.is_record() { + continue; + } + let value = field_expr.take(); + super::relation::assign_belongs_to_key(&rel.foreign_key, value, |field, value| { + expr.entry_mut(field.index).insert(value); + }); + } + } + + // Initialize version fields to 1 if not already set by the user. + // For embedded newtypes (e.g. `Version(u64)`), wrap in one Record per + // embed layer so the value round-trips through the embed's `Load` impl. + for field in &model.fields { + if field.is_versionable() { + let mut field_expr = expr.entry_mut(field.id.index); + if field_expr.is_default() || field_expr.is_value_null() { + let target = self.auto_target(field.id); + field_expr.insert(target.wrap(stmt::Value::U64(1)).into()); + } + } + } + + // We have to handle auto fields first because they are often the + // identifier which may be referenced to handle associations. + for field in &model.fields { + let mut field_expr = expr.entry_mut(field.id.index); + + if field_expr.is_default() { + // If the field is defined to be auto-populated, then populate + // it here. + if let Some(auto) = &field.auto { + // For an embedded newtype, the auto value is generated as + // the inner primitive and wrapped in a single-element + // record so it round-trips through the embed `Load` impl. + let target = self.auto_target(field.id); + match auto { + app::AutoStrategy::Uuid(version) => { + let id = match version { + app::UuidVersion::V4 => uuid::Uuid::new_v4(), + app::UuidVersion::V7 => uuid::Uuid::now_v7(), + }; + let primitive = match &target.ty { + stmt::Type::String => stmt::Value::String(id.to_string()), + stmt::Type::Uuid => stmt::Value::Uuid(id), + other => panic!( + "auto-generated UUID cannot be inserted into column of type {other:?}" + ), + }; + field_expr.insert(target.wrap(primitive).into()); + } + app::AutoStrategy::Increment => { + // Leave value as `Expr::Default` for primitives so + // the database fills it in. For embedded newtypes + // the column-projection step needs a Record of the + // right shape (one wrapping per nested embed) so + // it can extract `Default` per column. + if target.wrap_depth > 0 { + field_expr.insert(target.wrap_expr(stmt::Expr::Default)); + } + } + } + } + } + + if !field_expr.is_value_null() { + set_fields.insert(field.id.index); + } + } + } + + fn convert_returning_for_insert( + &mut self, + values: &stmt::Values, + returning: &mut Option, + single: bool, + ) { + // If there is no returning statement, there is nothing to convert + let Some(stmt::Returning::Project(projection)) = returning else { + return; + }; + + #[derive(Debug)] + struct Input(usize); + + impl stmt::Input for Input { + fn resolve_arg( + &mut self, + expr_arg: &stmt::ExprArg, + projection: &stmt::Projection, + ) -> Option { + let _ = (expr_arg, projection); + None + } + + fn resolve_ref( + &mut self, + expr_reference: &stmt::ExprReference, + projection: &stmt::Projection, + ) -> Option { + let expr_column = expr_reference.as_expr_column()?; + + assert!( + expr_column.nesting == 0 && expr_column.table == 0, + "expr_reference={expr_reference:#?}" + ); + assert!(projection.is_identity(), "TODO"); + + Some(stmt::Expr::project(*expr_reference, self.0)) + } + } + + let mut converted = vec![]; + + for i in 0..values.rows.len() { + let mut converted_row = projection.clone(); + converted_row.substitute(Input(i)); + converted.push(converted_row); + } + + *returning = Some(stmt::Returning::Expr(if single { + assert!(converted.len() == 1); + converted.into_iter().next().unwrap() + } else { + stmt::Expr::list_from_vec(converted) + })); + } + + fn verify_field_constraints(&mut self, model: &app::ModelRoot, expr: &mut stmt::Expr) { + for field in &model.fields { + if field.nullable && field.constraints.is_empty() { + continue; + } + + let field_expr = expr.entry(field.id.index).unwrap(); + + if !field.nullable && field_expr.is_value_null() { + // Relations are handled differently + if !field.ty.is_relation() && field.auto.is_none() { + self.state + .errors + .push(toasty_core::Error::validation_failed(format!( + "insert missing non-nullable field `{}` in model `{}`", + field.name, + model.name.upper_camel_case() + ))); + } + } + + for constraint in &field.constraints { + if let Err(err) = constraint.check(&field_expr) { + self.state.errors.push(err); + } + } + } + } +} + +impl<'b> LowerStatement<'_, 'b> { + /// Returns the primitive type the auto value should be encoded as, plus + /// how many record wrappings to apply for the embed chain. Walks down + /// through nested newtype embeds (`Outer(Inner(u64))`) until it reaches + /// the leaf primitive. + fn auto_target(&self, field_id: app::FieldId) -> AutoTarget<'b> { + let mut ty = &self.schema().app.field(field_id).ty; + let mut wrap_depth = 0; + loop { + match ty { + app::FieldTy::Primitive(primitive) => { + return AutoTarget { + ty: &primitive.ty, + wrap_depth, + }; + } + app::FieldTy::Embedded(embedded) => { + let target = self.schema().app.model(embedded.target); + let app::Model::EmbeddedStruct(es) = target else { + panic!( + "#[auto] on embedded enum is not supported (target {:?})", + embedded.target + ); + }; + let [inner] = es.fields.as_slice() else { + panic!( + "#[auto] on embedded type with {} fields; expected exactly one", + es.fields.len() + ); + }; + ty = &inner.ty; + wrap_depth += 1; + } + _ => panic!("#[auto] not allowed on non-primitive fields"), + } + } + } + + /// The target projection and per-row delta for a versionable field's update + /// assignment, used to build an atomic `version = version + 1` via + /// [`Assignment::Add`](stmt::Assignment::Add). + /// + /// The projection walks past any embed-newtype layers (`Version(u64)`) to + /// the leaf primitive column — `[field, 0, …]` — and the delta is the bare + /// literal `1`. Arithmetic operators apply only to primitive columns, so + /// unlike the whole-record `Set` the instance path uses, the increment must + /// reach the leaf rather than sit on the embed field. + pub(super) fn version_increment_target( + &self, + field_id: app::FieldId, + ) -> (stmt::Projection, stmt::Expr) { + let target = self.auto_target(field_id); + + // `[field_id.index]` then one `0` per embed layer, reaching the leaf. + let mut steps = vec![field_id.index]; + steps.resize(1 + target.wrap_depth, 0); + + ( + stmt::Projection::from(steps.as_slice()), + stmt::Expr::Value(stmt::Value::U64(1)), + ) + } + + /// Adds an atomic version increment unless the statement already assigns + /// the version field. + pub(super) fn inject_version_increment( + &self, + assignments: &mut stmt::Assignments, + ) -> Option { + let version_id = self.model().and_then(|m| m.version_field()).map(|f| f.id)?; + + let already_assigned = assignments + .keys() + .any(|projection| projection.as_slice().first() == Some(&version_id.index)); + if already_assigned { + return None; + } + + let (projection, delta) = self.version_increment_target(version_id); + assignments.add(projection.clone(), delta); + Some(projection) + } +} + +impl ApplyInsertScope<'_> { + fn apply_expr(&mut self, stmt: &stmt::Expr) { + match stmt { + stmt::Expr::And(exprs) => { + for expr in exprs { + self.apply_expr(expr); + } + } + stmt::Expr::BinaryOp(e) if e.op.is_eq() => match (&*e.lhs, &*e.rhs) { + ( + stmt::Expr::Reference(expr_ref @ stmt::ExprReference::Field { .. }), + rhs @ stmt::Expr::Value(..), + ) => { + self.apply_eq_constraint(expr_ref, rhs); + } + ( + lhs @ stmt::Expr::Value(..), + stmt::Expr::Reference(expr_ref @ stmt::ExprReference::Field { .. }), + ) => { + self.apply_eq_constraint(expr_ref, lhs); + } + ( + lhs_expr @ stmt::Expr::Reference( + lhs @ stmt::ExprReference::Field { + nesting: nesting_lhs, + .. + }, + ), + rhs_expr @ stmt::Expr::Reference( + rhs @ stmt::ExprReference::Field { + nesting: nesting_rhs, + .. + }, + ), + ) => match (nesting_lhs, nesting_rhs) { + (0, _) if *nesting_rhs > 0 => self.apply_eq_constraint(lhs, rhs_expr), + (_, 0) if *nesting_lhs > 0 => self.apply_eq_constraint(rhs, lhs_expr), + _ => panic!("exactly one field must reference parent"), + }, + _ => todo!("EXPR = {:#?}", stmt), + }, + // Constants are ignored + stmt::Expr::Value(_) => {} + _ => todo!("EXPR = {:#?}", stmt), + } + } + + fn apply_eq_constraint(&mut self, expr_ref: &stmt::ExprReference, val: &stmt::Expr) { + let stmt::ExprReference::Field { nesting, index } = expr_ref else { + todo!("handle non-field reference"); + }; + + assert!(*nesting == 0, "TODO: handle references to parent scopes"); + + let mut existing = self.expr.entry_mut(*index); + + if !existing.is_value_null() && !existing.is_default() { + if let stmt::EntryMut::Value(existing) = existing { + if let stmt::Expr::Value(val) = val { + assert_eq!(existing, val); + } else { + todo!() + } + } else { + todo!() + } + } else { + existing.insert(val.clone()); + } + } +} diff --git a/packages/toasty/src/engine/lower/lift_in_subquery.rs b/packages/toasty/src/engine/lower/lift_in_subquery.rs new file mode 100644 index 00000000000..273d8dec15c --- /dev/null +++ b/packages/toasty/src/engine/lower/lift_in_subquery.rs @@ -0,0 +1,773 @@ +//! App-level rewrites that lift relation references out of `IN` subqueries +//! into direct foreign-key comparisons. +//! +//! [`LiftInSubquery`] runs as a whole-statement pre-pass before the main +//! lowering walk. The visitor overrides `visit_expr_mut` to fire two +//! rewrites pre-children: +//! +//! - [`lift_in_subquery`] fires on `Expr::InSubquery` where the LHS is a +//! relation field reference (`BelongsTo`/`HasOne`/`HasMany`). For +//! `BelongsTo` it tries to lift the subquery's filter into FK comparisons +//! on the parent, falling back to a re-targeted IN subquery on the +//! foreign-key column. For `HasOne`/`HasMany` it always rewrites to a +//! foreign-key IN subquery against the related table. +//! +//! - [`try_lift_relation_path_predicate`] fires on a comparison +//! (`Expr::BinaryOp`), a `LIKE`/`ILIKE` (`Expr::Like`), or a variant +//! check (`Expr::IsVariant`) whose subject walks a relation field. For +//! example, filtering profiles by their user's name +//! (`profile.user.name = 'alice'`) rewrites to: +//! +//! ```text +//! Profile.user_id IN (SELECT User.id FROM User WHERE User.name = 'alice') +//! ``` +//! +//! Simplification combines compatible subqueries after this pass, before +//! lowering extracts subqueries into separate statements. +//! +//! The relation a path walks may be a field of the model or a `belongs_to` +//! inside one of its embedded types (`v.human.name = 'alice'`, where +//! `human` is a field of the `Human` variant of `v`). [`resolve_relation_path`] +//! resolves either to a [`Relation`]; the rewrites above are the same for +//! both, differing only in how the relation's foreign-key `IN` subquery +//! names the key on the host side. A predicate *at* an embedded relation +//! (`v.human = alice`, `v.human IN list`) compares keys and is not lifted: +//! `LowerStatement`'s operand hooks substitute the key projection. +//! +//! A pre-pass is necessary (rather than folding into +//! `LowerStatement::visit_expr_mut` per #823's pattern) because not every +//! expression that contains an `IN` subquery flows through +//! `LowerStatement::visit_expr_mut`; `ApplyInsertScope::apply_expr` in +//! particular walks insert-scope constraint expressions through its own +//! recursion and would panic on an unlifted relation `IN` subquery. +//! +//! The free functions [`lift_in_subquery`] and +//! [`try_lift_relation_path_predicate`] are exposed on `&ExprContext` so +//! the visitor and the unit tests can both call them without constructing +//! a `LiftInSubquery`. + +use toasty_core::{ + schema::app::{self, BelongsTo, FieldId, FieldTy, ModelId}, + stmt::{self, Expr, ExprContext, IntoExprTarget, Visit, VisitMut}, +}; + +use super::relation_expr; + +/// Pre-lowering pass that lifts relation references out of `IN`-subquery +/// and projection comparisons into direct foreign-key forms. Runs as a +/// whole-statement visitor before the main lowering walk: code paths that +/// process expressions outside the lowering walk (notably +/// `ApplyInsertScope::apply_expr`) see the already-lifted form. +pub(super) struct LiftInSubquery<'a> { + cx: ExprContext<'a>, + exclude_nulls: bool, +} + +impl<'a> LiftInSubquery<'a> { + pub(super) fn new(cx: ExprContext<'a>, exclude_nulls: bool) -> Self { + Self { cx, exclude_nulls } + } + + pub(super) fn rewrite(&mut self, stmt: &mut stmt::Statement) { + self.visit_mut(stmt); + } + + fn scope<'scope>(&'scope self, target: impl IntoExprTarget<'scope>) -> LiftInSubquery<'scope> { + LiftInSubquery { + cx: self.cx.scope(target), + exclude_nulls: self.exclude_nulls, + } + } + + fn exclude_nulls(&self, expr: &mut stmt::Expr) { + if !self.exclude_nulls { + return; + } + + let stmt::Expr::InSubquery(expr) = expr else { + return; + }; + let select = expr.query.body.as_select_mut_unwrap(); + let target = select.source.model_id_unwrap(); + let returning = select.returning.as_project_unwrap().clone(); + + for field in returning + .as_record() + .map_or(std::slice::from_ref(&returning), |record| &record.fields) + { + let stmt::Expr::Reference(stmt::ExprReference::Field { index, .. }) = field else { + unreachable!(); + }; + if self.cx.schema().app.field(target.field(*index)).nullable { + select.add_filter(stmt::Expr::is_not_null(field.clone())); + } + } + } +} + +impl VisitMut for LiftInSubquery<'_> { + fn visit_expr_mut(&mut self, expr: &mut stmt::Expr) { + // Apply lifts pre-children: the rewrites pattern-match on app- + // level `Reference::Field` to a relation, and the children walk + // would not introduce relation references that were not there. + let lifted = match expr { + // Normalization expands `NOT IN` to `Not(InSubquery(..))`, so the + // lift below fires on the inner membership and keeps the + // negation outside. + stmt::Expr::InSubquery(e) if !e.negated => { + lift_in_subquery(&self.cx, &e.expr, &e.query) + } + stmt::Expr::BinaryOp(_) | stmt::Expr::Like(_) | stmt::Expr::IsVariant(_) => { + try_lift_relation_path_predicate(&self.cx, expr) + } + _ => None, + }; + + if let Some(mut lifted) = lifted { + self.exclude_nulls(&mut lifted); + *expr = lifted; + } + + // Walk children (which may themselves be expressions needing + // lifts on subtrees). + stmt::visit_mut::visit_expr_mut(self, expr); + } + + fn visit_stmt_delete_mut(&mut self, stmt: &mut stmt::Delete) { + self.visit_source_mut(&mut stmt.from); + + let mut s = self.scope(&stmt.from); + + s.visit_filter_mut(&mut stmt.filter); + + if let Some(returning) = &mut stmt.returning { + s.visit_returning_mut(returning); + } + } + + fn visit_stmt_insert_mut(&mut self, stmt: &mut stmt::Insert) { + self.visit_insert_target_mut(&mut stmt.target); + + let mut s = self.scope(&stmt.target); + + s.visit_stmt_query_mut(&mut stmt.source); + + if let Some(returning) = &mut stmt.returning { + s.visit_returning_mut(returning); + } + } + + fn visit_stmt_select_mut(&mut self, stmt: &mut stmt::Select) { + self.visit_source_mut(&mut stmt.source); + + let mut s = self.scope(&stmt.source); + + s.visit_filter_mut(&mut stmt.filter); + s.visit_returning_mut(&mut stmt.returning); + } + + fn visit_stmt_update_mut(&mut self, stmt: &mut stmt::Update) { + self.visit_update_target_mut(&mut stmt.target); + + let mut s = self.scope(&stmt.target); + + s.visit_assignments_mut(&mut stmt.assignments); + s.visit_filter_mut(&mut stmt.filter); + + if let Some(expr) = &mut stmt.condition.expr { + s.visit_expr_mut(expr); + } + + if let Some(returning) = &mut stmt.returning { + s.visit_returning_mut(returning); + } + } +} + +struct LiftBelongsTo<'a> { + cx: ExprContext<'a>, + belongs_to: &'a BelongsTo, + // TODO: switch to bit field set + fk_field_matches: Vec, + fail: bool, + operands: Vec, +} + +/// Lift `expr IN (subquery)` into a foreign-key-based comparison when +/// `expr` is a relation field reference and `subquery` targets the +/// relation's target model. +/// +/// Returns `None` when the LHS is not a relation field reference or when +/// the lift cannot apply. When the lift succeeds, the returned +/// expression may itself contain unlowered references and the caller is +/// expected to re-visit it through the lowering walk. +pub(super) fn lift_in_subquery( + cx: &ExprContext, + expr: &stmt::Expr, + query: &stmt::Query, +) -> Option { + // The expression is a path expression referencing a relation. + match expr { + // `Project(Ref(rel), [head, ...tail])` — the path traverses through a + // relation field (`rel`) before reaching the relation the subquery + // targets, or reaches a relation inside an embedded type. Re-root + // the path at the relation's target model and rebuild as a nested + // IN-subquery on that model, then recurse to lift the outer hop. + stmt::Expr::Project(_) | stmt::Expr::Variant(_) => { + lift_projection_in_subquery(cx, expr, query) + } + stmt::Expr::Reference(expr_reference @ stmt::ExprReference::Field { .. }) => { + let field = cx.resolve_expr_reference(expr_reference).as_field_unwrap(); + lift_relation_in_subquery(cx, &Relation::Field(field), query) + } + _ => None, + } +} + +/// A resolved relation with the expressions needed for subquery lifting. +struct RelationPath<'a> { + relation: Relation<'a>, + target: ModelId, + target_expr: Option, +} + +/// The relation a path walks. Both lift to a foreign-key `IN` subquery on +/// the target model; they differ in how the host side names the key. +enum Relation<'a> { + /// A relation field of the model: `BelongsTo`, `Has`, or `Via`. + Field(&'a app::Field), + + /// A `belongs_to` inside an embedded type, with the expression + /// projecting its key field(s) out of the host row. + Embedded { + belongs_to: &'a BelongsTo, + key_expr: Expr, + }, +} + +fn resolve_relation_path<'a>(cx: &ExprContext<'a>, expr: &Expr) -> Option> { + let resolved = relation_expr::resolve(cx, expr)?; + let relation = match resolved.embedded() { + Some(belongs_to) => Relation::Embedded { + belongs_to, + key_expr: resolved.key_expr()?, + }, + None => Relation::Field(resolved.field), + }; + Some(RelationPath { + relation, + target: resolved.target, + target_expr: resolved.target_expr(), + }) +} + +/// Lift `relation IN (subquery)` for a resolved relation. +/// +/// Direct relations lift through their paired foreign keys. A `via` has no +/// single pair, so normalize its relation chain into a projected path and +/// use the projection lift. An embedded relation's foreign key is projected +/// out of the host row. Returns `None` when the field is not a relation. +fn lift_relation_in_subquery( + cx: &ExprContext, + relation: &Relation<'_>, + query: &stmt::Query, +) -> Option { + match relation { + Relation::Field(field) => match &field.ty { + FieldTy::BelongsTo(belongs_to) => lift_belongs_to_in_subquery(cx, belongs_to, query), + FieldTy::Has(has) => { + lift_has_n_in_subquery(has.target, has.pair(&cx.schema().app), query) + } + FieldTy::Via(via) => lift_via_in_subquery(cx, via, query), + _ => None, + }, + Relation::Embedded { + belongs_to, + key_expr, + } => lift_fk_in_subquery( + belongs_to.target, + key_expr.clone(), + super::key_field_refs(0, belongs_to.foreign_key.fields.iter().map(|fk| fk.target)), + query, + ), + } +} + +/// Lift an `IN` subquery whose left side names a `via` relation. +/// +/// For `User.groups.any(Group.name == "Rust")`, where `groups` follows +/// `User.memberships.group`, this builds: +/// +/// ```text +/// User.id IN ( +/// SELECT Membership.user_id FROM Membership +/// WHERE Membership.group_id IN ( +/// SELECT Group.id FROM Group WHERE Group.name == "Rust" +/// ) +/// ) +/// ``` +/// +/// The path is expanded into direct relation fields, converted to the same +/// projected expression produced by an explicit relation chain, and then +/// handled by the regular projection lift. +fn lift_via_in_subquery( + cx: &ExprContext, + via: &app::Via, + query: &stmt::Query, +) -> Option { + if via.is_scalar() { + return None; + } + + let fields = super::relation_path::flatten_via_path(cx.schema(), via)?; + let target = fields + .last() + .and_then(|field| cx.schema().app.field(*field).relation_target_id())?; + + if target != via.target || target != query.body.as_select_unwrap().source.model_id_unwrap() { + return None; + } + + let (base, projection) = fields.split_first()?; + let base = stmt::Expr::ref_self_field(*base); + let path = if projection.is_empty() { + base + } else { + let projection = projection + .iter() + .map(|field| field.index) + .collect::>(); + stmt::Expr::project(base, projection.as_slice()) + }; + + lift_in_subquery(cx, &path, query) +} + +/// Lifts an `IN`-subquery whose left side is a path through a relation field. +/// +/// `.any()` on a relation chain — for example, `Release.project.topics.any(name +/// == "rust")` — builds an `InSubquery` whose LHS projects through the +/// chain: +/// +/// ```text +/// InSubquery { +/// expr: Project(Ref(Release.project), [Project.topics_idx]), +/// query: SELECT FROM Topic WHERE name == "rust", +/// } +/// ``` +/// +/// Two paths handle this: +/// +/// **Fused.** [`try_fuse_paired_relations`] recognizes the common +/// `BelongsTo → Has` chain over a shared primary key and emits a single +/// FK-on-FK `IN` that bypasses the intermediate model: +/// +/// ```text +/// Release.project_id IN (SELECT Topic.project_id FROM Topic WHERE name == "rust") +/// ``` +/// +/// **General.** For chains the fast path doesn't match (multi-hop +/// projections, non-paired relations), re-root the projection at the +/// relation's target model and wrap the original subquery in a nested `IN` +/// against that model, then recurse on the outer hop. For the same input, +/// the fallback produces: +/// +/// ```text +/// Release.project IN ( +/// SELECT FROM Project +/// WHERE Project.topics IN (SELECT FROM Topic WHERE name == "rust") +/// ) +/// ``` +/// +/// The recursive call lifts the outer `Release.project` hop via the standard +/// `BelongsTo` branch; the inner `Project.topics` hop is lifted on the next +/// visitor pass when [`LiftInSubquery`]'s children walk reaches it. +/// +/// A path may instead reach a relation inside an embedded type +/// (`v.human` or `v.human.posts`). The steps through the embed resolve to +/// that relation, whose foreign key is projected out of the host row; any +/// steps past it are re-rooted at its target as above. +fn lift_projection_in_subquery( + cx: &ExprContext, + path: &stmt::Expr, + query: &stmt::Query, +) -> Option { + let RelationPath { + relation, + target, + target_expr, + } = resolve_relation_path(cx, path)?; + + let Some(inner_lhs) = target_expr else { + // The path ends at the relation (`v.human IN (subquery)`). + return lift_relation_in_subquery(cx, &relation, query); + }; + + if let Some(index) = inner_lhs.as_self_field_index() + && let Relation::Field(field) = relation + && let Some(direct) = try_fuse_paired_relations(cx, field, target, index, query) + { + return Some(direct); + } + + let new_subquery = stmt::Query::new_select( + stmt::Source::from(target), + Expr::in_subquery(inner_lhs, query.clone()), + ); + + lift_relation_in_subquery(cx, &relation, &new_subquery) +} + +/// Fuses a `BelongsTo → Has` chain into a single FK-on-FK `IN` when both +/// relations meet at the same primary key. +/// +/// # Example +/// +/// Given the schema: +/// +/// ```text +/// Todo { category_id: Category.id, ... } // BelongsTo Todo.category +/// Category { todos: HasMany Todo, ... } // Has, paired with Todo.category +/// ``` +/// +/// The path `Todo.category.todos` lifts as follows: +/// +/// ```text +/// // Input +/// InSubquery { +/// expr: Project(Ref(Todo.category), [Category.todos_idx]), +/// query: SELECT FROM Todo WHERE title == "salad", +/// } +/// +/// // Output +/// InSubquery { +/// expr: Ref(Todo.category_id), +/// query: SELECT Todo.category_id FROM Todo WHERE title == "salad", +/// } +/// ``` +/// +/// The outer relation's FK source columns become the LHS; the inner +/// relation's paired-BelongsTo FK source columns become the subquery's +/// returning list. The user's original filter is preserved verbatim. +/// +/// # Why the fusion is sound +/// +/// Composing the two hops without fusion routes through the intermediate +/// model: +/// +/// ```text +/// Todo.category_id IN ( +/// SELECT Category.id FROM Category +/// WHERE Category.id IN (SELECT Todo.category_id FROM Todo WHERE title == "salad") +/// ) +/// ``` +/// +/// Both FKs target `Category.id`, so every value the innermost subquery +/// returns exists in `Category.id` under FK integrity (which the engine +/// already assumes). The middle filter therefore admits every row the inner +/// returns, and the middle `SELECT Category.id` simply re-emits them. The +/// `Category` scan is a no-op and can be dropped. +/// +/// Returns `None` when the chain doesn't match the pattern — outer isn't a +/// `BelongsTo`, inner isn't a `Has`, or the FK columns don't line up. +fn try_fuse_paired_relations( + cx: &ExprContext, + outer_field: &app::Field, + target_model_id: ModelId, + head_idx: usize, + query: &stmt::Query, +) -> Option { + let outer_belongs_to = match &outer_field.ty { + FieldTy::BelongsTo(rel) => rel, + _ => return None, + }; + + let target_model = cx.schema().app.model(target_model_id).as_root_unwrap(); + let head_field = target_model.fields.get(head_idx)?; + let inner_has = match &head_field.ty { + FieldTy::Has(has) => has, + _ => return None, + }; + let inner_pair = inner_has.pair(&cx.schema().app); + + // Both FKs must reference the same PK columns in the same order. + if outer_belongs_to.foreign_key.fields.len() != inner_pair.foreign_key.fields.len() { + return None; + } + for (outer_fk, inner_fk) in outer_belongs_to + .foreign_key + .fields + .iter() + .zip(inner_pair.foreign_key.fields.iter()) + { + if outer_fk.target != inner_fk.target { + return None; + } + } + + lift_fk_in_subquery( + inner_has.target, + super::key_field_refs( + 0, + outer_belongs_to + .foreign_key + .fields + .iter() + .map(|fk| fk.source), + ), + super::key_field_refs(0, inner_pair.foreign_key.fields.iter().map(|fk| fk.source)), + query, + ) +} + +/// Rewrites a predicate whose subject walks a relation into a foreign-key +/// subquery. +/// +/// `Profile::filter(Profile::fields().user().name().eq("alice"))` starts as +/// the filter `profile.user.name = 'alice'`, where the left side projects +/// through the `user` relation. This moves the comparison into a subquery on +/// the target model and defers to [`lift_relation_in_subquery`]: +/// +/// ```text +/// Profile.user_id IN (SELECT User.id FROM User WHERE User.name = 'alice') +/// ``` +/// +/// The same applies to `LIKE`/`ILIKE` (`profile.user.name LIKE 'al%'`) and +/// to a variant check on an enum reached through the relation +/// (`link.item.state IS Selected`, the guard the typed layer fixes next to +/// `link.item.state.selected.value == x`). +/// +/// Returns `None` when `expr` is not one of those predicates or its subject +/// does not walk a relation. +pub(super) fn try_lift_relation_path_predicate( + cx: &ExprContext, + expr: &stmt::Expr, +) -> Option { + let (path, filter) = resolve_relation_predicate(cx, expr)?; + lift_relation_predicate(cx, &path, filter) +} + +/// Resolve a predicate whose subject walks a relation to that relation and +/// the predicate re-rooted at the relation's target model — the subquery +/// filter [`lift_relation_predicate`] wraps. +/// +/// A comparison's subject may be either operand: `'alice' = profile.user.name` +/// resolves with the operator commuted. +fn resolve_relation_predicate<'a>( + cx: &ExprContext<'a>, + expr: &stmt::Expr, +) -> Option<(RelationPath<'a>, stmt::Expr)> { + match expr { + Expr::BinaryOp(e) => { + let comparison = |op: stmt::BinaryOp, subject: &Expr, other: &Expr| { + resolve_relation_path_predicate(cx, subject, |target_lhs| { + Expr::binary_op(target_lhs, op, other.clone()) + }) + }; + comparison(e.op, &e.lhs, &e.rhs).or_else(|| comparison(e.op.commute()?, &e.rhs, &e.lhs)) + } + Expr::Like(e) => resolve_relation_path_predicate(cx, &e.expr, |target_lhs| { + stmt::ExprLike { + expr: Box::new(target_lhs), + pattern: e.pattern.clone(), + escape: e.escape, + case_insensitive: e.case_insensitive, + } + .into() + }), + Expr::IsVariant(e) => resolve_relation_path_predicate(cx, &e.expr, |target_lhs| { + Expr::is_variant(target_lhs, e.variant) + }), + _ => None, + } +} + +/// Shared core of [`resolve_relation_predicate`]. +/// +/// `subject` is the side of the predicate that walks a relation — the +/// `profile.user.name` in `profile.user.name = 'alice'`. Its steps up to the +/// relation name the `user` relation on `Profile` (through an embedded type, +/// when the relation sits in one); the remaining steps index into `User`. +/// This re-roots the remainder at the target model (so it reads `User.name`) +/// and hands it to `make_filter`, the only difference between predicate +/// forms. +/// +/// Returns `None` when `subject` does not walk into a relation's target — +/// including a path that ends *at* the relation, which is a key comparison +/// rather than a predicate on the target. +fn resolve_relation_path_predicate<'a>( + cx: &ExprContext<'a>, + subject: &stmt::Expr, + make_filter: impl FnOnce(stmt::Expr) -> stmt::Expr, +) -> Option<(RelationPath<'a>, stmt::Expr)> { + let path = resolve_relation_path(cx, subject)?; + + let filter = make_filter(path.target_expr.clone()?); + + Some((path, filter)) +} + +/// Lift `filter`, a predicate on the target model of `path`'s relation, +/// into a foreign-key `IN` subquery on that relation. +fn lift_relation_predicate( + cx: &ExprContext, + path: &RelationPath<'_>, + filter: stmt::Expr, +) -> Option { + let subquery = stmt::Query::new_select(stmt::Source::from(path.target), filter); + lift_relation_in_subquery(cx, &path.relation, &subquery) +} + +/// Build a foreign-key `IN` subquery for one direct relation edge. +/// +/// `lhs` references key fields on the current model. `returning` references +/// the matching key fields on `target`, which is also the subquery source. +fn lift_fk_in_subquery( + target: ModelId, + lhs: stmt::Expr, + returning: stmt::Expr, + query: &stmt::Query, +) -> Option { + if target != query.body.as_select_unwrap().source.model_id_unwrap() { + return None; + } + + let mut subquery = query.clone(); + subquery.body.as_select_mut_unwrap().returning = stmt::Returning::Project(returning); + + Some(stmt::Expr::in_subquery(lhs, subquery)) +} + +/// BelongsTo branch: try to lift the subquery's filter into direct FK +/// comparisons. When the filter references only FK-mapped fields, return +/// the AND of per-FK equalities. Otherwise, fall back to an IN subquery +/// on the foreign-key column(s) — a tuple-form IN for composite FKs. +/// +/// Returns `None` when the subquery does not target the BelongsTo's +/// target model. +fn lift_belongs_to_in_subquery( + cx: &ExprContext, + belongs_to: &BelongsTo, + query: &stmt::Query, +) -> Option { + if belongs_to.target != query.body.as_select_unwrap().source.model_id_unwrap() { + return None; + } + + let select = query.body.as_select_unwrap(); + + let mut lift = LiftBelongsTo { + cx: cx.scope(&select.source), + belongs_to, + fk_field_matches: vec![false; belongs_to.foreign_key.fields.len()], + operands: vec![], + fail: false, + }; + + lift.visit_filter(&select.filter); + + // Fall back to the IN-subquery form whenever we couldn't account for + // every FK column with a direct equality from the filter. This covers + // both `fail=true` (a binary op referenced something that isn't on the + // FK) and the case where the filter contained no liftable binary ops at + // all — e.g. when it's a nested `InSubquery` that the LiftBelongsTo + // visitor deliberately skips (see `visit_expr_in_subquery`). + let all_fks_matched = lift.fk_field_matches.iter().all(|m| *m); + + if lift.fail || !all_fks_matched { + lift_fk_in_subquery( + belongs_to.target, + super::key_field_refs(0, belongs_to.foreign_key.fields.iter().map(|fk| fk.source)), + super::key_field_refs(0, belongs_to.foreign_key.fields.iter().map(|fk| fk.target)), + query, + ) + } else { + Some(if lift.operands.len() == 1 { + lift.operands.into_iter().next().unwrap() + } else { + stmt::ExprAnd { + operands: lift.operands, + } + .into() + }) + } +} + +/// HasOne/HasMany branch: rewrite to a foreign-key IN subquery against +/// the related table. Single-column FKs produce a scalar IN; composite +/// FKs produce a tuple-form IN that the SQL serializer renders as +/// `(a, b) IN (SELECT a, b FROM ...)`. +fn lift_has_n_in_subquery( + target: ModelId, + pair: &BelongsTo, + query: &stmt::Query, +) -> Option { + lift_fk_in_subquery( + target, + super::key_field_refs(0, pair.foreign_key.fields.iter().map(|fk| fk.target)), + super::key_field_refs(0, pair.foreign_key.fields.iter().map(|fk| fk.source)), + query, + ) +} + +impl Visit for LiftBelongsTo<'_> { + fn visit_expr_in_subquery(&mut self, _i: &stmt::ExprInSubquery) { + // Stop the walk at a nested IN-subquery boundary. Field references + // inside resolve in the nested query's own scope, but the visitor's + // `cx` is scoped to the current (outer) subquery's source — recursing + // would misresolve inner refs as outer-model fields and produce + // bogus FK matches. The main LiftInSubquery walker handles the + // nested subquery on its own pass, with the correct scope. + } + + fn visit_expr_binary_op(&mut self, i: &stmt::ExprBinaryOp) { + match (&*i.lhs, &*i.rhs) { + (stmt::Expr::Reference(expr_reference), other) + | (other, stmt::Expr::Reference(expr_reference)) => { + assert!(i.op.is_eq() || i.op.is_ne()); + + if i.op.is_eq() || i.op.is_ne() { + let field = self + .cx + .resolve_expr_reference(expr_reference) + .as_field_unwrap(); + + self.lift_fk_constraint(field.id, i.op, other); + } else { + self.fail = true; + } + } + // Constraints we can't lift to a direct FK comparison (e.g. a + // projection through an embedded field). Bail to the IN-subquery + // form so the filter is preserved verbatim; without this, the + // empty `operands` list silently produced an empty AND (= true) + // and the subquery returned every row. + _ => { + self.fail = true; + } + } + } +} + +impl LiftBelongsTo<'_> { + fn lift_fk_constraint(&mut self, field: FieldId, op: stmt::BinaryOp, expr: &stmt::Expr) { + for (i, fk_field) in self.belongs_to.foreign_key.fields.iter().enumerate() { + if fk_field.target == field { + if self.fk_field_matches[i] { + todo!("not handled"); + } + + self.operands.push(stmt::Expr::binary_op( + stmt::Expr::ref_self_field(fk_field.source), + op, + expr.clone(), + )); + self.fk_field_matches[i] = true; + + return; + } + } + + self.fail = true; + } +} diff --git a/packages/toasty/src/engine/lower/lift_update_query.rs b/packages/toasty/src/engine/lower/lift_update_query.rs new file mode 100644 index 00000000000..14891cc3e99 --- /dev/null +++ b/packages/toasty/src/engine/lower/lift_update_query.rs @@ -0,0 +1,50 @@ +//! Pre-lowering rewrite that converts `Update { target: Query(q), .. }` +//! into `Update { target: Model(m), filter: filter ∧ q.filter, .. }`. +//! +//! [`LiftUpdateQuery`] runs as a whole-statement pre-pass before the +//! main lowering walk. The visitor overrides `visit_stmt_update_mut` +//! to lift the inner select's source model out as the new +//! `UpdateTarget::Model` and merge its filter into the outer update's +//! filter. +//! +//! `UpdateTarget::Query` is the app-shaped form of "update the rows +//! produced by this query". The db-level form is always +//! `UpdateTarget::Table`; the lowering walk's +//! `LowerStatement::visit_update_target_mut` panics on +//! `UpdateTarget::Query`. The lift fires before lowering so the walk +//! only ever sees `UpdateTarget::Model`, which it converts to +//! `UpdateTarget::Table`. + +use toasty_core::stmt::{self, VisitMut}; + +/// Pre-lowering pass that lifts `UpdateTarget::Query` into +/// `UpdateTarget::Model` with the inner query's filter merged onto the +/// outer update. +pub(super) struct LiftUpdateQuery; + +impl LiftUpdateQuery { + pub(super) fn new() -> Self { + Self + } + + pub(super) fn rewrite(&mut self, stmt: &mut stmt::Statement) { + self.visit_mut(stmt); + } +} + +impl VisitMut for LiftUpdateQuery { + fn visit_stmt_update_mut(&mut self, stmt: &mut stmt::Update) { + if let stmt::UpdateTarget::Query(query) = &mut stmt.target { + let stmt::ExprSet::Select(select) = &mut query.body else { + todo!() + }; + + assert!(select.returning.is_model()); + + stmt.filter.add_filter(select.filter.take()); + stmt.target = stmt::UpdateTarget::Model(select.source.model_id_unwrap()); + } + + stmt::visit_mut::visit_stmt_update_mut(self, stmt); + } +} diff --git a/packages/toasty/src/engine/lower/paginate.rs b/packages/toasty/src/engine/lower/paginate.rs new file mode 100644 index 00000000000..0f8b1dd80a4 --- /dev/null +++ b/packages/toasty/src/engine/lower/paginate.rs @@ -0,0 +1,109 @@ +use toasty_core::stmt; + +use super::LowerStatement; + +impl LowerStatement<'_, '_> { + pub(super) fn rewrite_offset_after_as_filter(&self, stmt: &mut stmt::Query) { + // Only do this for SQL statements + if !self.capability().sql() { + return; + } + + let Some(order_by) = &mut stmt.order_by else { + return; + }; + + let Some(stmt::Limit::Cursor(cursor)) = &mut stmt.limit else { + return; + }; + + let Some(after) = cursor.after.take() else { + return; + }; + + let stmt::ExprSet::Select(body) = &mut stmt.body else { + todo!("stmt={stmt:#?}"); + }; + + match after { + stmt::Expr::Value(stmt::Value::Record(value)) => { + // Rows strictly beyond the cursor in lexicographic order: + // `e0 > v0 OR (e0 = v0 AND e1 > v1) OR ...`, flipping each + // comparison for descending columns. A plain conjunction of + // per-column comparisons would skip rows that tie the cursor + // on a leading column. + let terms = value + .fields + .iter() + .enumerate() + .map(|(index, field_value)| { + let mut operands: Vec<_> = order_by.exprs[..index] + .iter() + .zip(value.fields.iter()) + .map(|(order_by, eq_value)| { + self.rewrite_cursor_field_as_equality(order_by, eq_value.clone()) + }) + .collect(); + + operands.push(self.rewrite_offset_after_field_as_filter( + &order_by.exprs[index], + field_value.clone(), + )); + + stmt::Expr::and_from_vec(operands) + }) + .collect(); + + body.filter.add_filter(stmt::Expr::or_from_vec(terms)); + } + stmt::Expr::Value(value) => { + let expr = self.rewrite_offset_after_field_as_filter(&order_by.exprs[0], value); + body.filter.add_filter(expr); + } + _ => todo!(), + } + } + + fn rewrite_offset_after_field_as_filter( + &self, + order_by: &stmt::OrderByExpr, + value: stmt::Value, + ) -> stmt::Expr { + let nulls_first = match order_by.order { + Some(stmt::Direction::Desc) => !self.capability().sql_nulls_first_on_asc, + _ => self.capability().sql_nulls_first_on_asc, + }; + + if value.is_null() { + return if nulls_first { + stmt::Expr::is_not_null(order_by.expr.clone()) + } else { + false.into() + }; + } + + let op = match order_by.order { + Some(stmt::Direction::Desc) => stmt::BinaryOp::Lt, + _ => stmt::BinaryOp::Gt, + }; + let comparison = stmt::Expr::binary_op(order_by.expr.clone(), op, value); + + if nulls_first { + comparison + } else { + stmt::Expr::or(comparison, stmt::Expr::is_null(order_by.expr.clone())) + } + } + + fn rewrite_cursor_field_as_equality( + &self, + order_by: &stmt::OrderByExpr, + value: stmt::Value, + ) -> stmt::Expr { + if value.is_null() { + stmt::Expr::is_null(order_by.expr.clone()) + } else { + stmt::Expr::eq(order_by.expr.clone(), value) + } + } +} diff --git a/packages/toasty/src/engine/lower/relation.rs b/packages/toasty/src/engine/lower/relation.rs new file mode 100644 index 00000000000..686a4bbeac3 --- /dev/null +++ b/packages/toasty/src/engine/lower/relation.rs @@ -0,0 +1,1217 @@ +use indexmap::IndexSet; +use toasty_core::{ + schema::app::{self, Field, FieldId, FieldTy}, + stmt, +}; + +use crate::engine::{hir, lower::LowerStatement}; +use crate::schema::lazy_slot; + +#[derive(Debug)] +enum Mutation { + /// An existing value is being associated with the relation source. + Associate { + // Target relation expression + expr: stmt::Expr, + + // When true, any previous values associated with the source relation + // need to be cleared. + exclusive: bool, + }, + + /// Disassociate an existing relation + Disassociate { + /// Target relation expression + expr: stmt::Expr, + }, + + /// Disassociate any target relations + DisassociateAll { + // True when disasociating before deleting a record. + delete: bool, + }, +} + +trait RelationSource: std::fmt::Debug { + /// Return a query representing the source selection + fn selection(&self, nesting: usize) -> stmt::Query; + + /// Update a source field expression + fn set_source_field(&mut self, field: FieldId, expr: stmt::Expr); + + /// Update a returning field expression + fn set_returning_field(&mut self, field: &Field, expr: stmt::Expr); + + /// Whether the source might produce zero rows. When true, relation + /// mutations must be wrapped in a conditional to avoid FK updates when + /// the source filter doesn't match. + fn needs_existence_check(&self) -> bool; +} + +/// A record whose relation slots can be consumed during planning. +trait RelationRecord: RelationSource { + fn take_source_field(&mut self, field: FieldId) -> stmt::Expr; +} + +#[derive(Debug)] +struct EmbeddedRelationSource<'a> { + fields: &'a [Field], + offset: usize, + row: &'a mut stmt::Expr, +} + +#[derive(Debug)] +struct InsertRelationSource<'a> { + model: &'a app::ModelRoot, + row: &'a mut stmt::Expr, + /// The index in stmt::Returning that represents the row + index: usize, + returning: &'a mut Option, +} + +#[derive(Debug)] +struct UpdateRelationSource<'a> { + model: &'a app::ModelRoot, + filter: &'a stmt::Filter, + assignments: &'a mut stmt::Assignments, + returning: &'a mut Option, + returning_changed: bool, +} + +impl LowerStatement<'_, '_> { + pub(super) fn plan_stmt_delete_relations(&mut self, mut stmt: &stmt::Delete) { + // Cascading deletes are only handled at the application level + let Some(model) = self.expr_cx.target().as_model() else { + return; + }; + + // Handle any cascading deletes + for field in model.fields.iter() { + self.plan_mut_relation_field( + field, + Mutation::DisassociateAll { delete: true }, + &mut stmt, + ); + } + } + + pub(super) fn plan_stmt_insert_relations( + &mut self, + row: &mut stmt::Expr, + returning: &mut Option, + index: usize, + ) { + let model = self.expr_cx.target().as_model_unwrap(); + + self.plan_record_relations( + &model.fields, + &mut InsertRelationSource { + model, + row, + returning, + index, + }, + ); + } + + /// Plan root and embedded record fields through the same relation dispatcher. + fn plan_record_relations(&mut self, fields: &[Field], source: &mut S) { + for field in fields { + if let FieldTy::Embedded(embedded) = &field.ty { + let mut value = source.take_source_field(field.id); + self.plan_embedded_record_relations(embedded.target, &mut value); + source.set_source_field(field.id, value); + } else if field.is_relation() { + let expr = source.take_source_field(field.id); + if expr.is_value_null() || expr.is_default() { + if !field.nullable && field.ty.is_has_one() { + panic!( + "Insert missing non-nullable field; model={}; name={:#?}; ty={:#?}; expr={:#?}", + self.schema() + .app + .model(field.id.model) + .name() + .upper_camel_case(), + field.name, + field.ty, + expr + ); + } + continue; + } + self.plan_mut_relation_field( + field, + Mutation::Associate { + expr, + exclusive: field.ty.is_belongs_to(), + }, + source, + ); + } + } + } + + /// Resolve the fields of an embedded value, including the active variant's + /// local record positions, before its columns are expanded. + fn plan_embedded_record_relations(&mut self, model: app::ModelId, expr: &mut stmt::Expr) { + if let stmt::Expr::Cast(cast) = expr + && cast.from.is_none() + && cast.ty == stmt::Type::Model(model) + && cast.expr.record_len().is_some() + { + *expr = cast.expr.take(); + } + let Some(len) = expr.record_len() else { + return; + }; + let (fields, offset): (&[Field], _) = match self.schema().app.model(model) { + app::Model::Root(_) => return, + app::Model::EmbeddedStruct(model) => (&model.fields, 0), + app::Model::EmbeddedEnum(model) => { + let Some(discriminant) = expr.entry(0) else { + return; + }; + let stmt::Expr::Value(discriminant) = discriminant.to_expr() else { + return; + }; + let Some(variant) = model + .variants + .iter() + .position(|v| v.discriminant == discriminant) + else { + return; + }; + (model.variant_fields(variant), 1) + } + }; + if len != fields.len() + offset { + return; + } + self.plan_record_relations( + fields, + &mut EmbeddedRelationSource { + fields, + offset, + row: expr, + }, + ); + } + + /// Typed embedded values also occur outside writes, for example in filters. + /// Keep their relation semantics identical to values assigned to a field. + pub(super) fn plan_typed_record_relations(&mut self, expr: &mut stmt::Expr) { + let stmt::Expr::Cast(cast) = expr else { + return; + }; + let stmt::Type::Model(model) = cast.ty else { + return; + }; + let schema_model = self.schema().app.model(model); + if cast.from.is_none() + && !schema_model.is_root() + && schema_model + .fields() + .iter() + .any(|field| field.ty.is_belongs_to()) + && cast.expr.record_len().is_some() + { + self.plan_embedded_record_relations(model, expr); + } + } + + fn plan_embedded_assignment(&mut self, model: app::ModelId, assignment: &mut stmt::Assignment) { + match assignment { + stmt::Assignment::Set(expr) => self.plan_embedded_record_relations(model, expr), + stmt::Assignment::Batch(entries) => { + for entry in entries { + self.plan_embedded_assignment(model, entry); + } + } + _ => (), + } + } + + pub(super) fn plan_stmt_update_relations( + &mut self, + assignments: &mut stmt::Assignments, + filter: &stmt::Filter, + returning: &mut Option, + returning_changed: bool, + ) { + let model = self.expr_cx.target().as_model_unwrap(); + + for (projection, assignment) in assignments.iter_mut() { + let schema = &self.schema().app; + if let Some(app::Resolved::Field(field)) = + schema.resolve(schema.model(model.id), projection) + && let FieldTy::Embedded(embedded) = &field.ty + { + self.plan_embedded_assignment(embedded.target, assignment); + } + } + + for (i, field) in model.fields.iter().enumerate() { + if !field.is_relation() { + continue; + } + + let Some(assignment) = assignments.take(&[i]) else { + continue; + }; + + // A `Batch` carries a sequence of discrete ops on the same + // projection, built by the `stmt::apply` surface API. Dispatch + // each entry as its own `Mutation`; `Insert` entries are first + // folded into one multi-row INSERT because only one Insert can + // own the relation's returning slot per update (a second + // Insert's `set_returning_slot` call would clobber the first). + // `Remove` entries pass through individually — they don't write + // to the returning slot. + let entries: Vec = match assignment { + stmt::Assignment::Batch(mut entries) => { + assert!(field.ty.is_has_many()); + flatten_relation_batch(&mut entries); + entries + } + other => vec![other], + }; + + // Execute the batch's entries in order. Each entry's statements + // depend on the previous entry's, so operations that touch the + // same target are sequenced deterministically rather than racing + // in the dependency graph. This matters when one op constrains a + // later one — associating then dissociating the same existing row, + // a unique-value swap that must delete before it re-inserts, or a + // leading `set`'s exclusive clear that must not wipe freshly + // inserted rows. (`flatten_relation_batch` has already placed the + // merged create-new Insert last.) + let mut source = UpdateRelationSource { + model, + filter, + assignments: &mut *assignments, + returning: &mut *returning, + returning_changed, + }; + + let mut prev_deps: IndexSet = IndexSet::new(); + for entry in entries { + let emitted = self.collect_dependencies(|lower| { + lower.with_dependencies(prev_deps.clone(), |lower| { + lower.plan_mut_relation_field( + field, + relation_mutation(field, entry), + &mut source, + ); + }); + }); + + // Chain the next entry onto this one's statements. An entry + // that emits nothing leaves the chain pointing at the last + // real statement. + if !emitted.is_empty() { + prev_deps = emitted; + } + } + } + } + + fn plan_mut_relation_field( + &mut self, + field: &app::Field, + op: Mutation, + source: &mut dyn RelationSource, + ) { + match &field.ty { + FieldTy::Has(rel) if rel.is_one() => { + self.relation_step(field, |lower| lower.plan_mut_has_one(field, op, source)); + } + FieldTy::Has(rel) if rel.is_many() => { + self.relation_step(field, |lower| lower.plan_mut_has_many(field, op, source)); + } + FieldTy::BelongsTo(_) => { + self.plan_mut_belongs_to(field, op, source); + } + _ => (), + } + } + + fn plan_mut_has_many(&mut self, field: &Field, op: Mutation, source: &mut dyn RelationSource) { + let has_many = field.ty.as_has_many_unwrap(); + let pair = has_many.pair_id; + let pair = self.field(pair); + + self.plan_mut_has_n(field, pair, op, source); + } + + fn plan_mut_has_one(&mut self, field: &Field, op: Mutation, source: &mut dyn RelationSource) { + let has_one = field.ty.as_has_one_unwrap(); + let pair = has_one.pair_id; + let pair = self.field(pair); + + self.plan_mut_has_n(field, pair, op, source); + } + + fn plan_mut_has_n( + &mut self, + field: &Field, + pair: &Field, + op: Mutation, + source: &mut dyn RelationSource, + ) { + match op { + Mutation::DisassociateAll { .. } => { + self.plan_mut_has_n_disassociate_all(pair, source); + } + Mutation::Associate { expr, exclusive } => { + let deps = self.collect_dependencies(|lower| { + if exclusive { + lower.plan_mut_has_n_disassociate_all(pair, source); + } + }); + + self.with_dependencies(deps, |lower| { + lower.plan_mut_has_n_associate_expr(field, pair, expr, source); + }); + } + Mutation::Disassociate { expr } => { + debug_assert!(field.ty.is_has_many()); + self.plan_mut_has_many_disassociate_expr(field, pair, expr, source); + } + } + } + + fn plan_mut_has_n_associate_expr( + &mut self, + field: &Field, + pair: &Field, + expr: stmt::Expr, + source: &mut dyn RelationSource, + ) { + match expr { + stmt::Expr::List(expr_list) => { + for expr in expr_list.items { + self.plan_mut_has_n_associate_expr(field, pair, expr, source); + } + } + stmt::Expr::Stmt(expr_stmt) => { + self.plan_mut_has_n_associate_stmt(field, pair, *expr_stmt.stmt, source); + } + stmt::Expr::Value(stmt::Value::List(value_list)) => { + for value in value_list { + self.plan_mut_has_n_associate_value(pair, value, source); + } + } + stmt::Expr::Value(value) => { + self.plan_mut_has_n_associate_value(pair, value, source); + } + _ => todo!("field={field:#?}, expr={expr:#?}"), + } + } + + fn plan_mut_has_n_associate_value( + &mut self, + pair: &Field, + value: stmt::Value, + source: &mut dyn RelationSource, + ) { + assert!(!value.is_list()); + + let mut stmt = stmt::Query::new_select( + pair.id.model, + stmt::Expr::eq(stmt::Expr::ref_ancestor_model(0), value), + ) + .update(); + + stmt.assignments + .set(pair.id, stmt::Expr::stmt(source.selection(2))); + + // Needed for the existence check. Only update *if* the relation source + // actually exists to be updated. + if source.needs_existence_check() { + stmt.filter.set(stmt::Expr::exists({ + let mut query = source.selection(2); + let stmt::ExprSet::Select(select) = &mut query.body else { + todo!() + }; + select.returning = stmt::Returning::Project(stmt::Expr::record([1])); + query + })); + } + + self.new_dependency(stmt); + } + + fn plan_mut_has_many_disassociate_expr( + &mut self, + field: &Field, + pair: &Field, + expr: stmt::Expr, + source: &dyn RelationSource, + ) { + match expr { + stmt::Expr::Value(value) => { + self.plan_mut_has_many_disassociate_value(pair, value, source) + } + _ => todo!("field={field:#?}; expr={expr:#?}"), + } + } + + fn plan_mut_has_many_disassociate_value( + &mut self, + pair: &Field, + value: stmt::Value, + source: &dyn RelationSource, + ) { + let selection = stmt::Query::new_select( + pair.id.model, + stmt::Expr::eq(stmt::Expr::ref_ancestor_model(0), value), + ); + + if pair.nullable { + let mut stmt = selection.update(); + + stmt.condition = stmt::Condition::new(self.relation_pair_filter(pair.id, source)); + stmt.assignments.set(pair, stmt::Value::Null); + self.new_dependency(stmt); + } else { + self.new_dependency(selection.delete()); + } + } + + fn plan_mut_has_n_disassociate_all(&mut self, pair: &Field, source: &dyn RelationSource) { + let query = self.relation_pair_scope(pair.id, source); + + if pair.nullable { + let mut update = query.update(); + update.assignments.set(pair.id, stmt::Value::Null); + + self.new_dependency(update); + } else { + self.new_dependency(query.delete()); + } + } + + fn plan_mut_has_n_associate_stmt( + &mut self, + field: &Field, + pair: &Field, + stmt: stmt::Statement, + source: &mut dyn RelationSource, + ) { + match stmt { + stmt::Statement::Insert(stmt) => { + self.plan_mut_has_n_associate_insert(field, pair, stmt, source) + } + _ => todo!("stmt={:#?}", stmt), + } + } + + fn plan_mut_has_n_associate_insert( + &mut self, + _field: &Field, + pair: &Field, + mut stmt: stmt::Insert, + source: &mut dyn RelationSource, + ) { + debug_assert_eq!(stmt.target.model_id_unwrap(), pair.id.model); + debug_assert!(stmt.target.is_model()); + + stmt.target = self.relation_pair_scope(pair.id, source).into(); + + // has_many fields represent collections and must return lists for type + // consistency, even when inserting a single record. The insert may arrive + // with `single = true` (semantically inserting one record into the set), + // but we set it to false to ensure the result is wrapped in a list. + if _field.ty.is_has_many() { + stmt.source.single = false; + } + + // Run the canonical pipeline on the synthesized child insert and + // stitch it onto the parent as an `Expr::Arg`. + let arg = self.lower_sub_stmt(stmt::Statement::Insert(stmt)); + source.set_returning_field(_field, arg); + } + + fn plan_mut_belongs_to( + &mut self, + field: &Field, + op: Mutation, + source: &mut dyn RelationSource, + ) { + // BelongsTo doesn't create child inserts, so `then` is unused + match op { + Mutation::Associate { expr, .. } => { + self.plan_mut_belongs_to_associate(field, expr, source) + } + Mutation::DisassociateAll { delete } => { + self.plan_mut_belongs_to_disassociate(field, delete, source); + } + Mutation::Disassociate { .. } => { + todo!("is this needed?") + } + } + } + + fn plan_mut_belongs_to_disassociate( + &mut self, + field: &Field, + delete: bool, + source: &mut dyn RelationSource, + ) { + if !field.nullable && !delete { + todo!("invalid statement. handle this case"); + } + + let belongs_to = field.ty.as_belongs_to_unwrap(); + + if let Some(pair_id) = belongs_to.pair { + let pair = self.field(pair_id); + + if pair.ty.is_has_one() && !pair.nullable { + self.relation_step(field, |planner| { + let delete = planner.relation_pair_scope(pair.id, source).delete(); + planner.new_dependency(delete); + }); + } + } + + if !delete { + for fk_field in &belongs_to.foreign_key.fields { + source.set_source_field(fk_field.source, stmt::Expr::null()); + } + } + } + + fn plan_mut_belongs_to_associate( + &mut self, + field: &Field, + expr: stmt::Expr, + source: &mut dyn RelationSource, + ) { + let expr = relation_key_expr(&field.ty.as_belongs_to_unwrap().foreign_key, expr); + let dependencies = self.collect_dependencies(|lower| { + if let Some(pair_id) = field.pair() + && lower.field(pair_id).ty.is_has_one() + { + // Disassociate an existing HasOne. This handles the case where + // if the HasOne association is required (i.e. *not* `Option`), + // then the record gets deleted. + lower.plan_mut_belongs_to_disassociate(field, false, source); + + // This handles disassociating any *other* instances of the current + // model that already are associated with the target model being + // passed it. This is necessary because of the 1-1 relation + // mapping. + if field.nullable { + lower.relation_step(field, |planner| { + assert!(expr.is_value()); + + let scope = stmt::Query::new_select( + field.id.model, + stmt::Expr::eq(stmt::Expr::ref_self_field(field), expr.clone()), + ); + + if field.nullable { + let mut stmt = scope.update(); + stmt.assignments.set(field.id, stmt::Value::Null); + planner.new_dependency(stmt); + } else { + todo!(); + } + }); + } else { + todo!() + } + } + }); + + self.with_dependencies(dependencies, |lower| match expr { + expr if expr.is_value() || expr.is_expr_reference() => { + assert!(!expr.is_value_null()); + + lower.set_relation_field(field, expr, source); + } + // Composite FK: a record whose fields are each a scalar value or + // reference — produced by the tuple `IntoExpr` impl when the + // target model has a composite primary key. `set_relation_field` + // destructures it via `record_len()` / `into_record_items()` and + // assigns each component to the matching FK source column. + stmt::Expr::Record(record) + if record + .fields + .iter() + .all(|f| f.is_value() || f.is_expr_reference()) => + { + lower.set_relation_field(field, stmt::Expr::Record(record), source); + } + stmt::Expr::Stmt(expr_stmt) => { + lower.plan_mut_belongs_to_associate_stmt(field, *expr_stmt.stmt, source); + } + _ => todo!("field={field:#?}; expr={expr:#?}"), + }); + } + + fn plan_mut_belongs_to_associate_stmt( + &mut self, + field: &Field, + stmt: stmt::Statement, + source: &mut dyn RelationSource, + ) { + let belongs_to = field.ty.as_belongs_to_unwrap(); + + match stmt { + stmt::Statement::Insert(mut insert) => { + debug_assert!(insert.source.single); + + if let stmt::ExprSet::Values(values) = &insert.source.body { + assert_eq!(1, values.rows.len()); + } + + // Only returning that makes sense here as that is the type that + // "belongs" in this field. We translate it to the key to set + // the FK fields in the source model. + assert!(matches!( + insert.returning, + Some(stmt::Returning::Model { .. }) + )); + + // Previous value of returning does nothing in this + // context + insert.returning = Some(stmt::Returning::Project(stmt::Expr::record( + belongs_to + .foreign_key + .fields + .iter() + .map(|fk_field| stmt::Expr::ref_self_field(fk_field.target)), + ))); + + let target_id = self.new_dependency(insert); + let stmt_info = &self.state.hir[target_id]; + + let returning = stmt_info.stmt.as_ref().unwrap().returning().expect("bug"); + + let expr = match returning { + stmt::Returning::Expr(expr) if expr.is_const() => { + // The insert returns a tuple of referenced fields, even + // when the relation has just one key field. + if belongs_to.foreign_key.fields.len() == 1 { + expr.entry(0).unwrap().to_expr() + } else { + expr.clone() + } + } + _ => { + // Make sure the source statement returns a single record + debug_assert!(match &**stmt_info.stmt.as_ref().unwrap() { + stmt::Statement::Insert(i) => i.source.single, + stmt::Statement::Query(stmt) => stmt.single, + stmt => todo!("stmt={stmt:#?}"), + }); + // The result dependency is needed to get the foreign key. + self.new_dependency_arg(self.scope_stmt_id(), target_id) + } + }; + + self.set_relation_field(field, expr, source); + } + stmt::Statement::Query(query) => { + // Try to extract the FK from the select without performing the query + let fields: Vec<_> = belongs_to + .foreign_key + .fields + .iter() + .map(|fk_field| fk_field.target) + .collect(); + + let Some(expr) = self.extract_key_expr(&fields, &query) else { + todo!("belongs_to={:#?}; stmt={:#?}", belongs_to, query); + }; + + self.plan_mut_belongs_to_associate(field, expr, source); + } + _ => todo!("stmt={:#?}", stmt), + } + } + + /// Extract constant key values from a subquery filter so the subquery can + /// be eliminated. + /// + /// Given a query like `SELECT User WHERE id = 123 AND name = "foo"` and + /// key fields `[id]`, this extracts `123`. For composite keys + /// `[id, org_id]` it extracts a record `(123, 456)`. + /// + /// Recursively walks `And` nodes collecting equality constraints for each + /// key field. Does not descend into `Or` or other non-conjunctive nodes. + fn extract_key_expr(&self, key: &[FieldId], query: &stmt::Query) -> Option { + let cx = self.expr_cx.scope(query); + + let stmt::ExprSet::Select(select) = &query.body else { + return None; + }; + + let mut values: Vec> = vec![None; key.len()]; + Self::collect_key_constraints(&cx, key, select.filter.as_expr(), &mut values); + + // All key fields must have been matched + let values: Vec = values.into_iter().collect::>>()?; + + if values.len() == 1 { + Some(values.into_iter().next().unwrap()) + } else { + Some(stmt::Expr::record(values)) + } + } + + /// Recursively walk a filter expression collecting equality constraints + /// for key fields. Only descends into conjunctive (`And`) nodes. + fn collect_key_constraints( + cx: &stmt::ExprContext, + key: &[FieldId], + filter: &stmt::Expr, + values: &mut [Option], + ) { + match filter { + stmt::Expr::And(and) => { + for operand in &and.operands { + Self::collect_key_constraints(cx, key, operand, values); + } + } + stmt::Expr::BinaryOp(binary) => { + Self::try_match_key_eq(cx, key, binary, values); + } + // Or, Not, etc. — don't walk into these; they don't guarantee + // a single constant value for the key field. + _ => {} + } + } + + /// If `binary` is an equality involving one of the key fields, record + /// the matched value. + fn try_match_key_eq( + cx: &stmt::ExprContext, + key: &[FieldId], + binary: &stmt::ExprBinaryOp, + values: &mut [Option], + ) { + if !binary.op.is_eq() { + return; + } + + for (i, key_field) in key.iter().enumerate() { + if values[i].is_some() { + continue; + } + if let Some(expr) = Self::extract_eq_value(cx, *key_field, binary) { + values[i] = Some(expr); + return; + } + } + } + + /// Extract the value for `key_field` from a single equality expression. + fn extract_eq_value( + cx: &stmt::ExprContext, + key_field: FieldId, + binary: &stmt::ExprBinaryOp, + ) -> Option { + debug_assert!(binary.op.is_eq()); + + match (&*binary.lhs, &*binary.rhs) { + // Inner field ref (nesting=0) matched against outer field ref (nesting>0) + ( + stmt::Expr::Reference(inner @ stmt::ExprReference::Field { nesting: 0, .. }), + stmt::Expr::Reference(outer @ stmt::ExprReference::Field { nesting, .. }), + ) + | ( + stmt::Expr::Reference(outer @ stmt::ExprReference::Field { nesting, .. }), + stmt::Expr::Reference(inner @ stmt::ExprReference::Field { nesting: 0, .. }), + ) if *nesting > 0 => { + let field_ref = cx.resolve_expr_reference(inner).as_field_unwrap(); + if key_field == field_ref.id { + let mut ret = *outer; + let stmt::ExprReference::Field { nesting, .. } = &mut ret else { + panic!() + }; + debug_assert!(*nesting > 0); + *nesting -= 1; + Some(ret.into()) + } else { + None + } + } + // Both nesting=0 refs — not handled + (stmt::Expr::Reference(_), stmt::Expr::Reference(_)) => None, + // Field ref matched against a constant value + (stmt::Expr::Reference(expr_ref), other) | (other, stmt::Expr::Reference(expr_ref)) => { + let field_ref = cx.resolve_expr_reference(expr_ref).as_field_unwrap(); + if key_field == field_ref.id { + if let stmt::Expr::Value(value) = other { + Some(value.clone().into()) + } else { + None + } + } else { + None + } + } + _ => None, + } + } + + /// Translate a source model scope to a target model scope for a has_one + /// relation. + fn relation_pair_scope(&self, pair: FieldId, source: &dyn RelationSource) -> stmt::Query { + stmt::Query::new_select(pair.model, self.relation_pair_filter(pair, source)) + } + + fn relation_pair_filter(&self, pair: FieldId, source: &dyn RelationSource) -> stmt::Expr { + stmt::Expr::in_subquery(stmt::Expr::ref_self_field(pair), source.selection(1)) + } + + fn relation_step(&mut self, field: &Field, f: impl FnOnce(&mut LowerStatement)) { + if let Some(pair) = field.pair() + && self.state.relations.last().copied() == Some(pair) + { + return; + } + + self.state.relations.push(field.id); + f(self); + self.state.relations.pop(); + } + + fn set_relation_field(&self, field: &Field, expr: stmt::Expr, source: &mut dyn RelationSource) { + let app::FieldTy::BelongsTo(belongs_to) = &field.ty else { + todo!("field={field:#?}") + }; + + assign_belongs_to_key(&belongs_to.foreign_key, expr, |field, value| { + source.set_source_field(field, value); + }); + } +} + +impl RelationSource for &stmt::Delete { + fn selection(&self, nesting: usize) -> stmt::Query { + assert_eq!(nesting, 1, "TODO"); + stmt::Delete::selection(self) + } + + fn set_source_field(&mut self, _field: FieldId, _expr: stmt::Expr) { + unimplemented!("delete statements do not need to update field values"); + } + + fn set_returning_field(&mut self, _field: &Field, _expr: stmt::Expr) { + unimplemented!("delete statements do not need to update field values"); + } + + fn needs_existence_check(&self) -> bool { + false + } +} + +impl RelationSource for UpdateRelationSource<'_> { + fn selection(&self, _nesting: usize) -> stmt::Query { + // In this context, the nesting does not matter. The filter entirely + // references the returned query. + stmt::Query::new_select(self.model, self.filter.clone()) + } + + fn set_source_field(&mut self, field: FieldId, expr: stmt::Expr) { + self.assignments.set(field, expr); + } + + fn set_returning_field(&mut self, field: &Field, expr: stmt::Expr) { + debug_assert!(self.returning_changed, "TODO"); + + let Some(stmt::Returning::Project(stmt::Expr::Cast(expr_cast))) = self.returning else { + todo!("UpdateRelationSource={self:#?}") + }; + + let stmt::Type::SparseRecord(path_field_set) = &mut expr_cast.ty else { + todo!("expr={expr:#?}") + }; + + let position = path_field_set + .iter() + .position(|field_id| field_id == field.id.index) + .unwrap(); + + let stmt::Expr::Record(record) = &mut *expr_cast.expr else { + todo!() + }; + + set_returning_slot(record, position, expr, field.deferred); + } + + fn needs_existence_check(&self) -> bool { + true + } +} + +impl RelationSource for InsertRelationSource<'_> { + fn selection(&self, nesting: usize) -> stmt::Query { + let mut args = vec![]; + + for pk_field in self.model.primary_key_fields() { + let entry = self.row.entry(pk_field.id.index).unwrap(); + + if entry.is_value() { + args.push(entry.to_expr()); + } else if entry.is_expr_default() { + args.push(stmt::Expr::ref_field(nesting, pk_field.id)) + } else { + todo!("{entry:#?}"); + } + } + + self.model.find_by_id(&args[..]) + } + + fn set_source_field(&mut self, field: FieldId, expr: stmt::Expr) { + assert_eq!(self.model.id, field.model); + self.row.entry_mut(field.index).insert(expr); + } + + fn set_returning_field(&mut self, field: &Field, expr: stmt::Expr) { + let record = match self.returning { + Some(stmt::Returning::Project(stmt::Expr::Record(record))) => record, + Some(stmt::Returning::Expr(stmt::Expr::List(rows))) => { + rows.items[self.index].as_record_mut_unwrap() + } + Some(stmt::Returning::Expr(stmt::Expr::Record(record))) => record, + _ => todo!("InsertRelationSource={self:#?}"), + }; + + set_returning_slot(record, field.id.index, expr, field.deferred); + } + + fn needs_existence_check(&self) -> bool { + false + } +} + +/// Map a single flattened relation assignment to its `Mutation`. +fn relation_mutation(field: &Field, entry: stmt::Assignment) -> Mutation { + match entry { + stmt::Assignment::Set(expr) => { + if expr.is_value_null() { + Mutation::DisassociateAll { delete: false } + } else { + Mutation::Associate { + expr, + exclusive: true, + } + } + } + stmt::Assignment::Insert(expr) => { + assert!(field.ty.is_has_many()); + debug_assert!(!expr.is_value_null()); + Mutation::Associate { + expr, + exclusive: false, + } + } + stmt::Assignment::Remove(expr) => { + assert!(field.ty.is_has_many()); + debug_assert!(!expr.is_value_null()); + Mutation::Disassociate { expr } + } + // `stmt::push` / `stmt::extend` target `Vec` model fields, not + // relation fields. The type system does not currently rule out passing + // them to a has-many setter, but the engine has no Append-on-relation + // lowering. + stmt::Assignment::Append(_) => { + unreachable!("Append assignment on relation field — use stmt::insert for has-many") + } + // `stmt::pop` / `stmt::remove_at` target ordered `Vec` fields. + // They have no relation analogue — has-many removal is by relation key, + // not by position. + stmt::Assignment::Pop | stmt::Assignment::RemoveAt(_) => { + unreachable!( + "Pop / RemoveAt assignment on relation field — these only apply to Vec" + ) + } + // Arithmetic ops only apply to scalar numeric model fields, never to + // relations. + stmt::Assignment::Add(_) | stmt::Assignment::Subtract(_) => { + unreachable!( + "Add / Subtract assignment on relation field — these only apply to scalar numerics" + ) + } + stmt::Assignment::Batch(_) => unreachable!("Batch entries are flattened above"), + } +} + +/// Flatten a has-many `Batch` in place into single-op assignments that +/// `plan_stmt_update_relations` can dispatch one at a time. +/// +/// `Insert` entries are folded into one multi-row INSERT via `Insert::merge` +/// — only one Insert can own the relation's returning slot per update, so +/// multiple inserts have to share a single sub-statement. The fold is done +/// in place via `retain_mut` (no extra allocation): non-`Insert` entries +/// (`Remove`, `Set`, …) keep their relative order, and the merged Insert is +/// appended last. Insert/Remove ops are independent, so the final +/// association set does not depend on their relative order. +fn flatten_relation_batch(entries: &mut Vec) { + let mut merged: Option = None; + + entries.retain_mut(|entry| { + // Keep everything that isn't an Insert sub-statement in place. + if !matches!(entry, stmt::Assignment::Insert(stmt::Expr::Stmt(_))) { + return true; + } + + // Move the Insert out, leaving a throwaway placeholder that + // `retain_mut` immediately drops since we return `false`. + let stmt::Assignment::Insert(stmt::Expr::Stmt(expr_stmt)) = + std::mem::replace(entry, stmt::Assignment::Pop) + else { + unreachable!() + }; + let insert = match *expr_stmt.stmt { + stmt::Statement::Insert(insert) => insert, + other => todo!("non-Insert sub-statement in has-many Batch: {other:#?}"), + }; + match &mut merged { + None => merged = Some(insert), + Some(acc) => acc.merge(insert), + } + false + }); + + if let Some(insert) = merged { + entries.push(stmt::Assignment::Insert(stmt::Expr::from(insert))); + } +} + +fn set_returning_slot( + record: &mut stmt::ExprRecord, + index: usize, + expr: stmt::Expr, + deferred: bool, +) { + record.fields[index] = if deferred { + lazy_slot::loaded_expr(expr) + } else { + expr + }; +} + +impl RelationRecord for InsertRelationSource<'_> { + fn take_source_field(&mut self, field: FieldId) -> stmt::Expr { + assert_eq!(self.model.id, field.model); + self.row.entry_mut(field.index).take() + } +} + +impl EmbeddedRelationSource<'_> { + fn slot(&self, field: FieldId) -> usize { + let first = self + .fields + .first() + .expect("field in an empty embedded record"); + let local = field.index.checked_sub(first.id.index).unwrap(); + assert_eq!(self.fields[local].id, field); + self.offset + local + } +} + +impl RelationRecord for EmbeddedRelationSource<'_> { + fn take_source_field(&mut self, field: FieldId) -> stmt::Expr { + let slot = self.slot(field); + self.row.entry_mut(slot).take() + } +} + +impl RelationSource for EmbeddedRelationSource<'_> { + fn selection(&self, _nesting: usize) -> stmt::Query { + unreachable!("embedded relations do not have inverse relations") + } + + fn set_source_field(&mut self, field: FieldId, expr: stmt::Expr) { + let slot = self.slot(field); + self.row.entry_mut(slot).insert(expr); + } + + fn set_returning_field(&mut self, _field: &Field, _expr: stmt::Expr) { + unreachable!("embedded belongs-to returning slots are filled by include lowering") + } + + fn needs_existence_check(&self) -> bool { + false + } +} + +/// Assign a relation's key expression to its source fields. A single-field +/// key stays intact even when its value is itself an embedded record. +pub(super) fn assign_belongs_to_key( + foreign_key: &app::ForeignKey, + expr: stmt::Expr, + mut assign: impl FnMut(app::FieldId, stmt::Expr), +) { + let key = relation_key_expr(foreign_key, expr); + + if matches!(key, stmt::Expr::Arg(_)) { + assign_projected_key(foreign_key, key, &mut assign); + return; + } + + if let [field] = &foreign_key.fields[..] { + assign(field.source, key); + return; + } + + if let Some(arity) = key.record_len() { + assert_eq!(arity, foreign_key.fields.len(), "key={key:#?}"); + let values = key.into_record_items().unwrap(); + + for (field, value) in foreign_key.fields.iter().zip(values) { + assign(field.source, value); + } + return; + } + + assign_projected_key(foreign_key, key, &mut assign); +} + +fn assign_projected_key( + foreign_key: &app::ForeignKey, + key: stmt::Expr, + assign: &mut impl FnMut(app::FieldId, stmt::Expr), +) { + for (index, field) in foreign_key.fields.iter().enumerate() { + assign(field.source, stmt::Expr::project(key.clone(), [index])); + } +} + +fn relation_key_expr(foreign_key: &app::ForeignKey, expr: stmt::Expr) -> stmt::Expr { + // A loaded model carries its stored fields, including non-primary keys. + // Ordinary relation expressions already contain the key itself. + let mut target = match expr { + stmt::Expr::Cast(target) => target, + expr => return expr, + }; + let stmt::Type::Model(target_model) = target.ty else { + return target.into(); + }; + if !foreign_key + .fields + .iter() + .all(|field| field.target.model == target_model) + { + return target.into(); + } + + let mut key = foreign_key + .fields + .iter() + .map(|field| target.expr.entry_mut(field.target.index).take()) + .collect::>(); + + if key.len() == 1 { + key.pop().unwrap() + } else { + stmt::Expr::record_from_vec(key) + } +} diff --git a/packages/toasty/src/engine/lower/relation_expr.rs b/packages/toasty/src/engine/lower/relation_expr.rs new file mode 100644 index 00000000000..9bc88cda3ec --- /dev/null +++ b/packages/toasty/src/engine/lower/relation_expr.rs @@ -0,0 +1,233 @@ +//! Resolves the first relation in an application-level path expression. +//! +//! Resolution walks field references, projections, and embedded-enum variant +//! selections directly. It separates the path into the expression that reaches +//! an embedded relation and the expression that continues from its target model. +//! +//! Variant selections identify fields but do not test the discriminant. Statement +//! normalization adds the corresponding variant predicates. + +use toasty_core::{ + schema::app::{self, BelongsTo, FieldTy, VariantId}, + stmt::{Expr, ExprContext, ExprReference, ResolvedRef}, +}; + +/// A relation found in an application-level expression. +pub(super) struct ResolvedRelation<'a> { + pub(super) field: &'a app::Field, + pub(super) target: app::ModelId, + embedded: Option>, + target_expr: Option, +} + +struct EmbeddedRelation<'a> { + belongs_to: &'a BelongsTo, + key_expr: Expr, +} + +impl<'a> ResolvedRelation<'a> { + fn new(field: &'a app::Field, embedded: Option>) -> Option { + let target = field.relation_target_id()?; + + Some(Self { + field, + target, + embedded, + target_expr: None, + }) + } + + fn project(&mut self, indices: &[usize]) { + let Some((index, remaining)) = indices.split_first() else { + return; + }; + self.target_expr = Some(match self.target_expr.take() { + Some(base) => project(base, indices), + None => project(Expr::ref_self_field(self.target.field(*index)), remaining), + }); + } + + pub(super) fn is_endpoint(&self) -> bool { + self.target_expr.is_none() + } + + pub(super) fn embedded(&self) -> Option<&'a BelongsTo> { + self.embedded.as_ref().map(|relation| relation.belongs_to) + } + + /// Returns the embedded relation's foreign key on the host record. + pub(super) fn key_expr(&self) -> Option { + self.embedded + .as_ref() + .map(|relation| relation.key_expr.clone()) + } + + /// Returns the part of the path evaluated against the target model. + pub(super) fn target_expr(&self) -> Option { + self.target_expr.clone() + } +} + +/// Locates the first direct or embedded relation in `expr`. +pub(super) fn resolve<'a>(cx: &ExprContext<'a>, expr: &Expr) -> Option> { + match resolve_expr(cx, expr)? { + PathTarget::Relation(relation) => Some(relation), + PathTarget::Embed(_) => None, + } +} + +enum PathTarget<'a> { + Embed(EmbedTarget<'a>), + Relation(ResolvedRelation<'a>), +} + +fn resolve_expr<'a>(cx: &ExprContext<'a>, expr: &Expr) -> Option> { + match expr { + Expr::Reference(reference @ ExprReference::Field { .. }) => { + let ResolvedRef::Field(field) = cx.resolve_expr_reference(reference) else { + return None; + }; + match &field.ty { + FieldTy::Embedded(embedded) => Some(PathTarget::Embed(EmbedTarget::embed( + &cx.schema().app, + embedded.target, + )?)), + _ => Some(PathTarget::Relation(ResolvedRelation::new(field, None)?)), + } + } + Expr::Project(projection) => { + let indices = projection.projection.as_slice(); + let mut target = match resolve_expr(cx, &projection.base)? { + PathTarget::Relation(mut relation) => { + relation.project(indices); + return Some(PathTarget::Relation(relation)); + } + PathTarget::Embed(target) => target, + }; + for (position, index) in indices.iter().enumerate() { + let field = target.field_at(*index)?; + + match &field.ty { + FieldTy::BelongsTo(belongs_to) => { + let host = project(copy_path(&projection.base), &indices[..position]); + let key_expr = embedded_key_expr(host, target, belongs_to)?; + + let mut relation = ResolvedRelation::new( + field, + Some(EmbeddedRelation { + belongs_to, + key_expr, + }), + )?; + relation.project(&indices[position + 1..]); + return Some(PathTarget::Relation(relation)); + } + FieldTy::Embedded(embedded) => { + target = EmbedTarget::embed(&cx.schema().app, embedded.target)?; + } + _ => return None, + } + } + Some(PathTarget::Embed(target)) + } + Expr::Variant(variant) => match resolve_expr(cx, &variant.base)? { + PathTarget::Embed(target) => Some(PathTarget::Embed(target.select(variant.variant)?)), + PathTarget::Relation(mut relation) => { + // A relation targets a model; a variant requires a target field first. + relation.target_expr = + Some(Expr::variant(relation.target_expr.take()?, variant.variant)); + Some(PathTarget::Relation(relation)) + } + }, + _ => None, + } +} + +fn embedded_key_expr(host: Expr, target: EmbedTarget<'_>, belongs_to: &BelongsTo) -> Option { + let mut fields = Vec::with_capacity(belongs_to.foreign_key.fields.len()); + + for foreign_key in &belongs_to.foreign_key.fields { + let index = target.step_of(foreign_key.source.index)?; + fields.push(Expr::project(host.clone(), [index])); + } + + Some(super::scalar_or_record(fields.into_iter())) +} + +/// Copies the host path once a relation is found, merging adjacent projections. +fn copy_path(expr: &Expr) -> Expr { + match expr { + Expr::Project(projection) => project( + copy_path(&projection.base), + projection.projection.as_slice(), + ), + Expr::Variant(variant) => Expr::variant(copy_path(&variant.base), variant.variant), + _ => expr.clone(), + } +} + +fn project(mut base: Expr, indices: &[usize]) -> Expr { + if let Expr::Project(project) = &mut base { + for index in indices { + project.projection.push(*index); + } + base + } else if indices.is_empty() { + base + } else { + Expr::project(base, indices) + } +} + +/// The embedded struct or enum, with an optional selected variant, reached by a path. +#[derive(Clone, Copy)] +pub(crate) enum EmbedTarget<'a> { + Struct(&'a app::EmbeddedStruct), + Enum(&'a app::EmbeddedEnum, Option), +} + +impl<'a> EmbedTarget<'a> { + /// Returns the embedded model for `model_id`. + pub(crate) fn embed(schema: &'a app::Schema, model_id: app::ModelId) -> Option { + match schema.model(model_id) { + app::Model::EmbeddedStruct(embedded) => Some(EmbedTarget::Struct(embedded)), + app::Model::EmbeddedEnum(embedded) => Some(EmbedTarget::Enum(embedded, None)), + app::Model::Root(_) => None, + } + } + + /// Selects a variant of the current embedded enum. + pub(crate) fn select(self, variant: VariantId) -> Option { + match self { + EmbedTarget::Enum(embedded, None) + if embedded.id == variant.model && variant.index < embedded.variants.len() => + { + Some(EmbedTarget::Enum(embedded, Some(variant))) + } + _ => None, + } + } + + /// Returns the field selected by a local path index. + pub(crate) fn field_at(&self, index: usize) -> Option<&'a app::Field> { + match self { + EmbedTarget::Struct(embedded) => embedded.fields.get(index), + EmbedTarget::Enum(embedded, Some(variant)) => { + embedded.variant_fields(variant.index).get(index) + } + EmbedTarget::Enum(_, None) => None, + } + } + + /// Returns a field's local index within this target. + fn step_of(&self, field_index: usize) -> Option { + match self { + EmbedTarget::Struct(_) => Some(field_index), + EmbedTarget::Enum(embedded, Some(variant)) => embedded + .variant_fields(variant.index) + .iter() + .position(|field| field.id.index == field_index), + EmbedTarget::Enum(_, None) => None, + } + } +} diff --git a/packages/toasty/src/engine/lower/relation_path.rs b/packages/toasty/src/engine/lower/relation_path.rs new file mode 100644 index 00000000000..28b83aac236 --- /dev/null +++ b/packages/toasty/src/engine/lower/relation_path.rs @@ -0,0 +1,60 @@ +//! Helpers for resolving app-level relation paths. + +use toasty_core::schema::app; + +/// Walk a relation path, inlining each `via` field's resolved relation chain. +/// +/// The result contains only direct relation fields. Scalar terminals are +/// omitted because they project a value from the model reached by the +/// relation chain; they do not add another relation step. +pub(super) fn flatten_relation_path( + schema: &toasty_core::Schema, + source_model_id: app::ModelId, + initial_steps: &[usize], +) -> Vec { + let mut result = Vec::with_capacity(initial_steps.len()); + let mut current_model = source_model_id; + let mut queue: Vec = initial_steps.to_vec(); + queue.reverse(); + + while let Some(index) = queue.pop() { + let field = &schema.app.model(current_model).as_root_unwrap().fields[index]; + let field_id = app::FieldId { + model: current_model, + index, + }; + + if let app::FieldTy::Via(via) = &field.ty { + debug_assert_eq!(via.path.root.as_model(), Some(current_model)); + + for step in via_relation_steps(via).iter().rev() { + queue.push(*step); + } + continue; + } + + current_model = field + .relation_target_id() + .expect("relation path step is not a relation"); + result.push(field_id); + } + + result +} + +/// Return the direct relation fields that a `via` relation follows. +pub(super) fn flatten_via_path( + schema: &toasty_core::Schema, + via: &app::Via, +) -> Option> { + let root = via.path.root.as_model()?; + Some(flatten_relation_path(schema, root, via_relation_steps(via))) +} + +fn via_relation_steps(via: &app::Via) -> &[usize] { + let projection = via.path.projection.as_slice(); + match via.terminal { + Some(_) => &projection[..projection.len() - 1], + None => projection, + } +} diff --git a/packages/toasty/src/engine/lower/returning.rs b/packages/toasty/src/engine/lower/returning.rs new file mode 100644 index 00000000000..e13fc7b59cb --- /dev/null +++ b/packages/toasty/src/engine/lower/returning.rs @@ -0,0 +1,460 @@ +use indexmap::IndexMap; +use toasty_core::{ + schema::{ + app::{FieldTy, ModelId}, + db::ColumnId, + }, + stmt, +}; + +use crate::engine::lower::{LowerStatement, LoweringContext}; + +#[derive(Debug)] +struct ConstantizeReturning<'a> { + cx: stmt::ExprContext<'a>, + source: ConstantizeSource<'a>, +} + +#[derive(Debug)] +enum ConstantizeSource<'a> { + InsertValues { + values: &'a stmt::Expr, + columns: &'a [ColumnId], + }, + UpdateAssignments { + assignments: &'a stmt::Assignments, + }, +} + +impl LowerStatement<'_, '_> { + /// Shape `Returning::Model` for the statement kind before include lowering. + /// + /// Plain relation fields are implicit includes, except during insert + /// returning, where relation slots are filled later with per-row context: + /// local `has_many` / `has_one` slots from nested relation inserts during + /// insert relation planning, and `belongs_to` slots from per-row load + /// subqueries in [`Self::plan_insert_returning_relations`]. Building an + /// include subquery here would create a stale (and, for `belongs_to`, + /// row-less) dependency for the same returning slot. + pub(super) fn prepare_model_returning_for_context( + &self, + returning: &mut stmt::Expr, + includes: &mut Vec, + is_insert: bool, + ) { + let model = self.model_unwrap(); + let stmt::Expr::Record(record) = returning else { + return; + }; + + if is_insert { + includes.retain(|inc| { + !first_model_field(&inc.path, model.id) + .is_some_and(|field| model.fields[field].ty.is_relation()) + }); + } + + for field in &model.fields { + if !field.ty.is_relation() || field.deferred { + continue; + } + + if is_insert { + record[field.id.index] = match field.ty { + FieldTy::Has(ref rel) if rel.is_many() => stmt::Expr::list_from_vec(vec![]), + FieldTy::Via(ref via) if via.is_many() => stmt::Expr::list_from_vec(vec![]), + FieldTy::Has(_) | FieldTy::Via(_) | FieldTy::BelongsTo(_) => stmt::Expr::null(), + _ => unreachable!(), + }; + continue; + } + + includes.push(stmt::Include::new(stmt::Path::field( + model.id, + field.id.index, + ))); + } + } + + /// Attempts to evaluate an INSERT statement's RETURNING clause at compile + /// time. + /// + /// This optimization transforms runtime RETURNING expressions into + /// compile-time constant values when possible. This is especially important + /// for databases like MySQL that don't support RETURNING clauses - by + /// constantizing the return values, we can avoid working around the lack o + /// database support. + /// + /// # What this does + /// + /// Converts `RETURNING` from an expression to be evaluated at runtime into + /// a constant value that's known at planning time. + /// + /// **Example:** + /// ```sql + /// INSERT INTO users (id, name) VALUES ('123', 'Alice'), ('456', 'Bob') + /// RETURNING id, name; + /// ``` + /// + /// Can be constantized to: + /// ```text + /// stmt::Returning::Expr(vec![ + /// Record { id: '123', name: 'Alice' }, + /// Record { id: '456', name: 'Bob' }, + /// ]) + /// ``` + /// + /// # How it works + /// + /// The algorithm has two main paths: + /// + /// ## Path 1: Full Constantization (all columns are constant) + /// + /// When ALL columns in the RETURNING clause can be evaluated to constants: + /// + /// 1. **Analyze** each column referenced in RETURNING to see if its values are: + /// - Constant across all rows (e.g., literal values like `'Alice'`, `123`, + /// `uuid()`) + /// - Stable and equal (same expression across all rows, e.g., all rows use + /// `DEFAULT`) + /// + /// 2. **Evaluate** the RETURNING expression for each row using the constant values: + /// - For each INSERT row, evaluate the RETURNING projection + /// - This produces a `stmt::Value` for each row + /// + /// 3. **Replace** `stmt::Returning::Project(projection)` with + /// `stmt::Returning::Expr(values)` + /// - Single-row inserts return a single value + /// - Multi-row inserts return a list of values + /// + /// ## Path 2: Partial Constantization (some columns are stable) + /// + /// When SOME columns are stable/equal across all rows but not all are constant: + /// + /// 1. **Identify** which column references have identical expressions across + /// all rows + /// 2. **Replace** those column references in the RETURNING expression with + /// the actual expression + /// 3. Leave other columns as-is (will be evaluated at runtime) + /// + /// This reduces the work needed at runtime without fully eliminating it. + pub(super) fn constantize_insert_returning( + &self, + returning: &mut stmt::Returning, + source: &stmt::Query, + ) { + match returning { + stmt::Returning::Project(project) => { + if let Some(xformed_returning) = + self.constantize_insert_returning_projection(project, source) + { + *returning = xformed_returning; + } + } + stmt::Returning::Expr(expr) => self.constantize_insert_returning_expr(expr, source), + _ => {} + } + } + + fn constantize_insert_returning_projection( + &self, + project: &mut stmt::Expr, + source: &stmt::Query, + ) -> Option { + use indexmap::map::Entry; + + // Only handle INSERT with VALUES (not INSERT from SELECT, etc.) + let stmt::ExprSet::Values(values) = &source.body else { + return None; + }; + + assert!(!values.is_empty(), "TODO: handle this case"); + + let LoweringContext::Insert(columns, None) = &self.cx else { + panic!( + "not currently lowering an insert statement; cx={:#?}", + self.cx + ); + }; + + // ==== Phase 1: Analyze which columns can be constantized ==== + // + // For each column referenced in the RETURNING clause, determine if: + // - Its value is constant across all rows (can be evaluated now) + // - Its expression is stable and identical across all rows (can be simplified) + // + // We track this in `columns_are_stable_and_equal`: + // - Some(expr) = all rows have the same stable expression for this column + // - None = column values vary across rows + + let mut columns_are_stable_and_equal = IndexMap::new(); + let mut all_const = true; // Track if ALL referenced columns are constant + + stmt::visit::for_each_expr(project, |expr| { + if let stmt::Expr::Reference(stmt::ExprReference::Column(expr_column)) = expr { + assert!(expr_column.nesting == 0, "TODO"); + + // Skip if we've already analyzed this column + let e = match columns_are_stable_and_equal.entry(*expr_column) { + Entry::Occupied(_) => return, + Entry::Vacant(e) => e, + }; + + // Find which field in the row corresponds to this column reference + let index = columns + .iter() + .position(|column| column.index == expr_column.column) + .unwrap(); + + // Check the first row to see if this field is constant + let first = &values.rows[0].as_record_unwrap().fields[index]; + all_const &= first.is_const(); + + // Check if this field has the same expression across all rows + let mut all_stable_and_equal = first.is_stable(); + + for row in &values.rows[1..] { + let field = &row.as_record_unwrap().fields[index]; + + // Check if this row's field equals the first row's field + if all_stable_and_equal { + all_stable_and_equal &= first == field; + } + + // Check if this row's field is constant + if all_const { + all_const &= field.is_const(); + } + + // Early exit if both checks have failed + if !all_stable_and_equal && !all_const { + break; + } + } + + // Store the result: Some(expr) if stable and equal, None otherwise + e.insert(if all_stable_and_equal { + Some(first) + } else { + None + }); + } + }); + + // ==== Phase 2: Apply constantization based on analysis ==== + + if all_const { + // **Full constantization path** + // All columns are constant - we can evaluate the RETURNING clause now + // and replace it with a Value variant. + + let mut constantized = vec![]; + + // Evaluate the RETURNING expression for each row + for row in &values.rows { + let input = ConstantizeReturning { + cx: self.expr_cx, + source: ConstantizeSource::InsertValues { + values: row, + columns, + }, + }; + + // Try to evaluate the projection expression with this row's values + let Ok(row) = project.eval(input) else { + // If evaluation fails, give up on constantization + return None; + }; + + constantized.push(row); + } + + // Replace the expression-based RETURNING with a constant value + Some(stmt::Returning::Expr(if source.single { + // Single row insert: return just the one value + constantized + .into_iter() + .next() + .unwrap_or(stmt::Value::Null) + .into() + } else { + // Multi-row insert: return a list of values + stmt::Value::List(constantized).into() + })) + } else { + // **Partial constantization path** + // Some columns are stable but not all are constant. + // Replace stable column references with their actual expressions. + + stmt::visit_mut::for_each_expr_mut(project, |expr| { + if let stmt::Expr::Reference(stmt::ExprReference::Column(expr_column)) = expr { + // If this column has a stable expression across all rows, inline it + if let Some(new_expr) = columns_are_stable_and_equal[&*expr_column] { + *expr = new_expr.clone(); + } + } + }); + + None + } + } + + fn constantize_insert_returning_expr(&self, expr: &mut stmt::Expr, source: &stmt::Query) { + // Only handle INSERT with VALUES (not INSERT from SELECT, etc.) + let stmt::ExprSet::Values(values) = &source.body else { + return; + }; + + assert!(!values.is_empty(), "TODO: handle this case"); + + let LoweringContext::Insert(columns, None) = &self.cx else { + panic!( + "not currently lowering an insert statement; cx={:#?}", + self.cx + ) + }; + + #[derive(Debug)] + struct Input<'a>(&'a [ColumnId], &'a stmt::Values); + + impl stmt::Input for Input<'_> { + fn resolve_ref( + &mut self, + expr_reference: &stmt::ExprReference, + projection: &stmt::Projection, + ) -> Option { + let stmt::ExprReference::Column(expr_column) = expr_reference else { + return None; + }; + + let [row] = projection.as_slice() else { + return None; + }; + + // Find which field in the row corresponds to this column reference + let index = self + .0 + .iter() + .position(|column| column.index == expr_column.column) + .unwrap(); + + let field = &self.1.rows[*row].as_record_unwrap()[index]; + + if field.is_eval() { + Some(field.clone()) + } else { + None + } + } + } + + expr.substitute(Input(columns, values)); + } +} + +/// Constantize an update's returning clause from its lowered, column-indexed +/// assignments: column references whose assignment is a constant `Set` are +/// inlined, and a fully-constant projection is evaluated now, sparing the +/// runtime returning path. A projection that still needs data (an unassigned +/// column, a relative mutation) is left for runtime. +pub(super) fn constantize_update_returning( + cx: stmt::ExprContext<'_>, + returning: &mut stmt::Returning, + assignments: &stmt::Assignments, +) { + let stmt::Returning::Project(project) = returning else { + // Already a constant value (e.g., empty record for batch + // unit-returning); nothing to constantize. + return; + }; + + project.substitute(ConstantizeReturning { + cx, + source: ConstantizeSource::UpdateAssignments { assignments }, + }); + + // Evaluate through the schema-bound input, not `eval_const`: a document + // column's projection is the raising cast over the assignment's lowering + // cast, and both conversions resolve the embed through the schema + // (`Input::resolve_model`). + let input = ConstantizeReturning { + cx, + source: ConstantizeSource::UpdateAssignments { assignments }, + }; + if let Ok(row) = project.eval(input) { + *returning = stmt::Returning::Project(row.into()); + } +} + +fn first_model_field(path: &stmt::Path, model_id: ModelId) -> Option { + if path.root.as_model()? != model_id { + return None; + } + + path.projection.as_slice().first().copied() +} + +impl stmt::Input for ConstantizeReturning<'_> { + fn resolve_ref( + &mut self, + expr_reference: &stmt::ExprReference, + projection: &stmt::Projection, + ) -> Option { + debug_assert_eq!(0, expr_reference.as_expr_column_unwrap().nesting, "TODO"); + + let needle = self + .cx + .resolve_expr_reference(expr_reference) + .as_column_unwrap(); + + match self.source { + ConstantizeSource::InsertValues { values, columns } => { + let index = columns.iter().position(|column| needle.id == *column)?; + match values { + stmt::Expr::Record(row) => { + Some(row[index].entry(projection).unwrap().to_expr()) + } + stmt::Expr::Value(stmt::Value::Record(row)) => { + Some(row[index].entry(projection).to_expr()) + } + _ => todo!("values={values:#?}"), + } + } + ConstantizeSource::UpdateAssignments { assignments } => { + if let Some(assignment) = assignments.get(&[needle.id.index]) { + match assignment { + stmt::Assignment::Set(expr) => { + assert!(expr.is_const(), "TODO; assignment={assignment:#?}"); + Some(expr.clone()) + } + // Relative mutations update a column from its + // current value — not constantizable from the + // input. Leave the reference unresolved so the + // engine fetches the post-update value from the + // driver: native RETURNING on PG/SQLite, + // ReturnValues=UPDATED_NEW on DynamoDB, or a + // follow-up SELECT on MySQL. + stmt::Assignment::Append(_) + | stmt::Assignment::Pop + | stmt::Assignment::RemoveAt(_) + | stmt::Assignment::Remove(_) + | stmt::Assignment::Add(_) + | stmt::Assignment::Subtract(_) => None, + _ => todo!( + "only SET / APPEND / collection / arithmetic mutations supported; got {assignment:#?}" + ), + } + } else { + None + } + } + } + } + + fn resolve_model( + &self, + id: toasty_core::schema::app::ModelId, + ) -> Option<&toasty_core::schema::app::Model> { + stmt::Resolve::model(self.cx.schema(), id) + } +} diff --git a/packages/toasty/src/engine/lower/tests.rs b/packages/toasty/src/engine/lower/tests.rs new file mode 100644 index 00000000000..a742233a9c4 --- /dev/null +++ b/packages/toasty/src/engine/lower/tests.rs @@ -0,0 +1,5 @@ +mod association; +mod constantize; +mod expr_or; +mod lift_in_subquery; +mod relation_expr; diff --git a/packages/toasty/src/engine/lower/tests/association.rs b/packages/toasty/src/engine/lower/tests/association.rs new file mode 100644 index 00000000000..5096dcad1e5 --- /dev/null +++ b/packages/toasty/src/engine/lower/tests/association.rs @@ -0,0 +1,261 @@ +use crate as toasty; +use crate::engine::lower::association::RewriteVia; +use crate::schema::Model; +use toasty_core::{ + driver::Capability, + schema::{Builder, app, app::FieldId, app::ModelId}, + stmt::{ + self, Association, Expr, ExprContext, ExprInSubquery, Path, Projection, Query, Returning, + SourceModel, Value, + }, +}; + +#[allow(dead_code)] +#[derive(toasty::Model)] +struct User { + #[key] + id: i64, + + #[has_many(pair = author)] + posts: toasty::Deferred>, +} + +#[allow(dead_code)] +#[derive(toasty::Model)] +struct Post { + #[key] + id: i64, + + #[index] + user_id: i64, + + #[belongs_to(key = user_id, references = id)] + author: toasty::Deferred, +} + +struct UserPostSchema { + schema: toasty_core::Schema, + user_model: ModelId, + user_id: FieldId, + user_posts: FieldId, + post_model: ModelId, + post_user_id: FieldId, + post_author: FieldId, +} + +impl UserPostSchema { + fn new() -> Self { + let app_schema = app::Schema::from_macro([User::schema(), Post::schema()]) + .expect("schema should build from macro"); + + let schema = Builder::new() + .build(app_schema, &Capability::SQLITE) + .expect("schema should build"); + + let user_model = User::id(); + let post_model = Post::id(); + + // Find field IDs by name from the generated schema + let user_id = schema + .app + .model(user_model) + .as_root_unwrap() + .field_by_name("id") + .unwrap() + .id; + let user_posts = schema + .app + .model(user_model) + .as_root_unwrap() + .field_by_name("posts") + .unwrap() + .id; + let post_user_id = schema + .app + .model(post_model) + .as_root_unwrap() + .field_by_name("user_id") + .unwrap() + .id; + let post_author = schema + .app + .model(post_model) + .as_root_unwrap() + .field_by_name("author") + .unwrap() + .id; + + Self { + schema, + user_model, + user_id, + user_posts, + post_model, + post_user_id, + post_author, + } + } +} + +#[test] +fn has_many_via_becomes_in_subquery() { + // `select(Post, via(User.posts)) → select(Post, in_subquery(author, user_query))` + let s = UserPostSchema::new(); + let cx = ExprContext::new(&s.schema); + let mut rewrite = RewriteVia::new(cx); + + let user_filter = Expr::eq( + Expr::ref_self_field(s.user_id), + Expr::Value(Value::from(42i64)), + ); + let user_query = Query::new_select(s.user_model, user_filter); + + let association = Association { + source: Box::new(user_query), + path: Path::field(s.user_model, s.user_posts.index), + }; + + let mut query = Query::new_select(s.post_model, Expr::Value(Value::Bool(true))); + if let stmt::ExprSet::Select(select) = &mut query.body + && let stmt::Source::Model(model) = &mut select.source + { + model.via = Some(association); + } + + rewrite.rewrite_via_for_query(&mut query); + + let stmt::ExprSet::Select(select) = &query.body else { + panic!("expected Select"); + }; + assert!(matches!( + &select.source, + stmt::Source::Model(SourceModel { via: None, .. }) + )); + + let filter_expr = select.filter.as_expr(); + let Expr::InSubquery(ExprInSubquery { + expr, + query: subquery, + .. + }) = filter_expr + else { + panic!("expected filter expression to be an `Expr::InSubquery`"); + }; + + // The expression should reference the pair field (`post_author`). + assert!(matches!( + &**expr, + Expr::Reference(stmt::ExprReference::Field { index, .. }) if *index == s.post_author.index + )); + + // The subquery should be the user query. + let stmt::ExprSet::Select(select) = &subquery.body else { + panic!("expected subquery body to be a `ExprSet::Select`"); + }; + + // Ensure the source of the subquery is the user model. + assert!(matches!( + &select.source, + stmt::Source::Model(SourceModel { id, .. }) if *id == s.user_model + )); +} + +#[test] +fn multi_step_via_unfolds_into_nested_in_subqueries() { + // `select(User, via(User.posts, Post.author))` should unfold into: + // User WHERE id IN (SELECT post.user_id FROM Post WHERE author IN user_query) + let s = UserPostSchema::new(); + let cx = ExprContext::new(&s.schema); + let mut rewrite = RewriteVia::new(cx); + + let user_filter = Expr::eq( + Expr::ref_self_field(s.user_id), + Expr::Value(Value::from(42i64)), + ); + let user_query = Query::new_select(s.user_model, user_filter); + + let mut path = Path::model(s.user_model); + path.projection = Projection::single(s.user_posts.index); + path.projection.push(s.post_author.index); + + let association = Association { + source: Box::new(user_query), + path, + }; + + let mut query = Query::new_select(s.user_model, Expr::Value(Value::Bool(true))); + if let stmt::ExprSet::Select(select) = &mut query.body + && let stmt::Source::Model(model) = &mut select.source + { + model.via = Some(association); + } + + rewrite.rewrite_via_for_query(&mut query); + + let stmt::ExprSet::Select(select) = &query.body else { + panic!("expected outer Select"); + }; + assert!(matches!( + &select.source, + stmt::Source::Model(SourceModel { id, via: None }) if *id == s.user_model + )); + + // Outer filter: `User.id IN (SELECT post.user_id FROM Post WHERE …)` + let Expr::InSubquery(ExprInSubquery { + expr, + query: outer_subquery, + .. + }) = select.filter.as_expr() + else { + panic!("expected outer filter to be Expr::InSubquery"); + }; + assert!(matches!( + &**expr, + Expr::Reference(stmt::ExprReference::Field { index, .. }) if *index == s.user_id.index + )); + + // The outer subquery is over Post, projects post.user_id, and has its own + // `author IN user_query` filter. + let stmt::ExprSet::Select(post_select) = &outer_subquery.body else { + panic!("expected outer subquery body to be Select"); + }; + assert!(matches!( + &post_select.source, + stmt::Source::Model(SourceModel { id, via: None }) if *id == s.post_model + )); + + let Returning::Project(project) = &post_select.returning else { + panic!("expected post subquery to project its returning"); + }; + assert!(matches!( + project, + Expr::Reference(stmt::ExprReference::Field { index, .. }) if *index == s.post_user_id.index + )); + + // The post subquery's filter must contain the inner `author IN user_query` + // produced by the recursive single-step rewrite. + let post_filter = post_select.filter.as_expr(); + let inner_in_subquery = find_in_subquery(post_filter) + .expect("expected inner Expr::InSubquery inside post subquery filter"); + assert!(matches!( + &*inner_in_subquery.expr, + Expr::Reference(stmt::ExprReference::Field { index, .. }) if *index == s.post_author.index + )); + let stmt::ExprSet::Select(user_select) = &inner_in_subquery.query.body else { + panic!("expected inner subquery body to be Select"); + }; + assert!(matches!( + &user_select.source, + stmt::Source::Model(SourceModel { id, via: None }) if *id == s.user_model + )); +} + +fn find_in_subquery(expr: &Expr) -> Option<&ExprInSubquery> { + match expr { + Expr::InSubquery(in_sub) => Some(in_sub), + Expr::BinaryOp(b) => find_in_subquery(&b.lhs).or_else(|| find_in_subquery(&b.rhs)), + Expr::And(and) => and.operands.iter().find_map(find_in_subquery), + Expr::Or(or) => or.operands.iter().find_map(find_in_subquery), + _ => None, + } +} diff --git a/packages/toasty/src/engine/lower/tests/constantize.rs b/packages/toasty/src/engine/lower/tests/constantize.rs new file mode 100644 index 00000000000..c74105bd6f7 --- /dev/null +++ b/packages/toasty/src/engine/lower/tests/constantize.rs @@ -0,0 +1,75 @@ +//! Tests for `lower::returning::constantize_update_returning`. + +use crate as toasty; +use crate::engine::lower::returning::constantize_update_returning; +use crate::engine::test_util::test_schema_with; +use crate::schema::{Embed, Model}; +use toasty_core::stmt::{self, Expr, Returning, Value}; + +#[derive(Debug, PartialEq, toasty::Embed)] +struct Profile { + name: String, + age: i64, +} + +#[derive(Debug, toasty::Model)] +#[allow(dead_code)] +struct Account { + #[key] + id: i64, + #[document] + profile: Profile, +} + +/// An update that sets a `#[document]` field constantizes its returning: the +/// projection reads the column through the mapping's raising cast, the +/// assignment carries the lowering cast over the set value, and evaluating +/// the pair round-trips to the positional record — a schema-directed +/// conversion, so the evaluation must have schema access. +#[test] +fn document_update_returning_constantizes() { + let schema = test_schema_with(&[Account::schema(), Profile::schema()]); + + let (column_id, doc_ty) = schema + .mapping + .document_columns + .first() + .expect("the schema has one document column"); + let column = schema.db.column(*column_id); + let table = schema.db.table(column_id.table); + + // The lowered shapes, straight from the mapping: the returning projection + // is the document field's raising expression (`Cast(col, Model)`), and + // the assignment is the `model_to_table` lowering cast with the set value + // substituted in — exactly what `lower_set_assignment` emits. + let mapping = schema.mapping.models.values().next().unwrap(); + let raising_expr = mapping + .fields + .iter() + .filter_map(|field| field.as_primitive()) + .find(|prim| prim.column == *column_id) + .expect("the document field maps to a primitive") + .column_expr + .clone(); + + let profile = Value::record_from_vec(vec![Value::from("new"), Value::I64(99)]); + let mut assignments = stmt::Assignments::new(); + assignments.set( + *column_id, + Expr::cast_from(Expr::Value(profile.clone()), doc_ty, &column.ty), + ); + + let mut returning = Returning::Project(Expr::record_from_vec(vec![raising_expr])); + + constantize_update_returning( + stmt::ExprContext::new_with_target(&schema, table), + &mut returning, + &assignments, + ); + + // Lowering cast then raising cast round-trip to the positional record. + assert_eq!( + returning, + Returning::Project(Expr::Value(Value::record_from_vec(vec![profile]))) + ); +} diff --git a/packages/toasty/src/engine/lower/tests/expr_or.rs b/packages/toasty/src/engine/lower/tests/expr_or.rs new file mode 100644 index 00000000000..c776696928b --- /dev/null +++ b/packages/toasty/src/engine/lower/tests/expr_or.rs @@ -0,0 +1,275 @@ +//! Tests for `lower::expr_or::is_variant_tautology_or`. +//! +//! Migrated from `simplify::tests::expr_or::variant_tautology` when the +//! rewrite moved out of simplify and into the lowering walk. + +use crate as toasty; +use crate::engine::lower::expr_or::is_variant_tautology_or; +use crate::engine::test_util::test_schema_with; +use crate::schema::Embed; +use toasty_core::schema::app::VariantId; +use toasty_core::stmt::{Expr, ExprOr}; + +#[derive(Debug, PartialEq, toasty::Embed)] +enum TwoVariant { + #[column(variant = 1)] + A, + #[column(variant = 2)] + B, +} + +#[derive(Debug, PartialEq, toasty::Embed)] +enum ThreeVariant { + #[column(variant = 1)] + X, + #[column(variant = 2)] + Y, + #[column(variant = 3)] + Z, +} + +#[test] +fn all_two_variants_is_tautology() { + let schema = test_schema_with(&[TwoVariant::schema()]); + let model_id = TwoVariant::id(); + + let expr = ExprOr { + operands: vec![ + Expr::is_variant( + Expr::arg(0), + VariantId { + model: model_id, + index: 0, + }, + ), + Expr::is_variant( + Expr::arg(0), + VariantId { + model: model_id, + index: 1, + }, + ), + ], + }; + + assert!(is_variant_tautology_or(&schema, &expr)); +} + +#[test] +fn all_three_variants_is_tautology() { + let schema = test_schema_with(&[ThreeVariant::schema()]); + let model_id = ThreeVariant::id(); + + let expr = ExprOr { + operands: vec![ + Expr::is_variant( + Expr::arg(0), + VariantId { + model: model_id, + index: 0, + }, + ), + Expr::is_variant( + Expr::arg(0), + VariantId { + model: model_id, + index: 1, + }, + ), + Expr::is_variant( + Expr::arg(0), + VariantId { + model: model_id, + index: 2, + }, + ), + ], + }; + + assert!(is_variant_tautology_or(&schema, &expr)); +} + +#[test] +fn subset_of_variants_is_not_tautology() { + let schema = test_schema_with(&[ThreeVariant::schema()]); + let model_id = ThreeVariant::id(); + + // 2 of 3 variants covered → not a tautology. + let expr = ExprOr { + operands: vec![ + Expr::is_variant( + Expr::arg(0), + VariantId { + model: model_id, + index: 0, + }, + ), + Expr::is_variant( + Expr::arg(0), + VariantId { + model: model_id, + index: 1, + }, + ), + ], + }; + + assert!(!is_variant_tautology_or(&schema, &expr)); +} + +#[test] +fn single_variant_of_two_with_extra_operand_is_not_tautology() { + let schema = test_schema_with(&[TwoVariant::schema()]); + let model_id = TwoVariant::id(); + + // Only 1 of 2 variants covered → not a tautology, regardless of the + // additional non-IsVariant operand. + let expr = ExprOr { + operands: vec![ + Expr::is_variant( + Expr::arg(0), + VariantId { + model: model_id, + index: 0, + }, + ), + Expr::arg(1), + ], + }; + + assert!(!is_variant_tautology_or(&schema, &expr)); +} + +#[test] +fn all_variants_with_extra_operands_is_tautology() { + let schema = test_schema_with(&[TwoVariant::schema()]); + let model_id = TwoVariant::id(); + + // The IsVariant operands alone cover all variants; an extra non-IsVariant + // operand does not block the tautology. + let expr = ExprOr { + operands: vec![ + Expr::is_variant( + Expr::arg(0), + VariantId { + model: model_id, + index: 0, + }, + ), + Expr::arg(5), + Expr::is_variant( + Expr::arg(0), + VariantId { + model: model_id, + index: 1, + }, + ), + ], + }; + + assert!(is_variant_tautology_or(&schema, &expr)); +} + +#[test] +fn different_inner_exprs_is_not_tautology() { + let schema = test_schema_with(&[TwoVariant::schema()]); + let model_id = TwoVariant::id(); + + // `is_variant(x, 0) or is_variant(y, 1)` — different anchors mean the + // pair could both be false at runtime. + let expr = ExprOr { + operands: vec![ + Expr::is_variant( + Expr::arg(0), + VariantId { + model: model_id, + index: 0, + }, + ), + Expr::is_variant( + Expr::arg(1), + VariantId { + model: model_id, + index: 1, + }, + ), + ], + }; + + assert!(!is_variant_tautology_or(&schema, &expr)); +} + +#[test] +fn duplicate_variants_is_not_tautology() { + let schema = test_schema_with(&[ThreeVariant::schema()]); + let model_id = ThreeVariant::id(); + + // Three IsVariants but only two distinct → 2 of 3 variants covered → + // not a tautology. + let expr = ExprOr { + operands: vec![ + Expr::is_variant( + Expr::arg(0), + VariantId { + model: model_id, + index: 0, + }, + ), + Expr::is_variant( + Expr::arg(0), + VariantId { + model: model_id, + index: 0, + }, + ), + Expr::is_variant( + Expr::arg(0), + VariantId { + model: model_id, + index: 1, + }, + ), + ], + }; + + assert!(!is_variant_tautology_or(&schema, &expr)); +} + +#[test] +fn reversed_order_is_tautology() { + let schema = test_schema_with(&[TwoVariant::schema()]); + let model_id = TwoVariant::id(); + + // Operand order does not matter. + let expr = ExprOr { + operands: vec![ + Expr::is_variant( + Expr::arg(0), + VariantId { + model: model_id, + index: 1, + }, + ), + Expr::is_variant( + Expr::arg(0), + VariantId { + model: model_id, + index: 0, + }, + ), + ], + }; + + assert!(is_variant_tautology_or(&schema, &expr)); +} + +#[test] +fn no_is_variant_operands_is_not_tautology() { + let schema = test_schema_with(&[TwoVariant::schema()]); + + let expr = ExprOr { + operands: vec![Expr::arg(0), Expr::arg(1)], + }; + + assert!(!is_variant_tautology_or(&schema, &expr)); +} diff --git a/packages/toasty/src/engine/lower/tests/lift_in_subquery.rs b/packages/toasty/src/engine/lower/tests/lift_in_subquery.rs new file mode 100644 index 00000000000..41f7bbf3517 --- /dev/null +++ b/packages/toasty/src/engine/lower/tests/lift_in_subquery.rs @@ -0,0 +1,218 @@ +//! Tests for `lower::lift_in_subquery::lift_in_subquery`. +//! +//! Migrated from `simplify::tests::lift_in_subquery` when the rewrite +//! moved out of simplify and into the lowering walk. + +use crate as toasty; +use crate::engine::lower::lift_in_subquery::lift_in_subquery; +use crate::schema::Model; +use toasty_core::{ + driver::Capability, + schema::{ + Builder, app, + app::{FieldId, ModelId}, + }, + stmt::{ + self, Expr, ExprBinaryOp, ExprContext, ExprInSubquery, Projection, Query, Returning, Value, + }, +}; + +#[allow(dead_code)] +#[derive(toasty::Model)] +struct User { + #[key] + id: i64, + + #[has_many] + posts: toasty::Deferred>, +} + +#[allow(dead_code)] +#[derive(toasty::Model)] +struct Post { + #[key] + id: i64, + + #[index] + user_id: i64, + + #[belongs_to(key = user_id, references = id)] + user: toasty::Deferred, +} + +/// Schema with `User` and `Post` models in a `HasMany`/`BelongsTo` +/// relationship. +struct UserPostSchema { + schema: toasty_core::Schema, + user_model: ModelId, + user_id: FieldId, + user_posts: FieldId, + post_model: ModelId, + post_id: FieldId, + post_user: FieldId, + post_user_id: FieldId, +} + +impl UserPostSchema { + fn new() -> Self { + let app_schema = app::Schema::from_macro([User::schema(), Post::schema()]) + .expect("schema should build from macro"); + + let schema = Builder::new() + .build(app_schema, &Capability::SQLITE) + .expect("schema should build"); + + let user_model = User::id(); + let post_model = Post::id(); + + let field = |model, name| { + schema + .app + .model(model) + .as_root_unwrap() + .field_by_name(name) + .unwrap() + .id + }; + + Self { + user_id: field(user_model, "id"), + user_posts: field(user_model, "posts"), + post_id: field(post_model, "id"), + post_user: field(post_model, "user"), + post_user_id: field(post_model, "user_id"), + schema, + user_model, + post_model, + } + } +} + +#[test] +fn belongs_to_lifts_fk_constraint_to_direct_eq() { + let s = UserPostSchema::new(); + let cx = ExprContext::new(&s.schema); + + let post_source: stmt::Source = s.post_model.into(); + let scoped_cx = cx.scope(&post_source); + + // `lift_in_subquery(user, select(User, eq(id, 42))) → eq(user_id, 42)` + let expr = Expr::ref_self_field(s.post_user); + let filter = Expr::eq( + Expr::ref_self_field(s.user_id), + Expr::Value(Value::from(42i64)), + ); + let query = Query::new_select(s.user_model, filter); + + let result = lift_in_subquery(&scoped_cx, &expr, &query); + + assert!(result.is_some()); + let lifted = result.unwrap(); + let Expr::BinaryOp(ExprBinaryOp { op, lhs, rhs }) = lifted else { + panic!("expected result to be an `Expr::BinaryOp`"); + }; + assert!(op.is_eq()); + assert!(matches!( + *lhs, + Expr::Reference(stmt::ExprReference::Field { index: 1, .. }) + )); + assert!(matches!(*rhs, Expr::Value(Value::I64(42)))); +} + +/// `Project(Ref(BelongsTo), [HasIdx])` paired across a shared PK fuses into a +/// single `outer_fk IN (SELECT inner_fk FROM target WHERE …)` — the +/// intermediate routing model (`User` here) is skipped. +/// +/// Input AST for `Post.user.posts IN (SELECT FROM Post WHERE id == 42)`: +/// - LHS: `Project(Ref(Post.user), [User.posts_idx])` +/// - RHS: `SELECT FROM Post WHERE Post.id == 42` +/// +/// Expected fused output: +/// `Post.user_id IN (SELECT Post.user_id FROM Post WHERE Post.id == 42)`. +#[test] +fn belongs_to_has_many_fuses_to_direct_fk() { + let s = UserPostSchema::new(); + let cx = ExprContext::new(&s.schema); + let post_source: stmt::Source = s.post_model.into(); + let scoped_cx = cx.scope(&post_source); + + let expr = Expr::project( + Expr::ref_self_field(s.post_user), + Projection::single(s.user_posts.index), + ); + let inner_filter = Expr::eq( + Expr::ref_self_field(s.post_id), + Expr::Value(Value::from(42i64)), + ); + let query = Query::new_select(s.post_model, inner_filter); + + let lifted = lift_in_subquery(&scoped_cx, &expr, &query).expect("lift to succeed"); + + let Expr::InSubquery(ExprInSubquery { + expr: lhs, + query: fused, + .. + }) = lifted + else { + panic!("expected InSubquery, got {lifted:?}"); + }; + + // Outer LHS is the FK column on Post (no projection through a relation). + assert!(matches!( + *lhs, + Expr::Reference(stmt::ExprReference::Field { nesting: 0, index }) + if index == s.post_user_id.index + )); + + let select = fused.body.as_select_unwrap(); + + // No routing through User: the fused subquery scans Post directly. + assert_eq!(select.source.model_id_unwrap(), s.post_model); + + // Projects the FK column (Post.user_id), not Post's PK. + let Returning::Project(returning) = &select.returning else { + panic!("expected Returning::Project, got {:?}", select.returning); + }; + let Expr::Reference(stmt::ExprReference::Field { + index: ret_index, .. + }) = returning + else { + panic!("expected single-column field projection, got {returning:?}"); + }; + assert_eq!(*ret_index, s.post_user_id.index); + + // The inner filter passes through unchanged: `Post.id == 42`. + let Some(Expr::BinaryOp(ExprBinaryOp { op, lhs, rhs })) = &select.filter.expr else { + panic!("expected BinaryOp filter, got {:?}", select.filter.expr); + }; + assert!(op.is_eq()); + assert!(matches!( + **lhs, + Expr::Reference(stmt::ExprReference::Field { index, .. }) + if index == s.post_id.index + )); + assert!(matches!(**rhs, Expr::Value(Value::I64(42)))); +} + +/// `Project` with a non-relation base bails (no lift, no panic) — the lift is +/// only meaningful when the path passes through a relation field. +#[test] +fn project_with_non_relation_base_returns_none() { + let s = UserPostSchema::new(); + let cx = ExprContext::new(&s.schema); + let post_source: stmt::Source = s.post_model.into(); + let scoped_cx = cx.scope(&post_source); + + // `Project(Ref(Post.id), [0])` — projecting through a scalar PK field, not + // a relation. The lift can't re-root this; it must return None. + let expr = Expr::project(Expr::ref_self_field(s.post_id), Projection::single(0)); + let query = Query::new_select( + s.post_model, + Expr::eq( + Expr::ref_self_field(s.post_id), + Expr::Value(Value::from(1i64)), + ), + ); + + assert!(lift_in_subquery(&scoped_cx, &expr, &query).is_none()); +} diff --git a/packages/toasty/src/engine/lower/tests/relation_expr.rs b/packages/toasty/src/engine/lower/tests/relation_expr.rs new file mode 100644 index 00000000000..8cad9ba0d71 --- /dev/null +++ b/packages/toasty/src/engine/lower/tests/relation_expr.rs @@ -0,0 +1,131 @@ +use crate as toasty; +use crate::{ + engine::{lower::relation_expr::resolve, test_util::test_schema_with}, + schema::{Embed, Model}, +}; +use toasty_core::{ + schema::app::VariantId, + stmt::{Expr, ExprContext, Source}, +}; + +#[derive(Debug, toasty::Embed)] +enum State { + Active { value: i64 }, + Inactive, +} + +#[derive(Debug, toasty::Model)] +struct Parent { + #[key] + id: i64, + state: State, +} + +#[derive(Debug, toasty::Embed)] +struct Link { + parent_id: i64, + #[belongs_to(key = parent_id, references = id)] + parent: toasty::Deferred, +} + +#[derive(Debug, toasty::Embed)] +enum Owner { + Linked { link: Link }, + Empty, +} + +#[derive(Debug, toasty::Model)] +struct Object { + #[key] + id: i64, + owner: Owner, +} + +fn schema() -> toasty_core::Schema { + test_schema_with(&[ + Object::schema(), + Owner::schema(), + Link::schema(), + Parent::schema(), + State::schema(), + ]) +} + +fn owner() -> Expr { + Expr::variant( + Expr::ref_self_field(Object::id().field(1)), + VariantId { + model: Owner::id(), + index: 0, + }, + ) +} + +fn active(base: Expr) -> Expr { + Expr::variant( + base, + VariantId { + model: State::id(), + index: 0, + }, + ) +} + +#[test] +fn projection_segments_preserve_relation_and_target_variants() { + let schema = schema(); + let source = Source::from(Object::id()); + let cx = ExprContext::new(&schema); + let cx = cx.scope(&source); + // Owner::Linked.link.parent.state, split at different projection boundaries. + let paths = [ + Expr::project(owner(), [0, 1, 1]), + Expr::project(Expr::project(owner(), [0]), [1, 1]), + Expr::project(Expr::project(owner(), [0, 1]), [1]), + Expr::project(Expr::project(Expr::project(owner(), [0]), [1]), [1]), + ]; + let endpoint = Expr::project(owner(), [0, 1]); + let relation = resolve(&cx, &endpoint).unwrap(); + assert!(relation.is_endpoint()); + for path in paths { + let expr = Expr::project(active(path), [0]); + let resolved = resolve(&cx, &expr).unwrap(); + assert_eq!(resolved.field.id, relation.field.id); + assert_eq!(resolved.key_expr(), relation.key_expr()); + assert_eq!(resolved.target, Parent::id()); + assert_eq!( + resolved.target_expr(), + Some(Expr::project( + active(Expr::ref_self_field(Parent::id().field(1))), + [0] + )) + ); + } +} + +#[test] +fn relation_endpoint_and_invalid_variant_selection() { + let schema = schema(); + let source = Source::from(Object::id()); + let cx = ExprContext::new(&schema); + let cx = cx.scope(&source); + let relation = Expr::project(owner(), [0, 1]); + let identity = Expr::project(relation.clone(), &[][..]); + let resolved = resolve(&cx, &identity).unwrap(); + assert_eq!( + resolved.key_expr(), + Some(Expr::project(Expr::project(owner(), [0]), [0])) + ); + assert!(resolved.is_endpoint()); + assert!(resolved.target_expr().is_none()); + // The relation targets a model, so selecting a variant before a field is invalid. + assert!(resolve(&cx, &active(relation)).is_none()); + // An enum's fields require a variant selection before projection. + assert!( + resolve( + &cx, + &Expr::project(Expr::ref_self_field(Object::id().field(1)), [0, 1]) + ) + .is_none() + ); +} diff --git a/packages/toasty/src/engine/lower/via_join.rs b/packages/toasty/src/engine/lower/via_join.rs new file mode 100644 index 00000000000..394ac9a6e24 --- /dev/null +++ b/packages/toasty/src/engine/lower/via_join.rs @@ -0,0 +1,330 @@ +//! Lowering for `.include()` / `.select()` of multi-step (`via`) relations. +//! +//! A `via` relation reaches its target through a path of existing relations. +//! [`ViaJoin`] resolves that path into a single JOIN from the target back to +//! the root so the engine can issue one query per include and group children +//! with their parent in `NestedMerge`. This relies on the database executing +//! the join, so it is SQL-only — a key-value backend would need a cascade of +//! per-step queries instead. + +use toasty_core::{ + schema::{app, mapping}, + stmt, +}; + +use crate::engine::lower::LowerStatement; + +impl LowerStatement<'_, '_> { + /// Build the include subquery for a multi-step (`via`) relation. + /// + /// The child query is rooted at the via target and joins back to the root + /// through every intermediate model (see [`ViaJoin`]), projecting the + /// linking foreign-key column so `NestedMerge` can group children by + /// parent. It is emitted in fully-lowered form, so the standard lowering + /// walk only has to rewrite the cross-statement parent-key reference into + /// an `Arg::Ref`. Each child row is `[link_key, target_record]`; the + /// trailing projection drops the link key so the parent sees only the + /// targets (a `Map` over the list for `has_many`, a direct project for a + /// single `has_one`). + pub(super) fn build_via_include_subquery( + &mut self, + field_index: usize, + via: &app::Via, + nested: &[stmt::Projection], + ) -> stmt::Expr { + if !nested.is_empty() { + todo!("nested `.include()` through a multi-step `via` relation"); + } + + let schema = self.schema(); + let model = self.model_unwrap(); + let single = match &model.fields[field_index].ty { + app::FieldTy::Via(via) => via.is_one(), + _ => unreachable!("build_via_include_subquery called on non-via field"), + }; + let nullable = model.fields[field_index].nullable(); + let join = ViaJoin::resolve(schema, model.id, via); + + // WHERE: the linking column (on the root-adjacent model) equals the + // parent's referenced key. Use the field's model-level expression + // (column ref + any storage→model cast) so it lines up type-wise with + // the parent's lowered key both here and in the `NestedMerge` group + // key. + let (link_field, parent_key_field) = join.link(); + let link_col = model_level_column_expr(schema, link_field, join.slot(1)); + let filter = stmt::Expr::eq(link_col.clone(), stmt::Expr::ref_field(1, parent_key_field)); + + // RETURNING `[link_col, value]`. The link column lands in + // `load_data_select_items` so the qualification resolves to a + // `SortLookup`. For a relation terminal, `value` is the schema's + // `default_returning` (the whole target record). For a scalar terminal, + // it is the projected terminal column on the target (table slot 0) — + // the same shape as `.select(Target::fields().field())`. + let value = match via.terminal { + Some(terminal) => model_level_column_expr( + schema, + app::FieldId { + model: join.target(), + index: terminal, + }, + 0, + ), + None => schema.mapping_for(join.target()).default_returning.clone(), + }; + let returning = stmt::Expr::record_from_vec(vec![link_col, value]); + + // `DISTINCT` collapses duplicate targets produced when the path fans + // out (e.g. two comments on the same article) — matching a direct via + // query's distinct-target semantics. + let mut select = stmt::Select::new(join.build_source(schema), filter); + select.returning = stmt::Returning::Project(returning); + select.distinct = true; + + let mut query = stmt::Query::builder(select).build(); + query.single = single; + + // The query is already lowered, so this is mostly a no-op beyond + // rewriting the parent-key `Reference::Field { nesting: 1 }` into an + // `Arg::Ref`. + let sub_expr = self.lower_sub_stmt(stmt::Statement::Query(query)); + + // Drop the link key from each `[link_key, target_record]` row; the + // parent wants only the target. + if !single { + // A `has_many` via yields a list, so map over it (`arg(0)` is the + // item) and project the target out of each row. + return stmt::Expr::map(sub_expr, stmt::Expr::project(stmt::Expr::arg(0), [1usize])); + } + + // A single (`has_one`) via yields one `[link_key, target_record]` + // record; project the target out. A nullable single relation, though, + // produces `Null` when the `INNER JOIN` matched nothing, and projecting + // into `Null` would panic — so strip the link key only on the non-null + // branch. + if nullable { + super::map_nullable_single(sub_expr, stmt::Expr::project(stmt::Expr::arg(0), [1usize])) + } else { + stmt::Expr::project(sub_expr, [1usize]) + } + } +} + +/// A multi-step (`via`) relation resolved into a JOIN from the target back to +/// the root. +/// +/// `models` is the path `[root, …intermediates, target]`; `edges[i]` is the +/// foreign key joining `models[i]` and `models[i + 1]`. +/// +/// The child query lays its tables out target-first so the target's pre-built +/// `default_returning` (whose column refs point at slot 0) is reused verbatim: +/// +/// ```text +/// slot 0 FROM target +/// slot 1.. JOIN intermediates, target-adjacent first +/// ``` +/// +/// The root is not a table — it is the parent query, reached through the WHERE +/// filter on `edges[0]`. +struct ViaJoin { + models: Vec, + edges: Vec, +} + +impl ViaJoin { + fn resolve(schema: &toasty_core::Schema, root: app::ModelId, via: &app::Via) -> ViaJoin { + // For a scalar terminal the path's last step is the projected field, + // not a relation; the relation chain (which the JOIN walks) is + // everything before it. + let steps = super::relation_path::flatten_via_path(schema, via) + .expect("via relation path must start at a model"); + debug_assert_eq!(steps.first().map(|field| field.model), Some(root)); + assert!( + !steps.is_empty(), + "via path must have at least one step (validated at schema build time)" + ); + + let mut models = Vec::with_capacity(steps.len() + 1); + let mut edges = Vec::with_capacity(steps.len()); + models.push(root); + for &field_id in &steps { + debug_assert_eq!(field_id.model, *models.last().unwrap()); + models.push(schema.app.field(field_id).relation_target_id().unwrap()); + edges.push(Edge::resolve(schema, field_id)); + } + + ViaJoin { models, edges } + } + + /// The via target — the model whose rows the include loads. + fn target(&self) -> app::ModelId { + *self.models.last().unwrap() + } + + /// Table slot for the model at chain position `pos` (`1..=edges.len()`). + /// Tables are target-first, so the target (highest position) is slot 0 and + /// the root-adjacent model (position 1) is the last slot. + fn slot(&self, pos: usize) -> usize { + self.models.len() - 1 - pos + } + + /// The `(link, parent_key)` field pair: `link` lives on the root-adjacent + /// model and matches `parent_key` on the root in the WHERE filter. + fn link(&self) -> (app::FieldId, app::FieldId) { + let root_edge = &self.edges[0]; + (root_edge.target_side, root_edge.root_side) + } + + /// The `FROM target JOIN …intermediates` source. + fn build_source(&self, schema: &toasty_core::Schema) -> stmt::Source { + let mut tables = Vec::with_capacity(self.edges.len()); + let mut joins = Vec::with_capacity(self.edges.len().saturating_sub(1)); + + tables.push(stmt::TableRef::Table(schema.table_id_for(self.target()))); + + // Walk intermediates target-adjacent first so table slots increase + // toward the root. `edges[pos]` joins this intermediate (`models[pos]`) + // to its already-placed neighbour (`models[pos + 1]`). + for pos in (1..self.edges.len()).rev() { + tables.push(stmt::TableRef::Table(schema.table_id_for(self.models[pos]))); + + let edge = &self.edges[pos]; + joins.push(stmt::Join { + table: stmt::SourceTableId(self.slot(pos)), + constraint: stmt::JoinOp::Inner(stmt::Expr::eq( + raw_column(schema, self.slot(pos), edge.root_side), + raw_column(schema, self.slot(pos + 1), edge.target_side), + )), + }); + } + + stmt::Source::Table(stmt::SourceTable { + tables, + from: vec![stmt::TableWithJoins { + relation: stmt::TableFactor::Table(stmt::SourceTableId(0)), + joins, + }], + }) + } +} + +/// A foreign-key edge between two adjacent models on a via path, with the FK +/// field on each side resolved regardless of which side declared the relation. +struct Edge { + /// FK field on the model nearer the root. + root_side: app::FieldId, + /// FK field on the model nearer the target. + target_side: app::FieldId, +} + +impl Edge { + fn resolve(schema: &toasty_core::Schema, field_id: app::FieldId) -> Edge { + let field = schema.app.field(field_id); + + // A has-relation is reached through its paired `BelongsTo`, which lives + // on the target-side model and owns the FK; a `BelongsTo` step is the + // edge itself, on the root-side model. + let (belongs_to, owner_is_target_side) = match &field.ty { + app::FieldTy::Has(_) => { + let pair = field + .pair() + .expect("via paths are unfolded into direct steps before edge resolution"); + (schema.app.field(pair).ty.as_belongs_to_unwrap(), true) + } + app::FieldTy::BelongsTo(belongs_to) => (belongs_to, false), + _ => unreachable!("via step is not a relation field"), + }; + + // The FK source lives on the BT owner, the target on the model it + // references; map those onto root/target sides via the owner's side. + let [fk] = &belongs_to.foreign_key.fields[..] else { + todo!("composite foreign keys in via include path"); + }; + if owner_is_target_side { + Edge { + root_side: fk.target, + target_side: fk.source, + } + } else { + Edge { + root_side: fk.source, + target_side: fk.target, + } + } + } +} + +/// The single-column mapping for a foreign-key field. +/// +/// Besides primitives, foreign keys may use embedded newtypes and unit enums. +/// Newtypes map through one-field structs to a primitive leaf; unit enums map +/// through their discriminant. +struct SingleColumnFk<'a> { + primitive: &'a mapping::FieldPrimitive, + newtype_depth: usize, +} + +fn single_column_fk(schema: &toasty_core::Schema, field_id: app::FieldId) -> SingleColumnFk<'_> { + fn resolve(field: &mapping::Field) -> Option> { + match field { + mapping::Field::Primitive(primitive) => Some(SingleColumnFk { + primitive, + newtype_depth: 0, + }), + mapping::Field::Struct(field) if field.fields.len() == 1 => { + let mut fk = resolve(&field.fields[0])?; + fk.newtype_depth += 1; + Some(fk) + } + mapping::Field::Enum(field) + if field + .variants + .iter() + .all(|variant| variant.fields.is_empty()) => + { + Some(SingleColumnFk { + primitive: &field.discriminant, + newtype_depth: 0, + }) + } + _ => None, + } + } + + resolve(&schema.mapping_for(field_id.model).fields[field_id.index]) + .expect("FK field maps to a single column") +} + +/// A raw (storage-level) column reference at table `slot` for a FK field. +/// Used in JOIN constraints, which compare stored values directly and so need +/// no storage→model cast. +fn raw_column(schema: &toasty_core::Schema, slot: usize, field_id: app::FieldId) -> stmt::Expr { + stmt::Expr::column(stmt::ExprReference::column( + slot, + single_column_fk(schema, field_id).primitive.column.index, + )) +} + +/// The single-column expression for a FK field, re-pointed at table `slot`. +/// +/// Primitive leaves and unit-enum discriminants use `column_expr` to preserve +/// storage casts. This path does not use an embed's `default_returning`, which +/// may contain nullable presence guards or deferred-field placeholders. +fn model_level_column_expr( + schema: &toasty_core::Schema, + field_id: app::FieldId, + slot: usize, +) -> stmt::Expr { + let fk = single_column_fk(schema, field_id); + let mut expr = fk.primitive.column_expr.clone(); + for _ in 0..fk.newtype_depth { + expr = stmt::Expr::record([expr]); + } + + stmt::visit_mut::for_each_expr_mut(&mut expr, |e| { + if let stmt::Expr::Reference(stmt::ExprReference::Column(col)) = e { + col.table = slot; + } + }); + + expr +} diff --git a/packages/toasty/src/engine/mir.rs b/packages/toasty/src/engine/mir.rs new file mode 100644 index 00000000000..5d1de8f7a00 --- /dev/null +++ b/packages/toasty/src/engine/mir.rs @@ -0,0 +1,102 @@ +mod alias; +pub(crate) use alias::Alias; + +mod r#const; +pub(crate) use r#const::Const; + +mod delete_by_key; +pub(crate) use delete_by_key::DeleteByKey; + +mod eval; +pub(crate) use eval::Eval; + +mod exec_statement; +pub(crate) use exec_statement::ExecStatement; + +mod filter; +pub(crate) use filter::Filter; + +mod find_pk_by_index; +pub(crate) use find_pk_by_index::FindPkByIndex; + +mod get_by_key; +pub(crate) use get_by_key::GetByKey; + +mod guards; +use guards::annotate_guards; + +mod input; +pub(crate) use input::InputRead; + +mod logical_plan; +pub(crate) use logical_plan::LogicalPlan; + +mod nested_merge; +pub(crate) use nested_merge::NestedMerge; + +mod node; +pub(crate) use node::Node; + +mod operation; +pub(crate) use operation::Operation; + +mod query_pk; +pub(crate) use query_pk::QueryPk; + +mod read_modify_write; +pub(crate) use read_modify_write::ReadModifyWrite; + +mod repeat; +pub(crate) use repeat::Repeat; + +mod scan; +pub(crate) use scan::Scan; + +mod store; +pub(crate) use store::{NodeId, Store}; + +mod update_by_key; +pub(crate) use update_by_key::UpdateByKey; + +mod upsert; +pub(crate) use upsert::Upsert; + +use indexmap::IndexSet; +use toasty_core::{ + schema::db::{ColumnId, TableId}, + stmt, +}; + +/// Resolves the columns a driver operation selects to [`ColumnId`]s. +pub(crate) fn column_ids( + table: TableId, + columns: &IndexSet, +) -> impl Iterator + '_ { + columns.iter().map(move |expr_reference| { + let stmt::ExprReference::Column(expr_column) = expr_reference else { + todo!() + }; + debug_assert_eq!(expr_column.nesting, 0); + debug_assert_eq!(expr_column.table, 0); + + ColumnId { + table, + index: expr_column.column, + } + }) +} + +/// Extracts the per-row column types from a node's return type. A node +/// returning rows has type `List>` — the record field types tell +/// the driver how to decode each row. `Unit` means the node returns only a row +/// count, so there are no column types. +pub(crate) fn row_field_types(ty: &stmt::Type) -> Option> { + match ty { + stmt::Type::List(rows) => match &**rows { + stmt::Type::Record(fields) => Some(fields.clone()), + _ => todo!("row_field_types: ty={ty:#?}"), + }, + stmt::Type::Unit => None, + _ => todo!("row_field_types: ty={ty:#?}"), + } +} diff --git a/packages/toasty/src/engine/mir/alias.rs b/packages/toasty/src/engine/mir/alias.rs new file mode 100644 index 00000000000..ffed7cff9d0 --- /dev/null +++ b/packages/toasty/src/engine/mir/alias.rs @@ -0,0 +1,26 @@ +use toasty_core::stmt; + +use crate::engine::mir; + +/// Passes another node's output through unchanged. +/// +/// Fills a reserved node slot (see [`Store::reserve`](super::Store::reserve)) +/// once planning knows which node actually produces the slot's value. Other +/// statements reference the reserved slot before its producer is planned; the +/// alias makes that reference concrete without rewriting their edges. At +/// execution the pass-through is a move between variable slots — rows are +/// never touched. A future copy-propagation pass may eliminate these. +#[derive(Debug)] +pub(crate) struct Alias { + /// The node whose output is passed through. + pub(crate) input: mir::NodeId, + + /// The output type (same as input). + pub(crate) ty: stmt::Type, +} + +impl From for mir::Node { + fn from(value: Alias) -> Self { + mir::Operation::Alias(value).into() + } +} diff --git a/packages/toasty/src/engine/mir/const.rs b/packages/toasty/src/engine/mir/const.rs new file mode 100644 index 00000000000..83705828f9f --- /dev/null +++ b/packages/toasty/src/engine/mir/const.rs @@ -0,0 +1,22 @@ +use toasty_core::stmt; + +use crate::engine::mir; + +/// A constant list of values. +/// +/// Used to inject static data into the operation graph, such as literal values +/// from the query or pre-computed results. +#[derive(Debug)] +pub(crate) struct Const { + /// The constant rows. + pub(crate) value: stmt::Value, + + /// The type of this constant. + pub(crate) ty: stmt::Type, +} + +impl From for mir::Node { + fn from(value: Const) -> Self { + mir::Operation::Const(value).into() + } +} diff --git a/packages/toasty/src/engine/mir/delete_by_key.rs b/packages/toasty/src/engine/mir/delete_by_key.rs new file mode 100644 index 00000000000..819a9d58121 --- /dev/null +++ b/packages/toasty/src/engine/mir/delete_by_key.rs @@ -0,0 +1,35 @@ +use toasty_core::{schema::db::TableId, stmt}; + +use crate::engine::mir; + +/// Deletes records by primary key. +/// +/// Used with NoSQL drivers to delete records given a list of primary key values. +/// +/// Keys are always specified as an input node, whether a [`Const`] or the +/// output of a dependent operation. +/// +/// [`Const`]: super::Const +#[derive(Debug)] +pub(crate) struct DeleteByKey { + /// The node producing the list of primary keys to delete. + pub(crate) input: mir::NodeId, + + /// The table to delete records from. + pub(crate) table: TableId, + + /// Optional additional filter applied before deletion. + pub(crate) filter: Option, + + /// Optional condition for optimistic locking (e.g., version check). + pub(crate) condition: Option, + + /// The return type. + pub(crate) ty: stmt::Type, +} + +impl From for mir::Node { + fn from(value: DeleteByKey) -> Self { + mir::Operation::DeleteByKey(value).into() + } +} diff --git a/packages/toasty/src/engine/mir/eval.rs b/packages/toasty/src/engine/mir/eval.rs new file mode 100644 index 00000000000..45e8d0a0a14 --- /dev/null +++ b/packages/toasty/src/engine/mir/eval.rs @@ -0,0 +1,91 @@ +use indexmap::IndexSet; +use toasty_core::stmt::{self, visit_mut}; + +use crate::engine::{eval, mir}; + +/// Evaluates `body` once over whole input values: with no `row_input`, +/// `arg(i)` is `inputs[i]`'s complete output; with a `row_input`, `arg(0)` is +/// `row_input`'s complete output and `arg(1 + i)` is `inputs[i]`'s. +/// +/// A `row_input` marks the operation as per-row: the body is a `map` over +/// `arg(0)` (built by [`Eval::map_over`]), so the output has one element per +/// input row and pagination metadata forwards from `row_input`. Zero input +/// rows means the map body never runs, so the other inputs are read only when +/// `row_input` returned rows — the guard pass reads this from +/// [`Operation::input_reads`](mir::Operation::input_reads). +#[derive(Debug)] +pub(crate) struct Eval { + /// When set, this operation returns one result for each row from this input. + pub(crate) row_input: Option, + + /// Nodes whose whole outputs the body reads. + pub(crate) inputs: IndexSet, + + /// The function to evaluate, over whole input values ordered + /// `[row_input?, inputs...]`. Its return type is the operation's output + /// type. + pub(crate) body: eval::Func, +} + +impl Eval { + /// Evaluates `body` once over the whole outputs of `inputs`: + /// `arg(i)` = `inputs[i]`. + pub(crate) fn compute(inputs: IndexSet, body: eval::Func) -> Self { + Eval { + row_input: None, + inputs, + body, + } + } + + /// Evaluates the per-row `body` once per row of `row_input`: `arg(0)` = + /// current row, `arg(1 + i)` = `inputs[i]`. + /// + /// The executor evaluates one function over whole input values, so the + /// per-row structure is erased here into a single `map` expression over + /// input 0, with inputs ordered `[row_input, inputs...]`. + pub(crate) fn map_over( + store: &mir::Store, + row_input: mir::NodeId, + inputs: IndexSet, + body: eval::Func, + ) -> Self { + debug_assert_eq!(body.args.len(), 1 + inputs.len()); + debug_assert!(!inputs.contains(&row_input)); + + let mut arg_tys = vec![store[row_input].ty().clone()]; + for input in &inputs { + arg_tys.push(store[input].ty().clone()); + } + + // Inside the map, the body's `arg(0)` (the row) resolves to the map's + // element scope unchanged, while references to other inputs must + // climb one extra scope — past the map — to reach the function + // inputs. A body arg references a body parameter when its nesting + // equals the number of scopes around it. + let ty = stmt::Type::list(body.ret.clone()); + let mut map_body = body.into_expr(); + visit_mut::walk_expr_scoped_mut(&mut map_body, 0, |expr, scope_depth| { + if let stmt::Expr::Arg(arg) = expr + && arg.nesting == scope_depth + && arg.position >= 1 + { + arg.nesting += 1; + } + true + }); + let expr = stmt::Expr::map(stmt::Expr::arg(0), map_body); + + Eval { + row_input: Some(row_input), + inputs, + body: eval::Func::from_stmt_typed(expr, arg_tys, ty), + } + } +} + +impl From for mir::Node { + fn from(value: Eval) -> Self { + mir::Operation::Eval(value).into() + } +} diff --git a/packages/toasty/src/engine/mir/exec_statement.rs b/packages/toasty/src/engine/mir/exec_statement.rs new file mode 100644 index 00000000000..9eea0f94ce5 --- /dev/null +++ b/packages/toasty/src/engine/mir/exec_statement.rs @@ -0,0 +1,44 @@ +use indexmap::IndexSet; +use toasty_core::stmt; + +use crate::engine::{exec, mir}; + +/// Executes a SQL statement against the database. +/// +/// Used with SQL-capable drivers to delegate query execution to the database's +/// query engine. The statement may reference inputs from other nodes. +#[derive(Debug)] +pub(crate) struct ExecStatement { + /// Nodes whose outputs are passed as arguments to the statement. + pub(crate) inputs: IndexSet, + + /// The SQL statement to execute. + pub(crate) stmt: stmt::Statement, + + /// The return type of this operation. + pub(crate) ty: stmt::Type, + + /// How this statement's output is interpreted. For a conditional write the + /// statement's leading two columns are probe counts checked against each + /// other; see [`exec::ConditionalOutput`]. + pub(crate) conditional: exec::ConditionalOutput, + + /// Pagination configuration (None if not paginated) + pub(crate) pagination: Option, +} + +impl From for mir::Node { + fn from(value: ExecStatement) -> Self { + debug_assert!( + { + match &value.stmt { + stmt::Statement::Query(query) => !query.single, + _ => true, + } + }, + "as of now, no database can execute single queries" + ); + + mir::Operation::ExecStatement(Box::new(value)).into() + } +} diff --git a/packages/toasty/src/engine/mir/filter.rs b/packages/toasty/src/engine/mir/filter.rs new file mode 100644 index 00000000000..ca712ee3e9d --- /dev/null +++ b/packages/toasty/src/engine/mir/filter.rs @@ -0,0 +1,38 @@ +use indexmap::IndexSet; +use toasty_core::stmt; + +use crate::engine::{eval, mir}; + +/// Keeps the input rows for which `predicate` holds. +/// +/// The predicate's `arg(0)` is the current row; `arg(1 + i)` is the whole +/// output of `args[i]`. Every input row is read (the whole input must be +/// scanned), so the input — and each predicate arg — is an +/// [`Always`](mir::InputRead::Always) read even though evaluation +/// is per-row. A predicate that reads only its args (a key-value +/// pre-filter) ignores `arg(0)`: when it is false every row is dropped, so +/// downstream operations see an empty list. +/// +/// Used when the database cannot apply all filter conditions natively (e.g., +/// NoSQL drivers with limited query capabilities). +#[derive(Debug)] +pub(crate) struct Filter { + /// The node producing the records to filter. + pub(crate) input: mir::NodeId, + + /// Nodes whose whole outputs the predicate reads as `arg(1 + i)`. + pub(crate) args: IndexSet, + + /// The predicate: `arg(0)` = current row, `arg(1 + i)` = `args[i]`; keep + /// the row when true. + pub(crate) predicate: eval::Func, + + /// The output type (same as input, but potentially fewer rows). + pub(crate) ty: stmt::Type, +} + +impl From for mir::Node { + fn from(value: Filter) -> Self { + mir::Operation::Filter(value).into() + } +} diff --git a/packages/toasty/src/engine/mir/find_pk_by_index.rs b/packages/toasty/src/engine/mir/find_pk_by_index.rs new file mode 100644 index 00000000000..7c54f565446 --- /dev/null +++ b/packages/toasty/src/engine/mir/find_pk_by_index.rs @@ -0,0 +1,35 @@ +use indexmap::IndexSet; +use toasty_core::{ + schema::db::{IndexId, TableId}, + stmt, +}; + +use crate::engine::mir; + +/// Finds primary keys via a secondary index lookup. +/// +/// Used with NoSQL drivers to locate records by a secondary index, returning +/// the primary keys which can then be used with [`GetByKey`](super::GetByKey). +#[derive(Debug)] +pub(crate) struct FindPkByIndex { + /// Nodes providing input arguments for the filter. + pub(crate) inputs: IndexSet, + + /// The table containing the index. + pub(crate) table: TableId, + + /// The secondary index to query. + pub(crate) index: IndexId, + + /// Filter expression for the index. + pub(crate) filter: stmt::Expr, + + /// The return type (a list of primary keys). + pub(crate) ty: stmt::Type, +} + +impl From for mir::Node { + fn from(value: FindPkByIndex) -> Self { + mir::Operation::FindPkByIndex(value).into() + } +} diff --git a/packages/toasty/src/engine/mir/get_by_key.rs b/packages/toasty/src/engine/mir/get_by_key.rs new file mode 100644 index 00000000000..39ab2e8f948 --- /dev/null +++ b/packages/toasty/src/engine/mir/get_by_key.rs @@ -0,0 +1,34 @@ +use indexmap::IndexSet; +use toasty_core::{schema::db::TableId, stmt}; + +use crate::engine::mir; + +/// Batch fetches records by primary key. +/// +/// Used with NoSQL drivers to retrieve multiple records given a list of +/// primary key values. +/// +/// Keys are always specified as an input node, whether a [`Const`] or the +/// output of a dependent operation. +/// +/// [`Const`]: super::Const +#[derive(Debug)] +pub(crate) struct GetByKey { + /// The node producing the list of primary keys to fetch. + pub(crate) input: mir::NodeId, + + /// The table to fetch records from. + pub(crate) table: TableId, + + /// The columns to include in the returned records. + pub(crate) columns: IndexSet, + + /// The return type. + pub(crate) ty: stmt::Type, +} + +impl From for mir::Node { + fn from(value: GetByKey) -> Self { + mir::Operation::GetByKey(value).into() + } +} diff --git a/packages/toasty/src/engine/mir/guards.rs b/packages/toasty/src/engine/mir/guards.rs new file mode 100644 index 00000000000..4f2f4f14aed --- /dev/null +++ b/packages/toasty/src/engine/mir/guards.rs @@ -0,0 +1,159 @@ +//! Attaches guards to plan nodes. A guard is a condition on a node; when the +//! condition is false at execution time, the node is skipped (see `exec::If`). +//! +//! # Why guards exist +//! +//! A plan often contains nodes whose output only matters when an earlier node +//! returned rows. Consider loading todos together with each todo's user: +//! +//! ```text +//! n0: query todos +//! n1: extract the user ids from n0's rows +//! n2: query users whose id is in n1 +//! n3: map over n0's rows, attaching each todo's user from n2 +//! ``` +//! +//! When n0 returns no rows, the map in n3 iterates zero times, so nothing +//! ever reads n2 — the user query is a wasted database round-trip. This pass +//! guards n1 and n2 with the condition "n0 returned at least one row" +//! (`non_empty(n0)`). At execution time, when n0 comes back empty, n1 and n2 +//! are skipped. +//! +//! # The two rules +//! +//! A node may be skipped only when nothing reads its output. Two rules prove +//! that: +//! +//! 1. A node read [`InputRead::PerRowOf`]`(X)` — only while iterating node +//! X's rows — gets guard `non_empty(X)`. Iterating zero rows never reads +//! the input, so the node's output goes unread whenever X is empty. This +//! is how n2 gets its guard above: n3 is a per-row `Eval` over n0, and n2 +//! is one of its other inputs. +//! +//! 2. A node whose consumers all carry guard `non_empty(X)` inherits that +//! guard. Whenever X is empty, every consumer is skipped, so the node's +//! output goes unread too. This is how n1 gets its guard: its only +//! consumer, n2, is guarded on n0. +//! +//! # When a node gets no guard +//! +//! The analysis is conservative: one consumer that might read the output +//! kills the guard, and the node always executes. In particular: +//! +//! - A node that writes to the database (`Operation::is_effectful`) is never +//! skipped, and never propagates a guard to its inputs. +//! - The completion node's output is the query result; the engine reads it +//! unconditionally. +//! - Consumers guarded on different nodes disagree, so their shared input +//! gets no guard. +//! +//! In the example, n0 stays unguarded: n3 reads it as the row input +//! ([`InputRead::Always`]), and n3 itself has no guard. +//! +//! # How the pass runs +//! +//! The pass keeps one [`Consumption`] value per node. It summarizes how the +//! consumers visited so far read that node's output, and is one of three +//! states: no consumer visited yet ([`Consumption::None`]), every consumer +//! so far reads the output only when node X returned rows +//! ([`Consumption::OnlyUnder`]`(X)`), or some consumer reads the output +//! regardless ([`Consumption::Observed`]). +//! +//! The pass visits nodes in reverse execution order. A node always executes +//! before its consumers, so the reverse walk visits every consumer first, +//! and a node's [`Consumption`] is complete by the time the walk reaches +//! it. Visiting a node does two things: +//! +//! 1. Decide the node's guard from its own [`Consumption`]: when every +//! consumer reads the output only when X returned rows, the node gets +//! guard `non_empty(X)`. +//! +//! 2. Update the [`Consumption`] of each input the node reads, from the +//! operation's declared [`InputRead`]s. A [`PerRowOf`](InputRead::PerRowOf) +//! input contributes "read only when the iterated node returned rows" +//! (rule 1). An [`Always`](InputRead::Always) input is read whenever the +//! node itself runs, so it contributes the node's own `non_empty` guard — +//! or "read regardless" when the node has none (rule 2). +//! +//! In the example, the walk visits n3, n2, n1, n0. n3 declares n2 as +//! `PerRowOf(n0)`, so visiting n3 sets n2's [`Consumption`] to "read only +//! when n0 returned rows". Visiting n2 turns that into guard `non_empty(n0)`, +//! and — since n2 now carries that guard — sets n1's [`Consumption`] to the +//! same state. Visiting n1 turns it into the same guard. +//! +//! The pass runs once, at `LogicalPlan::new`, after planning has produced +//! the complete graph. Guards cannot be assigned earlier, while statements +//! are still being planned: a guard is only valid if every consumer agrees, +//! and planning may still add a consumer that reads the node +//! unconditionally. + +use index_vec::IndexVec; + +use super::{InputRead, NodeId, Store}; + +/// The merge, over a node's already-visited consumers, of how each one +/// consumes the node's output. +#[derive(Clone, Copy)] +enum Consumption { + /// No consumer visited yet. + None, + + /// Every consumer so far is unobservable when the referenced node's + /// output is empty. + OnlyUnder(NodeId), + + /// Some consumer observes the output unconditionally, or consumers + /// disagree on the condition; the node must always execute. + Observed, +} + +pub(super) fn annotate_guards(store: &mut Store, execution_order: &[NodeId], completion: NodeId) { + // Execution position; a guard's condition node must run before the nodes + // it guards so the condition variable exists when the branch is taken. + // Pure nodes unreachable from the completion node are absent from the + // execution order and keep the zero sentinel — never read, since guard + // subjects are inputs of reachable nodes and inputs are deps. + let mut position: IndexVec = IndexVec::from_vec(vec![0; store.node_count()]); + for (i, &id) in execution_order.iter().enumerate() { + position[id] = i; + } + + let mut consumption: IndexVec = + IndexVec::from_vec(vec![Consumption::None; store.node_count()]); + + for &id in execution_order.iter().rev() { + // Effects always run; the completion node is consumed by the engine. + let eligible = !store[id].op.is_effectful() && id != completion; + if eligible + && let Consumption::OnlyUnder(x) = consumption[id] + && position[x] < position[id] + { + store[id].guard = Some(x); + } + + // What this node's `Always` reads contribute to its producers: + // `Some(x)` when the read is unobservable under `non_empty(x)` + // failing because the node itself is guarded on `x` (rule 2), `None` + // otherwise. + let own = store[id].guard; + + // Push onto the value edges — read from operation inputs, never from + // `deps`, so an ordering-only edge never looks like a consumer. + for (input, read) in store[id].op.input_reads() { + let contribution = match read { + // Rule 1: read only while iterating `row_input`'s rows, so + // unobservable whenever `row_input` is empty — regardless + // of the node's own guard. + InputRead::PerRowOf(row_input) => Some(row_input), + InputRead::Always => own, + }; + consumption[input] = match (consumption[input], contribution) { + (Consumption::None, Some(x)) => Consumption::OnlyUnder(x), + (Consumption::OnlyUnder(prev), Some(x)) if prev == x => { + Consumption::OnlyUnder(prev) + } + _ => Consumption::Observed, + }; + } + } +} diff --git a/packages/toasty/src/engine/mir/input.rs b/packages/toasty/src/engine/mir/input.rs new file mode 100644 index 00000000000..46969a1a36e --- /dev/null +++ b/packages/toasty/src/engine/mir/input.rs @@ -0,0 +1,72 @@ +use indexmap::IndexSet; + +use super::{NodeId, Operation}; + +/// How an operation reads one of its inputs. +#[derive(Debug, Clone, Copy)] +pub(crate) enum InputRead { + /// Read whenever the operation runs. + Always, + + /// Read only while iterating the referenced node's rows; when that node + /// returns no rows, the input is never read. + PerRowOf(NodeId), +} + +impl Operation { + /// The operation's value edges — nodes whose outputs it reads — each + /// paired with how the output is read. + /// + /// `Node.deps` is seeded from these nodes and may then be extended with + /// ordering-only edges (e.g. "child INSERT before parent INSERT"). The + /// ordering-only edges are derivable as `deps − inputs()`. + pub(crate) fn input_reads(&self) -> Vec<(NodeId, InputRead)> { + use InputRead::{Always, PerRowOf}; + + match self { + Operation::Alias(m) => vec![(m.input, Always)], + Operation::Const(_m) => vec![], + Operation::DeleteByKey(m) => vec![(m.input, Always)], + Operation::Eval(m) => match m.row_input { + Some(row_input) => [(row_input, Always)] + .into_iter() + .chain(m.inputs.iter().map(|&input| (input, PerRowOf(row_input)))) + .collect(), + None => m.inputs.iter().map(|&input| (input, Always)).collect(), + }, + Operation::ExecStatement(m) => m.inputs.iter().map(|&i| (i, Always)).collect(), + Operation::Filter(m) => [(m.input, Always)] + .into_iter() + .chain(m.args.iter().map(|&a| (a, Always))) + .collect(), + Operation::FindPkByIndex(m) => m.inputs.iter().map(|&i| (i, Always)).collect(), + Operation::GetByKey(m) => vec![(m.input, Always)], + Operation::NestedMerge(m) => m.inputs.iter().map(|&i| (i, Always)).collect(), + Operation::ReadModifyWrite(m) => m.inputs.iter().map(|&i| (i, Always)).collect(), + // The input's cardinality is observed, so the read is + // unconditional even though no row data is used. + Operation::Repeat(m) => vec![(m.input, Always)], + Operation::QueryPk(m) => m.input.into_iter().map(|i| (i, Always)).collect(), + Operation::Scan(m) => m.input.into_iter().map(|i| (i, Always)).collect(), + Operation::UpdateByKey(m) => vec![(m.input, Always)], + Operation::Upsert(m) => m.inputs.iter().map(|&i| (i, Always)).collect(), + } + } + + /// The operation's value edges: nodes whose outputs it reads. + pub(crate) fn inputs(&self) -> IndexSet { + self.input_reads().into_iter().map(|(id, _)| id).collect() + } + + /// The variable loads the operation's execution performs, with + /// multiplicity — currently one per declared input. `num_uses` refcounts + /// are the sum of these loads across consumers, so each operation must + /// load every listed input exactly once — or release it on any path that + /// declines the load. (Guards peek without loading.) + pub(crate) fn input_loads(&self) -> impl Iterator + use<> { + // ReadModifyWrite declares `inputs` but its exec action asserts them + // empty; count them anyway so a future non-empty RMW input must load + // them rather than silently violating the counting. + self.inputs().into_iter() + } +} diff --git a/packages/toasty/src/engine/mir/logical_plan.rs b/packages/toasty/src/engine/mir/logical_plan.rs new file mode 100644 index 00000000000..1bab3f7a3ca --- /dev/null +++ b/packages/toasty/src/engine/mir/logical_plan.rs @@ -0,0 +1,126 @@ +use std::ops; + +use index_vec::IndexVec; + +use crate::engine::mir::{Node, NodeId, Store, annotate_guards}; + +/// The complete operation graph for a query. +/// +/// [`LogicalPlan`] is a directed acyclic graph of operations produced by the +/// planning phase. It contains all nodes, their topologically sorted execution +/// order, and the final completion node whose output is returned to the user. +#[derive(Debug)] +pub(crate) struct LogicalPlan { + /// All nodes in the operation graph. + store: Store, + + /// Topologically sorted order in which to execute operations. + execution_order: Vec, + + /// The final node whose output is the query result. + completion: NodeId, +} + +impl LogicalPlan { + pub(crate) fn new(mut store: Store, completion: NodeId) -> LogicalPlan { + let execution_order = compute_operation_execution_order(completion, &store); + + // Every reserved slot must be filled by the time planning completes — + // an unfilled slot means a statement was referenced but never planned. + debug_assert!( + store.all_filled(), + "reserved MIR slot left unfilled at plan completion" + ); + + // `num_uses` counts the variable loads each node's output receives: + // one per entry in its consumers' `input_loads()`. Ordering-only + // `deps` edges schedule but do not count — no load ever drains them. + // The completion node's exit use is the engine's load of the query + // result. (Guards are annotated after this loop and peek without + // loading.) + store[completion].num_uses += 1; + + for node_id in &execution_order { + for load in store[node_id].op.input_loads() { + store[load].num_uses += 1; + } + } + + annotate_guards(&mut store, &execution_order, completion); + + LogicalPlan { + store, + execution_order, + completion, + } + } + + /// Node IDs in topologically sorted execution order. + pub(crate) fn execution_order(&self) -> &[NodeId] { + &self.execution_order + } + + /// The final node whose output is the query result. + pub(crate) fn completion(&self) -> NodeId { + self.completion + } + + /// Number of node slots; [`NodeId`]s are dense in `0..node_count()`. + pub(crate) fn node_count(&self) -> usize { + self.store.node_count() + } +} + +impl ops::Index for LogicalPlan { + type Output = Node; + + fn index(&self, index: NodeId) -> &Self::Output { + self.store.index(index) + } +} + +impl ops::Index<&NodeId> for LogicalPlan { + type Output = Node; + + fn index(&self, index: &NodeId) -> &Self::Output { + self.store.index(index) + } +} + +fn compute_operation_execution_order(node_id: NodeId, mir: &Store) -> Vec { + fn visit_operation( + node_id: NodeId, + mir: &Store, + visited: &mut IndexVec, + execution_order: &mut Vec, + ) { + if visited[node_id] { + return; + } + + visited[node_id] = true; + + for &dep_id in &mir[node_id].deps { + visit_operation(dep_id, mir, visited, execution_order); + } + + execution_order.push(node_id); + } + + let mut visited = IndexVec::from_vec(vec![false; mir.node_count()]); + let mut execution_order = vec![]; + + visit_operation(node_id, mir, &mut visited, &mut execution_order); + + // Nodes unreachable from the completion node are dropped from the + // execution order and never run. That is fine for pure nodes; a + // dropped mutation would silently lose its database effect. + debug_assert!( + visited + .iter_enumerated() + .all(|(id, visited)| *visited || !mir[id].op.is_effectful()), + "effectful node unreachable from the completion node" + ); + + execution_order +} diff --git a/packages/toasty/src/engine/mir/nested_merge.rs b/packages/toasty/src/engine/mir/nested_merge.rs new file mode 100644 index 00000000000..d589c9d259b --- /dev/null +++ b/packages/toasty/src/engine/mir/nested_merge.rs @@ -0,0 +1,52 @@ +use indexmap::IndexSet; +use toasty_core::stmt; + +use crate::engine::{exec, mir}; + +/// Merges child records into parent records. +/// +/// Used to combine the results of parent and child queries when loading +/// associations (e.g., users with their todos). The merge produces nested +/// records where each parent contains its associated children. +#[derive(Debug)] +pub(crate) struct NestedMerge { + /// The nodes providing parent and child data to merge. + /// [`NestedLevel`](exec::NestedLevel) references them by position. + pub(crate) inputs: IndexSet, + + /// Configuration for how to perform the merge at each nesting level. + pub(crate) root: exec::NestedLevel, + + /// Flat list of hash indexes to build before the merge, computed at plan time. + pub(crate) hash_indexes: Vec, + + /// Flat list of sorted indexes to build before the merge, computed at plan time. + pub(crate) sort_indexes: Vec, + + /// Output type: `List`. + pub(crate) ty: stmt::Type, +} + +impl NestedMerge { + pub(crate) fn new( + inputs: IndexSet, + root: exec::NestedLevel, + hash_indexes: Vec, + sort_indexes: Vec, + ) -> Self { + let ty = stmt::Type::list(root.projection.ret.clone()); + NestedMerge { + inputs, + root, + hash_indexes, + sort_indexes, + ty, + } + } +} + +impl From for mir::Node { + fn from(value: NestedMerge) -> Self { + mir::Operation::NestedMerge(value).into() + } +} diff --git a/packages/toasty/src/engine/mir/node.rs b/packages/toasty/src/engine/mir/node.rs new file mode 100644 index 00000000000..7c655f03747 --- /dev/null +++ b/packages/toasty/src/engine/mir/node.rs @@ -0,0 +1,60 @@ +use indexmap::IndexSet; +use toasty_core::stmt; + +use super::{NodeId, Operation}; + +/// A single node in the MIR operation graph. +/// +/// Each [`Node`] represents one operation to execute. It contains the +/// operation itself, its dependencies on other nodes, and its execution +/// guard. +#[derive(Debug)] +pub(crate) struct Node { + /// The operation this node performs. + pub(crate) op: Operation, + + /// Nodes that must execute before this one. + /// + /// This is a superset of the node's data inputs; it may include additional + /// ordering dependencies (e.g., an `UPDATE` depending on a prior `INSERT`). + pub(crate) deps: IndexSet, + + /// When set, this node only executes if the referenced node produced at + /// least one row, evaluated with a non-consuming peek. Assigned by the + /// guard-annotation pass in [`LogicalPlan::new`], which proves the + /// guarded node's output unobservable when the condition fails; only + /// pure (non-effectful) nodes may carry a guard. + /// + /// [`LogicalPlan::new`]: super::LogicalPlan::new + pub(crate) guard: Option, + + /// Number of variable loads this node's output receives during execution, + /// plus one exit use when this is the completion node. + /// + /// [`LogicalPlan::new`](super::LogicalPlan::new) sets this after planning + /// finishes. The executor uses it for reference counting and frees the + /// output after the last use. + pub(crate) num_uses: usize, +} + +impl Node { + pub(crate) fn ty(&self) -> &stmt::Type { + match &self.op { + Operation::Alias(m) => &m.ty, + Operation::Const(m) => &m.ty, + Operation::DeleteByKey(m) => &m.ty, + Operation::Eval(m) => &m.body.ret, + Operation::ExecStatement(m) => &m.ty, + Operation::Filter(m) => &m.ty, + Operation::FindPkByIndex(m) => &m.ty, + Operation::GetByKey(m) => &m.ty, + Operation::QueryPk(m) => &m.ty, + Operation::Repeat(m) => &m.ty, + Operation::Scan(m) => &m.ty, + Operation::UpdateByKey(m) => &m.ty, + Operation::Upsert(m) => &m.ty, + Operation::NestedMerge(m) => &m.ty, + Operation::ReadModifyWrite(m) => &m.ty, + } + } +} diff --git a/packages/toasty/src/engine/mir/operation.rs b/packages/toasty/src/engine/mir/operation.rs new file mode 100644 index 00000000000..5933c60a27f --- /dev/null +++ b/packages/toasty/src/engine/mir/operation.rs @@ -0,0 +1,150 @@ +use crate::engine::effect; + +use super::{ + Alias, Const, DeleteByKey, Eval, ExecStatement, Filter, FindPkByIndex, GetByKey, NestedMerge, + Node, QueryPk, ReadModifyWrite, Repeat, Scan, UpdateByKey, Upsert, +}; + +/// A step in the query execution plan. +/// +/// Operations represent units of work: loading data from the database, +/// filtering results, transforming records, or combining nested data. +#[derive(Debug)] +pub(crate) enum Operation { + /// Pass another node's output through unchanged (fills a reserved slot) + Alias(Alias), + + /// A constant value + Const(Const), + + DeleteByKey(DeleteByKey), + + /// Evaluate a function, once over whole input values or once per row of + /// a row input + Eval(Eval), + + /// Execute a database query + ExecStatement(Box), + + /// Filter results + Filter(Filter), + + /// Find primary keys by index + FindPkByIndex(FindPkByIndex), + + /// Get records by primary key + GetByKey(GetByKey), + + /// Execute a nested merge + NestedMerge(NestedMerge), + + /// Read-modify-write. The write only succeeds if the values read are not + /// modified. + ReadModifyWrite(Box), + + /// Produce a constant value once per input row + Repeat(Repeat), + + QueryPk(QueryPk), + + /// Full-table scan — emitted when no index covers the filter on a scan-capable driver. + Scan(Scan), + + UpdateByKey(UpdateByKey), + + /// Atomically insert or update one record on a non-SQL database. + Upsert(Box), +} + +impl Operation { + /// Returns the operation's variant name for logging. + pub(crate) fn name(&self) -> &'static str { + match self { + Operation::Alias(_) => "alias", + Operation::Const(_) => "const", + Operation::DeleteByKey(_) => "delete_by_key", + Operation::Eval(_) => "eval", + Operation::ExecStatement(_) => "exec_statement", + Operation::Filter(_) => "filter", + Operation::FindPkByIndex(_) => "find_pk_by_index", + Operation::GetByKey(_) => "get_by_key", + Operation::NestedMerge(_) => "nested_merge", + Operation::ReadModifyWrite(_) => "read_modify_write", + Operation::Repeat(_) => "repeat", + Operation::QueryPk(_) => "query_pk", + Operation::Scan(_) => "scan", + Operation::UpdateByKey(_) => "update_by_key", + Operation::Upsert(_) => "upsert", + } + } + + /// True for operations that issue a driver operation. + /// + /// Used to determine whether a plan needs to be wrapped in a transaction. + /// Distinct from [`Operation::is_effectful`]: queries are db ops but not + /// effectful. In-memory operations count zero. + pub(crate) fn is_db_op(&self) -> bool { + match self { + Operation::DeleteByKey(_) + | Operation::ExecStatement(_) + | Operation::FindPkByIndex(_) + | Operation::GetByKey(_) + | Operation::QueryPk(_) + | Operation::ReadModifyWrite(_) + | Operation::Scan(_) + | Operation::UpdateByKey(_) + | Operation::Upsert(_) => true, + + Operation::Alias(_) + | Operation::Const(_) + | Operation::Eval(_) + | Operation::Filter(_) + | Operation::NestedMerge(_) + | Operation::Repeat(_) => false, + } + } + + /// True for operations that write to the database. + /// + /// Distinct from `exec::Action::is_db_op`, which answers "issues a driver + /// operation" for transaction wrapping — queries are db ops but not + /// effectful. Only pure (non-effectful) nodes may be guarded, and every + /// effectful node must be reachable from the completion node. + pub(crate) fn is_effectful(&self) -> bool { + match self { + Operation::DeleteByKey(_) + | Operation::ReadModifyWrite(_) + | Operation::UpdateByKey(_) + | Operation::Upsert(_) => true, + + // `classify` walks the statement tree, so a `Query` wrapping a + // data-modifying CTE (the OCC conditional-write path) classifies + // as mutating even though its statement kind is a read. + Operation::ExecStatement(m) => effect::classify(&m.stmt) == effect::Effect::Mutating, + + Operation::Alias(_) + | Operation::Const(_) + | Operation::Eval(_) + | Operation::Filter(_) + | Operation::FindPkByIndex(_) + | Operation::GetByKey(_) + | Operation::NestedMerge(_) + | Operation::QueryPk(_) + | Operation::Repeat(_) + | Operation::Scan(_) => false, + } + } +} + +impl From for Node { + fn from(value: Operation) -> Self { + let deps = value.inputs(); + + Node { + op: value, + deps, + guard: None, + num_uses: 0, + } + } +} diff --git a/packages/toasty/src/engine/mir/query_pk.rs b/packages/toasty/src/engine/mir/query_pk.rs new file mode 100644 index 00000000000..794eed08039 --- /dev/null +++ b/packages/toasty/src/engine/mir/query_pk.rs @@ -0,0 +1,48 @@ +use indexmap::IndexSet; +use toasty_core::{ + driver::operation::Pagination, + schema::db::{IndexId, TableId}, + stmt, +}; + +use crate::engine::mir; + +/// Queries records using a primary key filter. +/// +/// Used with NoSQL drivers to query a table's primary key index with optional +/// additional row filtering. +#[derive(Debug)] +pub(crate) struct QueryPk { + /// Optional node providing input arguments for the filter. + pub(crate) input: Option, + + /// The table to query. + pub(crate) table: TableId, + + /// Optional index to query. None = primary key, Some(id) = secondary index + pub(crate) index: Option, + + /// The columns to include in the returned records. + pub(crate) columns: IndexSet, + + /// Filter expression for the primary key index. + pub(crate) pk_filter: stmt::Expr, + + /// Additional filter applied to matching rows. + pub(crate) row_filter: Option, + + /// The return type. + pub(crate) ty: stmt::Type, + + /// Limit and pagination bounds for this query. `None` means unbounded. + pub(crate) limit: Option, + + /// Sort key ordering direction. + pub(crate) order: Option, +} + +impl From for mir::Node { + fn from(value: QueryPk) -> Self { + mir::Operation::QueryPk(value).into() + } +} diff --git a/packages/toasty/src/engine/mir/read_modify_write.rs b/packages/toasty/src/engine/mir/read_modify_write.rs new file mode 100644 index 00000000000..e5cd89b248e --- /dev/null +++ b/packages/toasty/src/engine/mir/read_modify_write.rs @@ -0,0 +1,33 @@ +use indexmap::IndexSet; +use toasty_core::stmt; + +use crate::engine::mir; + +/// Performs an optimistic read-modify-write operation. +/// +/// Used for conditional updates where the write only succeeds if the values +/// read have not been modified since reading. This is a fallback for databases +/// that do not support conditional updates in a single statement (e.g., SQLite, +/// MySQL without CTE support). +#[derive(Debug)] +pub(crate) struct ReadModifyWrite { + /// Nodes providing input arguments for the statements. + pub(crate) inputs: IndexSet, + + /// The read query that fetches current values. + pub(crate) read: stmt::Query, + + /// The write statement to execute if the condition holds. + pub(crate) write: stmt::Statement, + + /// The return type. When the write carries a `RETURNING`, this is + /// `List`; without one the type is `Unit` and the write reports + /// only a row count. + pub(crate) ty: stmt::Type, +} + +impl From for mir::Node { + fn from(value: ReadModifyWrite) -> Self { + mir::Operation::ReadModifyWrite(Box::new(value)).into() + } +} diff --git a/packages/toasty/src/engine/mir/repeat.rs b/packages/toasty/src/engine/mir/repeat.rs new file mode 100644 index 00000000000..acdd79eb0b6 --- /dev/null +++ b/packages/toasty/src/engine/mir/repeat.rs @@ -0,0 +1,29 @@ +use toasty_core::stmt; + +use crate::engine::mir; + +/// Produces `value` once per row of `input`. +/// +/// The input supplies only a cardinality — its rows are never read, and a +/// write that reports just an affected-row count works the same as one that +/// returns rows. Used for a returning clause whose value is fully known +/// client-side (e.g. an UPDATE whose assignments were all literals): the +/// database is not asked for any columns, and the result is the value +/// repeated once per affected row. +#[derive(Debug)] +pub(crate) struct Repeat { + /// The node whose cardinality drives the repetition. + pub(crate) input: mir::NodeId, + + /// The value produced for each input row. + pub(crate) value: stmt::Value, + + /// Output type: `List`. + pub(crate) ty: stmt::Type, +} + +impl From for mir::Node { + fn from(value: Repeat) -> Self { + mir::Operation::Repeat(value).into() + } +} diff --git a/packages/toasty/src/engine/mir/scan.rs b/packages/toasty/src/engine/mir/scan.rs new file mode 100644 index 00000000000..f3d1d11afd2 --- /dev/null +++ b/packages/toasty/src/engine/mir/scan.rs @@ -0,0 +1,36 @@ +use indexmap::IndexSet; +use toasty_core::{driver::operation::Pagination, schema::db::TableId, stmt}; + +use crate::engine::mir; + +/// Performs a full-table scan with optional filter, limit, and pagination. +/// +/// `Scan` is emitted by the planner when no index covers the query filter on a +/// DynamoDB-backed model. The driver applies `row_filter` to each scanned row +/// before returning results. +#[derive(Debug)] +pub(crate) struct Scan { + /// Optional node providing input arguments for the filter expression. + pub(crate) input: Option, + + /// The table to scan. + pub(crate) table: TableId, + + /// The columns to include in the returned records. + pub(crate) columns: IndexSet, + + /// Filter expression applied to each scanned row. + pub(crate) row_filter: Option, + + /// Limit and pagination bounds. `None` means return all rows. + pub(crate) limit: Option, + + /// The return type. + pub(crate) ty: stmt::Type, +} + +impl From for mir::Node { + fn from(value: Scan) -> Self { + mir::Operation::Scan(value).into() + } +} diff --git a/packages/toasty/src/engine/mir/store.rs b/packages/toasty/src/engine/mir/store.rs new file mode 100644 index 00000000000..3716c39c893 --- /dev/null +++ b/packages/toasty/src/engine/mir/store.rs @@ -0,0 +1,132 @@ +use std::ops; + +use index_vec::IndexVec; +use toasty_core::stmt; + +use super::Node; + +/// Storage for MIR operation nodes. +/// +/// [`Store`] is an arena that holds all [`Node`]s in the operation graph, +/// indexed by [`NodeId`]. It provides insertion and lookup operations used +/// during planning to build the graph incrementally. +/// +/// A slot may be reserved before its node exists: [`Store::reserve`] hands +/// out a [`NodeId`] that other nodes can reference immediately, and +/// [`Store::fill`] supplies the node later — typically an +/// [`Alias`](super::Alias) pass-through to whatever node planning actually +/// produced. This lets a statement be referenced (e.g. by an ordering edge +/// from a descendant) before it is planned. Every reserved slot must be +/// filled before the graph is used. +#[derive(Debug)] +pub(crate) struct Store { + /// All slots in the graph, indexed by [`NodeId`]. + slots: IndexVec, +} + +// Boxing `Node` to shrink the (transient, rare) `Reserved` variant would cost +// an allocation and an indirection on every node access. +#[expect(clippy::large_enum_variant)] +#[derive(Debug)] +enum Slot { + /// Allocated by [`Store::reserve`]; awaiting its node. + Reserved, + + /// A filled slot holding its node. + Node(Node), +} + +index_vec::define_index_type! { + pub(crate) struct NodeId = u32; +} + +impl Store { + pub(crate) fn new() -> Store { + Store { + slots: IndexVec::new(), + } + } + + /// Insert a node into the graph + pub(crate) fn insert(&mut self, node: impl Into) -> NodeId { + self.slots.push(Slot::Node(node.into())) + } + + pub(crate) fn insert_with_deps(&mut self, node: impl Into, deps: I) -> NodeId + where + I: IntoIterator, + { + let mut node = node.into(); + node.deps.extend(deps); + self.slots.push(Slot::Node(node)) + } + + /// Allocate a slot whose node is supplied later via [`Store::fill`]. + pub(crate) fn reserve(&mut self) -> NodeId { + self.slots.push(Slot::Reserved) + } + + /// Fill a reserved slot with its node. + #[track_caller] + pub(crate) fn fill(&mut self, id: NodeId, node: impl Into) { + let slot = &mut self.slots[id]; + assert!( + matches!(slot, Slot::Reserved), + "fill of non-reserved slot {id:?}" + ); + *slot = Slot::Node(node.into()); + } + + /// True when no reserved slot is left unfilled. + pub(crate) fn all_filled(&self) -> bool { + self.slots.iter().all(|slot| matches!(slot, Slot::Node(_))) + } + + /// Number of slots, filled or reserved. [`NodeId`]s are dense in + /// `0..node_count()`, so this sizes per-node side tables. + pub(crate) fn node_count(&self) -> usize { + self.slots.len() + } + + pub(crate) fn ty(&self, node_id: NodeId) -> &stmt::Type { + self[node_id].ty() + } +} + +impl ops::Index for Store { + type Output = Node; + + #[track_caller] + fn index(&self, index: NodeId) -> &Self::Output { + match &self.slots[index] { + Slot::Node(node) => node, + Slot::Reserved => panic!("reserved MIR slot {index:?} read before fill"), + } + } +} + +impl ops::IndexMut for Store { + #[track_caller] + fn index_mut(&mut self, index: NodeId) -> &mut Self::Output { + match &mut self.slots[index] { + Slot::Node(node) => node, + Slot::Reserved => panic!("reserved MIR slot {index:?} read before fill"), + } + } +} + +impl ops::Index<&NodeId> for Store { + type Output = Node; + + #[track_caller] + fn index(&self, index: &NodeId) -> &Self::Output { + self.index(*index) + } +} + +impl ops::IndexMut<&NodeId> for Store { + #[track_caller] + fn index_mut(&mut self, index: &NodeId) -> &mut Self::Output { + self.index_mut(*index) + } +} diff --git a/packages/toasty/src/engine/mir/update_by_key.rs b/packages/toasty/src/engine/mir/update_by_key.rs new file mode 100644 index 00000000000..882fbf4f359 --- /dev/null +++ b/packages/toasty/src/engine/mir/update_by_key.rs @@ -0,0 +1,43 @@ +use indexmap::IndexSet; +use toasty_core::{schema::db::TableId, stmt}; + +use crate::engine::mir; + +/// Updates records by primary key. +/// +/// Used with NoSQL drivers to update records given a list of primary key values. +/// +/// Keys are always specified as an input node, whether a [`Const`] or the +/// output of a dependent operation. +/// +/// [`Const`]: super::Const +#[derive(Debug)] +pub(crate) struct UpdateByKey { + /// The node producing the list of primary keys to update. + pub(crate) input: mir::NodeId, + + /// The table to update records in. + pub(crate) table: TableId, + + /// The field assignments to apply. + pub(crate) assignments: stmt::Assignments, + + /// Optional additional filter applied before update. + pub(crate) filter: Option, + + /// Optional condition for optimistic locking. + pub(crate) condition: Option, + + /// The columns to return for each updated row. Empty when the update + /// returns only an affected-row count (the type is then `Unit`). + pub(crate) columns: IndexSet, + + /// The return type. + pub(crate) ty: stmt::Type, +} + +impl From for mir::Node { + fn from(value: UpdateByKey) -> Self { + mir::Operation::UpdateByKey(value).into() + } +} diff --git a/packages/toasty/src/engine/mir/upsert.rs b/packages/toasty/src/engine/mir/upsert.rs new file mode 100644 index 00000000000..9a016854ba4 --- /dev/null +++ b/packages/toasty/src/engine/mir/upsert.rs @@ -0,0 +1,26 @@ +use indexmap::IndexSet; +use toasty_core::stmt; + +use crate::engine::mir; + +/// Executes a single-row upsert on a non-SQL database. +/// +/// The planner emits this operation after lowering the conflict target and +/// verifying the requested upsert behavior against the driver's capabilities. +#[derive(Debug)] +pub(crate) struct Upsert { + /// Nodes whose outputs are passed as arguments to the statement. + pub(crate) inputs: IndexSet, + + /// The lowered insert and conflict action. + pub(crate) stmt: stmt::Insert, + + /// The return type of this operation. + pub(crate) ty: stmt::Type, +} + +impl From for mir::Node { + fn from(value: Upsert) -> Self { + mir::Operation::Upsert(Box::new(value)).into() + } +} diff --git a/packages/toasty/src/engine/normalize.rs b/packages/toasty/src/engine/normalize.rs new file mode 100644 index 00000000000..12bc8f99824 --- /dev/null +++ b/packages/toasty/src/engine/normalize.rs @@ -0,0 +1,93 @@ +mod paginate; +mod upsert; +mod variant_guards; + +#[cfg(test)] +mod tests; + +use toasty_core::{ + Error, Result, Schema, + driver::Capability, + stmt::{self, VisitMut}, +}; + +use super::Engine; + +/// Normalizes application-level statements in one recursive traversal. +/// +/// Node-specific helpers may inspect or mutate the node they receive, but they +/// must not recurse. The [`VisitMut`] implementation owns traversal so adding a +/// normalization rule does not add another full AST walk. +struct Normalize<'a> { + schema: &'a Schema, + capability: &'a Capability, + error: Option, + + /// Variant guards the enclosing conjunctions state, innermost last. + /// + /// A predicate inside `is_variant(x, v) AND ..` need not repeat that + /// guard. The stack is scoped to the statement being walked: a subquery + /// starts empty. + guards: Vec, +} + +impl Engine { + pub(super) fn normalize_stmt(&self, stmt: &mut stmt::Statement) -> Result<()> { + let mut normalize = Normalize { + schema: &self.schema, + capability: self.capability, + error: None, + guards: vec![], + }; + normalize.visit_stmt_mut(stmt); + + match normalize.error { + Some(error) => Err(error), + None => Ok(()), + } + } +} + +impl Normalize<'_> { + fn record(&mut self, error: Error) { + if self.error.is_none() { + self.error = Some(error); + } + } + + /// Walks a nested statement with its own guard scope. + fn scoped(&mut self, walk: impl FnOnce(&mut Self)) { + let guards = std::mem::take(&mut self.guards); + walk(self); + self.guards = guards; + } +} + +impl VisitMut for Normalize<'_> { + fn visit_expr_mut(&mut self, expr: &mut stmt::Expr) { + if let stmt::Expr::And(and) = expr { + self.normalize_conjunction(and); + } else { + stmt::visit_mut::visit_expr_mut(self, expr); + self.normalize_predicate_guards(expr); + } + } + + fn visit_stmt_mut(&mut self, stmt: &mut stmt::Statement) { + self.scoped(|this| stmt::visit_mut::visit_stmt_mut(this, stmt)); + } + + fn visit_stmt_insert_mut(&mut self, insert: &mut stmt::Insert) { + self.normalize_upsert_defaults(insert); + if self.error.is_none() { + stmt::visit_mut::visit_stmt_insert_mut(self, insert); + } + } + + fn visit_stmt_query_mut(&mut self, query: &mut stmt::Query) { + self.normalize_cursor_order(query); + if self.error.is_none() { + self.scoped(|this| stmt::visit_mut::visit_stmt_query_mut(this, query)); + } + } +} diff --git a/packages/toasty/src/engine/normalize/paginate.rs b/packages/toasty/src/engine/normalize/paginate.rs new file mode 100644 index 00000000000..f8a878ae4c2 --- /dev/null +++ b/packages/toasty/src/engine/normalize/paginate.rs @@ -0,0 +1,107 @@ +use hashbrown::HashSet; +use toasty_core::{ + schema::db::ColumnId, + stmt::{self, ExprReference}, +}; + +use super::Normalize; + +impl Normalize<'_> { + /// Appends missing physical primary-key columns to ambiguous SQL cursor + /// ordering. + /// + /// The public query retains only the user-provided ordering. Normalization + /// runs on the cloned statement passed to the engine, making these + /// tie-breakers internal to execution and cursor generation. Physical + /// column references keep embedded newtype keys scalar after lowering. + pub(super) fn normalize_cursor_order(&mut self, query: &mut stmt::Query) { + if !self.capability.sql() || !matches!(query.limit, Some(stmt::Limit::Cursor(_))) { + return; + } + + let Some(order_by) = &mut query.order_by else { + return; + }; + if order_by.exprs.is_empty() { + return; + } + let Some(select) = query.body.as_select() else { + return; + }; + let Some(model_id) = select.source.model_id() else { + return; + }; + + let mapping = self.schema.mapping.model(model_id); + let table = self.schema.db.table(mapping.table); + let mut ordered_columns: HashSet<_> = order_by + .exprs + .iter() + .filter_map(|order| resolve_order_column(mapping, &order.expr)) + .collect(); + + let unambiguous = table.indices.iter().any(|index| { + (index.unique || index.primary_key) + && index + .columns + .iter() + .all(|index_column| ordered_columns.contains(&index_column.column)) + && index + .columns + .iter() + .all(|index_column| !table.column(index_column.column).nullable) + }); + if unambiguous { + return; + } + + let direction = order_by.exprs.last().and_then(|order| order.order); + for column in &table.primary_key.columns { + if !ordered_columns.insert(*column) { + continue; + } + + order_by.exprs.push(stmt::OrderByExpr { + expr: stmt::Expr::column(*column), + order: direction, + }); + } + } +} + +/// Resolves a model field path used by `order_by` to its single physical +/// column. Expressions that do not resolve to exactly one column cannot prove +/// that an order is unique. +fn resolve_order_column( + mapping: &toasty_core::schema::mapping::Model, + expr: &stmt::Expr, +) -> Option { + if let stmt::Expr::Reference(ExprReference::Column(column)) = expr { + return (column.nesting == 0 && column.table == 0).then_some(ColumnId { + table: mapping.table, + index: column.column, + }); + } + + let projection = field_projection(expr)?; + let field = mapping.resolve_field_mapping(&projection)?; + let mut columns = field.columns(); + let (column, _) = columns.next()?; + columns.next().is_none().then_some(column) +} + +fn field_projection(expr: &stmt::Expr) -> Option { + match expr { + stmt::Expr::Reference(ExprReference::Field { nesting: 0, index }) => { + Some(stmt::Projection::single(*index)) + } + stmt::Expr::Project(project) => { + let mut projection = field_projection(&project.base)?; + for step in &project.projection { + projection.push(step); + } + Some(projection) + } + _ => None, + } +} diff --git a/packages/toasty/src/engine/normalize/tests.rs b/packages/toasty/src/engine/normalize/tests.rs new file mode 100644 index 00000000000..5ca57549773 --- /dev/null +++ b/packages/toasty/src/engine/normalize/tests.rs @@ -0,0 +1,46 @@ +mod variant_guards; + +use std::sync::Arc; + +use toasty_core::{ + driver::Capability, + stmt::{self, Direction, Expr, Limit, LimitCursor, OrderByExpr, Statement}, +}; + +use crate as toasty; +use crate::{ + engine::{Engine, test_util::test_schema_with}, + schema::Model, +}; + +#[test] +fn normalization_is_idempotent() { + #[derive(Debug, toasty::Model)] + struct Item { + #[key] + id: i64, + group: i64, + } + + let schema = Arc::new(test_schema_with(&[Item::schema()])); + let engine = Engine::new(schema, &Capability::SQLITE); + let mut query = stmt::Query::new_select(Item::id(), true); + query.order_by = Some( + OrderByExpr { + expr: Expr::ref_self_field(Item::id().field(1)), + order: Some(Direction::Asc), + } + .into(), + ); + query.limit = Some(Limit::Cursor(LimitCursor { + page_size: 2_i64.into(), + after: None, + })); + let mut stmt = Statement::Query(query); + + engine.normalize_stmt(&mut stmt).unwrap(); + let once = stmt.clone(); + engine.normalize_stmt(&mut stmt).unwrap(); + + assert_eq!(stmt, once); +} diff --git a/packages/toasty/src/engine/normalize/tests/variant_guards.rs b/packages/toasty/src/engine/normalize/tests/variant_guards.rs new file mode 100644 index 00000000000..f089d8932ea --- /dev/null +++ b/packages/toasty/src/engine/normalize/tests/variant_guards.rs @@ -0,0 +1,318 @@ +use std::sync::Arc; + +use toasty_core::{ + driver::Capability, + schema::app::VariantId, + stmt::{self, Expr, Query, Statement}, +}; + +use crate as toasty; +use crate::{ + engine::{Engine, test_util::test_schema_with}, + schema::{Embed, Model}, +}; + +#[derive(Debug, toasty::Embed)] +enum Inner { + A { value: i64 }, + B, +} + +#[derive(Debug, toasty::Embed)] +enum Owner { + Primary { + #[shared(id)] + id: i64, + note: Option, + inner: Inner, + }, + Other { + #[shared(id)] + id: i64, + }, +} + +#[derive(Debug, toasty::Model)] +struct Object { + #[key] + id: i64, + lhs: Owner, + rhs: Owner, +} + +const PRIMARY: usize = 0; + +fn engine() -> Engine { + let schema = Arc::new(test_schema_with(&[ + Object::schema(), + Owner::schema(), + Inner::schema(), + ])); + Engine::new(schema, &Capability::SQLITE) +} + +fn normalize_stmt(stmt: &mut Statement) { + engine().normalize_stmt(stmt).unwrap(); +} + +/// Normalizes a query on `Object` filtered by `filter` and returns the +/// normalized filter. +fn normalize(filter: impl Into) -> Expr { + let mut stmt = Statement::Query(Query::new_select(Object::id(), filter.into())); + normalize_stmt(&mut stmt); + filter_of(&stmt) +} + +fn filter_of(stmt: &Statement) -> Expr { + let select = stmt.as_query().unwrap().body.as_select_unwrap(); + select.filter.expr.clone().unwrap() +} + +fn path_expr(path: impl Into>) -> Expr { + stmt::Path::from(path.into()).into_stmt() +} + +fn owner(index: usize) -> VariantId { + VariantId { + model: Owner::id(), + index, + } +} + +fn lhs_is_primary() -> Expr { + Expr::is_variant(path_expr(Object::fields().lhs()), owner(PRIMARY)) +} + +fn rhs_is_primary() -> Expr { + Expr::is_variant(path_expr(Object::fields().rhs()), owner(PRIMARY)) +} + +fn lhs_id() -> Expr { + path_expr(Object::fields().lhs().primary().id()) +} + +fn lhs_note() -> Expr { + path_expr(Object::fields().lhs().primary().note()) +} + +#[test] +fn comparison_requires_the_selected_variant() { + let filter = normalize(Object::fields().lhs().primary().id().eq(1)); + + assert_eq!(filter, Expr::and(lhs_is_primary(), Expr::eq(lhs_id(), 1))); +} + +#[test] +fn guards_are_collected_from_both_operands() { + let rhs_id = || path_expr(Object::fields().rhs().primary().id()); + let filter = normalize( + Object::fields() + .lhs() + .primary() + .id() + .ne(Object::fields().rhs().primary().id()), + ); + + assert_eq!( + filter, + Expr::and_from_vec(vec![ + lhs_is_primary(), + rhs_is_primary(), + Expr::ne(lhs_id(), rhs_id()), + ]) + ); +} + +#[test] +fn identical_selections_guard_once() { + let filter = normalize(Expr::eq(lhs_id(), lhs_id())); + + assert_eq!( + filter, + Expr::and(lhs_is_primary(), Expr::eq(lhs_id(), lhs_id())) + ); +} + +#[test] +fn nested_selections_guard_every_enclosing_variant() { + let inner_is_a = Expr::is_variant( + path_expr(Object::fields().lhs().primary().inner()), + VariantId { + model: Inner::id(), + index: 0, + }, + ); + let value = path_expr(Object::fields().lhs().primary().inner().a().value()); + let filter = normalize(Expr::eq(value.clone(), 1)); + + // Outermost variant first: the inner check is only meaningful once the + // outer variant holds. + assert_eq!( + filter, + Expr::and_from_vec(vec![lhs_is_primary(), inner_is_a, Expr::eq(value, 1)]) + ); +} + +#[test] +fn negating_a_predicate_negates_the_guarded_predicate() { + let filter = normalize(Object::fields().lhs().primary().id().eq(1).not()); + + assert_eq!( + filter, + Expr::not(Expr::and(lhs_is_primary(), Expr::eq(lhs_id(), 1))) + ); +} + +#[test] +fn negative_null_check_is_guarded_as_a_whole() { + // `is_some()` is a predicate: the guard applies to it. + let filter = normalize(Object::fields().lhs().primary().note().is_some()); + assert_eq!( + filter, + Expr::and(lhs_is_primary(), Expr::not(Expr::is_null(lhs_note()))) + ); + + // `is_none().not()` negates the guarded predicate instead. + let filter = normalize(Object::fields().lhs().primary().note().is_none().not()); + assert_eq!( + filter, + Expr::not(Expr::and(lhs_is_primary(), Expr::is_null(lhs_note()))) + ); +} + +#[test] +fn negative_membership_is_guarded_as_a_whole() { + let subquery = || { + Query::new_select( + Object::id(), + Expr::eq(Expr::ref_self_field(Object::id().field(0)), 1), + ) + }; + let filter = normalize(Expr::not_in_subquery(lhs_id(), subquery())); + + assert_eq!( + filter, + Expr::and( + lhs_is_primary(), + Expr::not(Expr::in_subquery(lhs_id(), subquery())) + ) + ); +} + +#[test] +fn nested_predicate_is_guarded_at_its_own_boundary() { + let filter = normalize(Object::fields().lhs().primary().note().is_none().eq(false)); + + assert_eq!( + filter, + Expr::eq( + Expr::and(lhs_is_primary(), Expr::is_null(lhs_note())), + false + ) + ); +} + +#[test] +fn subquery_predicates_keep_their_own_guards() { + let outer = Expr::ref_self_field(Object::id().field(0)); + let guarded_subquery = || { + Query::new_select( + Object::id(), + Expr::and(lhs_is_primary(), Expr::eq(lhs_id(), 1)), + ) + }; + + // The selection inside the subquery is guarded there, not outside. + let membership = Expr::in_subquery( + outer.clone(), + Query::new_select(Object::id(), Expr::eq(lhs_id(), 1)), + ); + assert_eq!( + normalize(membership), + Expr::in_subquery(outer.clone(), guarded_subquery()) + ); + + // A guard in force outside the subquery does not reach into it. + let membership = Expr::in_subquery( + outer.clone(), + Query::new_select(Object::id(), Expr::eq(lhs_id(), 1)), + ); + assert_eq!( + normalize(Expr::and(lhs_is_primary(), membership)), + Expr::and( + lhs_is_primary(), + Expr::in_subquery(outer, guarded_subquery()) + ) + ); +} + +#[test] +fn guards_flatten_into_the_enclosing_conjunction() { + let unrelated = || Expr::eq(Expr::ref_self_field(Object::id().field(0)), 1); + let rhs_id = || path_expr(Object::fields().rhs().primary().id()); + let filter = normalize(Expr::and_from_vec(vec![ + unrelated(), + Expr::eq(lhs_id(), 1), + Expr::eq(rhs_id(), 2), + ])); + + // Each guard is a sibling of the comparison it scopes. + assert_eq!( + filter, + Expr::and_from_vec(vec![ + unrelated(), + lhs_is_primary(), + Expr::eq(lhs_id(), 1), + rhs_is_primary(), + Expr::eq(rhs_id(), 2), + ]) + ); +} + +#[test] +fn conjunction_does_not_repeat_a_guard_it_states() { + // `matches()` combines the variant check with its body explicitly, so + // normalization has nothing to add. + let filter = Object::fields() + .lhs() + .primary() + .matches(|primary| primary.id().eq(1).and(primary.id().ne(2))); + let explicit = Expr::from(filter.clone()); + let Expr::And(operands) = &explicit else { + panic!("expected a conjunction, got {explicit:?}"); + }; + assert_eq!(operands.len(), 3); + + assert_eq!(normalize(filter), explicit); +} + +#[test] +fn ordering_selections_stay_unguarded() { + let mut query = Query::new_select(Object::id(), true); + query.order_by = Some(Object::fields().lhs().primary().id().asc().into()); + let mut stmt = Statement::Query(query); + let before = stmt.clone(); + + normalize_stmt(&mut stmt); + + assert_eq!(stmt, before); +} + +#[test] +fn guard_normalization_is_idempotent() { + let filter = Object::fields() + .lhs() + .primary() + .id() + .ne(Object::fields().rhs().primary().id()) + .and(Object::fields().lhs().primary().note().is_some()) + .and(Object::fields().lhs().primary().id().eq(1).not()) + .or(Object::fields().lhs().primary().inner().a().value().eq(2)); + let mut stmt = Statement::Query(Query::new_select(Object::id(), filter)); + + normalize_stmt(&mut stmt); + let once = stmt.clone(); + normalize_stmt(&mut stmt); + + assert_eq!(stmt, once); +} diff --git a/packages/toasty/src/engine/normalize/upsert.rs b/packages/toasty/src/engine/normalize/upsert.rs new file mode 100644 index 00000000000..43ec00a597e --- /dev/null +++ b/packages/toasty/src/engine/normalize/upsert.rs @@ -0,0 +1,35 @@ +use toasty_core::stmt; + +use super::Normalize; + +impl Normalize<'_> { + /// Routes model `#[update]` values to branches that do not have an explicit + /// assignment. Model `#[default]` values stay separate until lower-stage + /// upsert normalization so shared mutations can use them as their initial + /// value. + pub(super) fn normalize_upsert_defaults(&mut self, insert: &mut stmt::Insert) { + let Some(upsert) = &mut insert.upsert else { + return; + }; + + for (projection, assignment) in std::mem::take(&mut upsert.update_defaults) { + let stmt::Assignment::Set(expr) = assignment else { + self.record(crate::Error::invalid_statement( + "upsert field defaults only support value assignments", + )); + return; + }; + let create = upsert.defaults.contains(&projection) + || upsert.shared.contains(&projection) + || upsert.create.contains(&projection); + let update = upsert.shared.contains(&projection) || upsert.update.contains(&projection); + + match (create, update) { + (false, false) => upsert.shared.set(projection, expr), + (false, true) => upsert.create.set(projection, expr), + (true, false) => upsert.update.set(projection, expr), + (true, true) => {} + } + } + } +} diff --git a/packages/toasty/src/engine/normalize/variant_guards.rs b/packages/toasty/src/engine/normalize/variant_guards.rs new file mode 100644 index 00000000000..3484df1df3c --- /dev/null +++ b/packages/toasty/src/engine/normalize/variant_guards.rs @@ -0,0 +1,145 @@ +//! Attaches the variant checks a predicate over an enum variant requires. +//! +//! A path into an enum variant (`contact().email().address()`) converts to an +//! [`ExprVariant`] selection: the variant's payload, without a check that the +//! value holds that variant. A predicate over such an operand only means +//! something for rows of that variant, so normalization conjoins an +//! `is_variant` check for every selection a predicate's operands reach — on +//! either operand, and for every enclosing variant of a nested selection, +//! outermost first. +//! +//! Guards attach at each predicate boundary: the comparison, null check, +//! membership test, or other boolean operator whose operands hold the +//! selection. Boolean connectives and subqueries are boundaries of their own, +//! so `a.eq(b).not()` becomes `NOT (is_variant AND a = b)` and negates the +//! guarded comparison as a whole, while `a.is_some()` — the `IS NOT NULL` +//! predicate — becomes `is_variant AND NOT (a IS NULL)`. A selection used as +//! a value only, such as an ordering expression, stays unguarded. +//! +//! Guards a conjunction already states are not repeated below it, which keeps +//! the rewrite idempotent and lets generated `matches()` filters keep their +//! explicit `is_variant AND body` shape. Conjunctions produced by the rewrite +//! flatten into the enclosing conjunction so that a guard and the comparison +//! it scopes stay siblings, which relation lifting relies on to fold them +//! into one subquery. + +use toasty_core::stmt::{ + Expr, ExprAnd, ExprNot, ExprOr, ExprVariant, Query, Statement, Visit, VisitMut, +}; + +use super::Normalize; + +impl Normalize<'_> { + /// Normalizes the operands of a conjunction and flattens conjunctions + /// they produce into it. + /// + /// The `is_variant` operands are in force for every operand, including + /// those the flattening adds ahead of the operands that follow. + pub(super) fn normalize_conjunction(&mut self, and: &mut ExprAnd) { + let depth = self.guards.len(); + self.guards + .extend(and.operands.iter().filter(|e| is_guard(e)).cloned()); + + for mut operand in std::mem::take(&mut and.operands) { + self.visit_expr_mut(&mut operand); + + match operand { + Expr::And(inner) => { + for operand in inner.operands { + if is_guard(&operand) && !self.guards.contains(&operand) { + self.guards.push(operand.clone()); + } + and.operands.push(operand); + } + } + operand => and.operands.push(operand), + } + } + + self.guards.truncate(depth); + } + + /// Conjoins the variant checks a predicate's operands require with the + /// predicate, and expands a negated predicate into its `NOT` form. + /// + /// The operands have already been normalized: a nested predicate carries + /// its own guards, and connectives and subqueries are not searched. + pub(super) fn normalize_predicate_guards(&mut self, expr: &mut Expr) { + let mut guards = vec![]; + let mut collect = CollectGuards { + guards: &mut guards, + existing_guards: &self.guards, + }; + + match expr { + Expr::BinaryOp(e) if e.op.is_arithmetic() => return, + Expr::AllOp(_) + | Expr::AnyOp(_) + | Expr::Between(_) + | Expr::BinaryOp(_) + | Expr::InList(_) + | Expr::InSubquery(_) + | Expr::Intersects(_) + | Expr::IsNull(_) + | Expr::IsSuperset(_) + | Expr::IsVariant(_) + | Expr::Like(_) + | Expr::StartsWith(_) => collect.visit_expr(expr), + Expr::Not(e) => collect.visit_expr(&e.expr), + _ => return, + } + + let mut predicate = expr.take(); + let negated = match &mut predicate { + Expr::IsNull(e) => std::mem::take(&mut e.negated), + Expr::InSubquery(e) => std::mem::take(&mut e.negated), + _ => false, + }; + if negated { + predicate = Expr::not(predicate); + } + + *expr = if guards.is_empty() { + predicate + } else { + guards.push(predicate); + Expr::and_from_vec(guards) + }; + } +} + +fn is_guard(expr: &Expr) -> bool { + matches!(expr, Expr::IsVariant(_)) +} + +/// Collects the `is_variant` checks for the selections an operand reaches. +/// +/// Outer selections come before the selections nested inside them, without +/// duplicates and without the checks already in force. Connectives and +/// subqueries are predicate scopes of their own and are not entered. +struct CollectGuards<'a> { + guards: &'a mut Vec, + existing_guards: &'a [Expr], +} + +impl Visit for CollectGuards<'_> { + fn visit_expr_variant(&mut self, i: &ExprVariant) { + // The base holds the enclosing selections, so its guards come first. + self.visit_expr(&i.base); + + let guard = Expr::is_variant((*i.base).clone(), i.variant); + if !self.existing_guards.contains(&guard) && !self.guards.contains(&guard) { + self.guards.push(guard); + } + } + + fn visit_expr_and(&mut self, _: &ExprAnd) {} + + fn visit_expr_or(&mut self, _: &ExprOr) {} + + fn visit_expr_not(&mut self, _: &ExprNot) {} + + fn visit_stmt(&mut self, _: &Statement) {} + + fn visit_stmt_query(&mut self, _: &Query) {} +} diff --git a/packages/toasty/src/engine/plan.rs b/packages/toasty/src/engine/plan.rs new file mode 100644 index 00000000000..d9ce4256107 --- /dev/null +++ b/packages/toasty/src/engine/plan.rs @@ -0,0 +1,98 @@ +mod execution; +mod nested_merge; +mod statement; + +use crate::{ + Result, + engine::{ + Engine, HirStatement, + exec::{ExecPlan, Step, VarStore}, + hir, + mir::{self, LogicalPlan}, + }, +}; + +#[derive(Debug)] +struct HirPlanner<'a> { + engine: &'a Engine, + + /// Root statement and all nested statements. + hir: &'a HirStatement, + + /// Graph of operations needed to execute the statement + mir: mir::Store, +} + +#[derive(Debug)] +struct ExecPlanner<'a> { + logical_plan: &'a LogicalPlan, + steps: Vec, + use_transactions: bool, +} + +impl Engine { + pub(super) fn plan_hir_statement(&self, hir: HirStatement) -> Result { + // Build the logical plan + let logical_plan = HirPlanner { + engine: self, + hir: &hir, + mir: mir::Store::new(), + } + .build_logical_plan()?; + + // Build the execution plan from the logical plan + Ok(self.plan_execution(logical_plan)) + } + + fn plan_execution(&self, logical_plan: mir::LogicalPlan) -> ExecPlan { + let (steps, needs_transaction) = ExecPlanner { + logical_plan: &logical_plan, + steps: vec![], + use_transactions: self.capability().sql(), + } + .plan_execution(); + + ExecPlan { + vars: VarStore::new(logical_plan.node_count(), self.schema.clone()), + returning: logical_plan.completion(), + plan: logical_plan, + steps, + needs_transaction, + } + } +} + +impl HirPlanner<'_> { + fn build_logical_plan(mut self) -> Result { + self.reserve_effect_dep_data_load_slots(); + + let root_id = self.hir.root_id(); + self.plan_statement(root_id)?; + + let exit = self.hir.root().output.get().unwrap(); + + Ok(mir::LogicalPlan::new(self.mir, exit)) + } + + fn reserve_effect_dep_data_load_slots(&mut self) { + // Reserve a data-load slot for every statement targeted by an effect + // dep. An effect dep's target may be an ancestor planned after the + // dependent (a statement-level cycle that is acyclic at the operation + // level), so the anchor node must exist before planning starts; the + // target's planning later fills the slot with an `Alias` to its + // actual data-loading node. + let hir = self.hir; + for stmt_info in hir.statements() { + for (&target, &kind) in &stmt_info.deps { + if kind != hir::DepKind::Effect { + continue; + } + + let slot = &hir[target].load_data_statement; + if slot.get().is_none() { + slot.set(Some(self.mir.reserve())); + } + } + } + } +} diff --git a/packages/toasty/src/engine/plan/execution.rs b/packages/toasty/src/engine/plan/execution.rs new file mode 100644 index 00000000000..3020f945a42 --- /dev/null +++ b/packages/toasty/src/engine/plan/execution.rs @@ -0,0 +1,135 @@ +use indexmap::IndexSet; + +use crate::engine::{ + exec::{self, Step}, + mir, + plan::ExecPlanner, +}; + +impl ExecPlanner<'_> { + /// Converts the logical plan's execution order into the exec program's + /// step sequence, returning the steps and whether the plan needs to be + /// wrapped in a transaction. + pub(super) fn plan_execution(mut self) -> (Vec, bool) { + self.plan_steps(); + + let needs_transaction = self.needs_transaction(); + + (self.steps, needs_transaction) + } + + fn plan_steps(&mut self) { + // Group each maximal run of consecutive same-guard nodes into one + // `If` block, emitted at the run's existing position. An unguarded + // chain interleaved between two same-guard runs yields several `If` + // blocks with the same condition; the guard rules guarantee an + // interleaved unguarded node never consumes a guarded output, and the + // skip-path variable classification below is per block. + let logical_plan = self.logical_plan; + let mut if_block = vec![]; + + for &node_id in logical_plan.execution_order() { + self.plan_node(node_id, &mut if_block); + } + + self.flush_if_block(&mut if_block); + } + + fn plan_node(&mut self, node_id: mir::NodeId, if_block: &mut Vec) { + let guard = self.logical_plan[node_id].guard; + + if let Some(&first_node_id) = if_block.first() + && guard != self.logical_plan[first_node_id].guard + { + self.flush_if_block(if_block); + } + + if guard.is_some() { + if_block.push(node_id); + } else { + self.steps.push(Step::Run(node_id)); + } + } + + fn flush_if_block(&mut self, block: &mut Vec) { + if block.is_empty() { + return; + } + + let step = self.emit_if_block(std::mem::take(block)); + self.steps.push(step); + } + + fn needs_transaction(&self) -> bool { + self.use_transactions + && self + .steps + .iter() + .map(|step| step.db_op_count(self.logical_plan)) + .sum::() + > 1 + } + + /// Wraps a run of same-guard nodes in an `If`, deriving the skip + /// bookkeeping from a static classification of the variables the `then` + /// arm touches: + /// + /// - **External inputs** (produced outside, loaded inside): released on + /// skip, one entry per load the `then` arm would have performed, + /// keeping use counts exact on both paths. + /// - **Escaping outputs** (produced inside, consumed outside): assigned a + /// non-loadable slot on skip with the node's external use count, so + /// outside consumers can release the skipped output. + /// - **Internal variables** (produced and consumed inside): untouched — + /// on the skip path their slots are never created. + fn emit_if_block(&self, block: Vec) -> Step { + debug_assert!( + block + .iter() + .all(|id| !self.logical_plan[id].op.is_effectful()), + "effectful node inside an `If` arm" + ); + + let skipped_inputs = self.collect_skipped_inputs(&block); + let skipped_outputs = self.collect_skipped_outputs(&block); + + Step::If(exec::If { + then: block, + skipped_inputs, + skipped_outputs, + }) + } + + /// Collects loads from variables produced outside the block. The skip arm + /// releases each load that the `then` arm would have performed. + fn collect_skipped_inputs(&self, block: &[mir::NodeId]) -> Vec { + let in_block: IndexSet = block.iter().copied().collect(); + + block + .iter() + .flat_map(|&node_id| self.logical_plan[node_id].op.input_loads()) + .filter(|load| !in_block.contains(load)) + .collect() + } + + /// Collects variables produced inside the block and consumed outside it. + fn collect_skipped_outputs(&self, block: &[mir::NodeId]) -> Vec<(mir::NodeId, usize)> { + block + .iter() + .filter_map(|&node_id| { + let external_uses = self.external_use_count(block, node_id); + (external_uses > 0).then_some((node_id, external_uses)) + }) + .collect() + } + + fn external_use_count(&self, block: &[mir::NodeId], node_id: mir::NodeId) -> usize { + let in_block_loads = block + .iter() + .flat_map(|&consumer_id| self.logical_plan[consumer_id].op.input_loads()) + .filter(|&load| load == node_id) + .count(); + + self.logical_plan[node_id].num_uses - in_block_loads + } +} diff --git a/packages/toasty/src/engine/plan/nested_merge.rs b/packages/toasty/src/engine/plan/nested_merge.rs new file mode 100644 index 00000000000..45993c8febf --- /dev/null +++ b/packages/toasty/src/engine/plan/nested_merge.rs @@ -0,0 +1,508 @@ +use indexmap::IndexSet; +use toasty_core::stmt::{self, visit_mut}; + +use crate::engine::{ + Engine, HirStatement, SelectItems, eval, + exec::{MergeIndex, MergeQualification, NestedChild, NestedLevel}, + hir, mir, + plan::HirPlanner, +}; + +#[derive(Debug)] +struct NestedMergePlanner<'a> { + engine: &'a Engine, + hir: &'a HirStatement, + mir: &'a mut mir::Store, + inputs: IndexSet, + /// Statements that must execute before the merge but whose output is not needed + deps: IndexSet, + /// Flat list of hash indexes to build, populated as HashLookup qualifications are planned. + hash_indexes: Vec, + /// Flat list of sorted indexes to build, populated as SortLookup qualifications are planned. + sort_indexes: Vec, + /// Statement stack, used to infer expression types + stack: Vec, +} + +impl HirPlanner<'_> { + /// Builds a nested merge operation for queries with sub-statement arguments + /// in the returning clause. + /// + /// When a query has `Arg::Sub { returning: true, .. }` arguments + /// (sub-statements used in the returning clause), those represent nested + /// data that needs to be merged with their parent rows. This method + /// constructs a `NestedMerge` execution plan that: + /// + /// 1. Identifies all batch-loaded inputs needed (parent and child queries) + /// 2. Builds a tree structure mirroring the nesting hierarchy + /// 3. For each level, captures: + /// - The source data (reference to batch-loaded results) + /// - How to filter child rows for each parent (qualification predicates) + /// - How to project the combined parent+children into the final shape + /// + /// The resulting `NestedMerge` will execute by: + /// - Loading all batch data upfront - fetches all input data for all levels before processing + /// - Processing each root row: + /// - For each nested child relationship, filters batch-loaded child data and recursively + /// merges matching rows with their own children + /// - Collects results into a list, or a single value if `single` is `true` + /// - Projects the final row with the current row and all nested children + /// - Returning all merged rows with their nested data + /// + /// # Example + /// + /// For a query like: + /// ```sql + /// SELECT user.*, (SELECT * FROM todos WHERE user_id = user.id) as todos + /// FROM users + /// ``` + /// + /// This builds a two-level merge where: + /// - Root level: user rows from batch load + /// - Nested level: todo rows filtered by user_id match, projected into a list + /// + /// Returns `None` if the statement has no sub-statements with `returning: true`. + pub(super) fn plan_nested_merge(&mut self, stmt_id: hir::StmtId) -> Option { + let stmt_state = &self.hir[stmt_id]; + + // Return if there is no nested merge to do + let need_nested_merge = stmt_state.args.iter().any(|arg| { + matches!( + arg, + hir::Arg::Sub { + returning: true, + .. + } + ) + }); + if !need_nested_merge { + return None; + } + + if stmt_state.stmt.as_ref().unwrap().is_insert() { + // todo!("stmt_state={stmt_state:#?}"); + return None; + } + + let nested_merge_planner = NestedMergePlanner { + engine: self.engine, + hir: self.hir, + mir: &mut self.mir, + inputs: IndexSet::new(), + deps: IndexSet::new(), + hash_indexes: vec![], + sort_indexes: vec![], + stack: vec![], + }; + + let node_id = nested_merge_planner.plan_nested_merge(stmt_id); + Some(node_id) + } +} + +impl NestedMergePlanner<'_> { + fn plan_nested_merge(mut self, root: hir::StmtId) -> mir::NodeId { + self.stack.push(root); + let root = self.plan_nested_level(root, 0); + self.stack.pop(); + + self.mir.insert_with_deps( + mir::NestedMerge::new(self.inputs, root, self.hash_indexes, self.sort_indexes), + self.deps, + ) + } + + fn plan_nested_child(&mut self, stmt_id: hir::StmtId, depth: usize) -> NestedChild { + self.stack.push(stmt_id); + + let level = self.plan_nested_level(stmt_id, depth); + let stmt_state = &self.hir[stmt_id]; + let selection = stmt_state.load_data_select_items.get().unwrap(); + + let ret = match stmt_state.stmt.as_deref().unwrap() { + stmt::Statement::Query(query) => { + let filter_expr = self.build_filter_for_nested_child(stmt_id, selection, depth); + + let filter_arg_tys = self.build_filter_arg_tys(); + let qualification = match try_eq_lookup(&filter_expr, &filter_arg_tys, depth) { + Some((child_projections, lookup_key)) if query.single => { + // has_one / belongs_to: unique key → HashIndex (O(1) lookup). + let index = self.hash_indexes.len(); + self.hash_indexes.push(MergeIndex { + source: level.source, + child_projections, + }); + MergeQualification::HashLookup { index, lookup_key } + } + Some((child_projections, lookup_key)) => { + // has_many: duplicate keys → SortedIndex (O(log M + k) lookup). + let index = self.sort_indexes.len(); + self.sort_indexes.push(MergeIndex { + source: level.source, + child_projections, + }); + MergeQualification::SortLookup { index, lookup_key } + } + // Filter does not reduce to a pure equality conjunction, so we + // cannot drive an index lookup. Fall back to a linear scan. + // See `try_eq_lookup` for discussion of how this could be + // improved to use an index with a residual post-filter. + None => { + MergeQualification::Scan(eval::Func::from_stmt(filter_expr, filter_arg_tys)) + } + }; + + NestedChild { + level, + qualification, + single: query.single, + } + } + stmt::Statement::Insert(insert) => NestedChild { + level, + qualification: MergeQualification::All, + single: insert.source.single, + }, + stmt => todo!("stmt={stmt:#?}"), + }; + + self.stack.pop(); + + ret + } + + fn plan_nested_level(&mut self, stmt_id: hir::StmtId, depth: usize) -> NestedLevel { + let stmt_state = &self.hir[stmt_id]; + let stmt = stmt_state.stmt.as_deref().unwrap(); + let returning = stmt.returning_unwrap(); + + let source; + let mut nested = vec![]; + + // Map the returning clause to projection expression + let projection = match returning { + stmt::Returning::Project(expr) => { + let (s, _) = self + .inputs + .insert_full(stmt_state.load_data_statement.get().unwrap()); + + source = s; + self.build_projection_from_expr(stmt_id, expr, depth, &mut nested) + } + _ => { + let node_id = stmt_state.output.get().unwrap(); + + let (s, _) = self.inputs.insert_full(node_id); + source = s; + + // Flatten list (bit of a hack) + let ty = match self.mir[node_id].ty().clone() { + stmt::Type::List(ty) => *ty, + ty => ty, + }; + + eval::Func::from_stmt(stmt::Expr::arg(0), vec![ty]) + } + }; + + NestedLevel { + source, + projection, + nested, + } + } + + fn build_filter_arg_tys(&self) -> Vec { + self.stack + .iter() + .map(|stmt_id| self.build_exec_statement_ty_for(*stmt_id)) + .collect() + } + + fn build_projection_arg_tys(&self, nested_children: &[NestedChild]) -> Vec { + let curr = self.stack.last().unwrap(); + let mut projection_arg_tys = vec![self.build_exec_statement_ty_for(*curr)]; + + for nested in nested_children { + projection_arg_tys.push(if nested.single { + nested.level.projection.ret.clone() + } else { + stmt::Type::list(nested.level.projection.ret.clone()) + }); + } + + projection_arg_tys + } + + fn build_exec_statement_ty_for(&self, stmt_id: hir::StmtId) -> stmt::Type { + let stmt_state = &self.hir[stmt_id]; + let stmt = stmt_state.stmt.as_deref().unwrap(); + + let cx = stmt::ExprContext::new_with_target(&*self.engine.schema, stmt); + + let mut fields = vec![]; + + for select_item in stmt_state.load_data_select_items.get().unwrap() { + fields.push(select_item.infer_ty(&cx)); + } + + stmt::Type::Record(fields) + } + + /// Rewrites a projection expression, replacing statement-level `Arg` and + /// `Reference` nodes with nested merge arg references. + /// + /// Uses `walk_expr_scoped_mut` to automatically track scope depth through + /// Let/Map scopes so that only statement-level args (where + /// `nesting == scope_depth`) are rewritten — inner Let/Map bindings are + /// left alone. + fn build_projection_from_expr( + &mut self, + stmt_id: hir::StmtId, + expr: &stmt::Expr, + depth: usize, + nested: &mut Vec, + ) -> eval::Func { + // Copy the shared hir reference out of self so the closure can access + // hir data without conflicting with the &mut self capture. + let hir = self.hir; + let selection = hir[stmt_id].load_data_select_items.get().unwrap(); + let mut projection = expr.clone(); + + visit_mut::walk_expr_scoped_mut(&mut projection, 0, |expr, scope_depth| match expr { + stmt::Expr::Arg(expr_arg) if expr_arg.nesting == scope_depth => { + let position = expr_arg.position; + let stmt_state = &hir[stmt_id]; + + match &stmt_state.args[position] { + hir::Arg::Sub { + stmt_id: child_stmt_id, + .. + } => { + let child_stmt_id = *child_stmt_id; + let child_stmt_state = &hir[child_stmt_id]; + let child_stmt = child_stmt_state.stmt.as_deref().unwrap(); + if let stmt::Statement::Query(query) = child_stmt + && let stmt::ExprSet::Values(values) = &query.body + && values.is_empty() + { + *expr = if query.single { + stmt::Expr::null() + } else { + stmt::Expr::list_from_vec(vec![]) + }; + return false; + } + let child_returning = child_stmt.returning_unwrap(); + + match child_returning { + stmt::Returning::Expr(returning_expr) if returning_expr.is_const() => { + match child_stmt { + stmt::Statement::Query(query) if query.single => { + let stmt::Expr::Value(v) = returning_expr else { + todo!() + }; + assert!(!v.is_list()); + } + stmt::Statement::Insert(insert) if insert.source.single => { + let stmt::Expr::Value(v) = returning_expr else { + todo!() + }; + assert!(!v.is_list()); + } + _ => {} + } + + self.deps + .insert(child_stmt_state.load_data_statement.get().unwrap()); + *expr = returning_expr.clone(); + } + _ => { + let nested_child = self.plan_nested_child(child_stmt_id, depth + 1); + nested.push(nested_child); + + *expr = stmt::Expr::arg(nested.len()); + } + } + } + hir::Arg::Ref { .. } => todo!(), + } + false + } + stmt::Expr::Reference(expr_reference) => { + let expr_column = expr_reference.as_expr_column_unwrap(); + debug_assert_eq!(0, expr_column.nesting); + let index = selection.get_index_of_expr_reference(*expr_column); + *expr = stmt::Expr::arg_project(0, [index]); + false + } + _ => true, + }); + + let projection_arg_tys = self.build_projection_arg_tys(nested); + eval::Func::from_stmt(projection, projection_arg_tys) + } + + fn build_filter_for_nested_child( + &self, + stmt_id: hir::StmtId, + selection: &SelectItems, + depth: usize, + ) -> stmt::Expr { + let stmt_state = &self.hir[stmt_id]; + let stmt::Statement::Query(query) = stmt_state.stmt.as_deref().unwrap() else { + unreachable!() + }; + let select = query.body.as_select_unwrap(); + + // Extract the qualification. For now, we will just re-run the + // entire where clause, but that can be improved later. + let mut filter = select.filter.clone(); + + visit_mut::for_each_expr_mut(&mut filter, |expr| match expr { + stmt::Expr::Arg(expr_arg) => { + let hir::Arg::Ref { + nesting, + stmt_id: target_id, + target_expr_ref, + .. + } = &stmt_state.args[expr_arg.position] + else { + todo!() + }; + + debug_assert!(*nesting > 0); + + // This is a bit of a roundabout way to get the data. We may + // want to find a better way to track the info for more direct + // access. + let target_stmt = &self.hir[target_id]; + + let target_exec_statement_index = target_stmt + .load_data_select_items + .get() + .unwrap() + .get_index_of_expr_reference(*target_expr_ref); + + *expr = stmt::Expr::arg_project(depth - *nesting, [target_exec_statement_index]); + } + stmt::Expr::Reference(expr_reference) => { + let index = selection.get_index_of_expr_reference(*expr_reference); + *expr = stmt::Expr::arg_project(depth, [index]); + } + _ => {} + }); + + filter.into_expr() + } +} + +/// Try to extract index lookup key fields from a transformed filter expression. +/// +/// Recognizes patterns of the form: +/// - Single equality: `arg_project(depth, [cf]) == arg_project(pos < depth, [pf])` +/// - Composite AND: `eq1 AND eq2 AND ...` where each `eqi` has the above form +/// +/// On success returns `(child_projections, lookup_key)` where: +/// - `child_projections[i]` is the projection into the child record for key field `i` +/// - `lookup_key` is an `eval::Func` that evaluates against the ancestor `RowStack` +/// and returns the lookup key (scalar for single-field, `Value::Record` for composite) +/// +/// # Limitations +/// +/// This function only succeeds when the *entire* filter is a pure equality conjunction. +/// Any more complex filter (e.g. `a = b AND c > d`, or an `OR`) causes it to return +/// `None` and the caller falls back to a full `Scan`, even if part of the filter could +/// drive an index lookup with the remainder applied as a post-filter. +/// +/// A more complete approach — similar to `IndexMatch` in the index planner — would +/// extract whichever equality terms can key an index, build the lookup from those, +/// and re-evaluate the full original predicate against each candidate row returned by +/// the index. That would turn O(N×M) into O(log M + k) even for compound filters. +/// For now we keep this conservative: only use an index when the whole filter matches. +fn try_eq_lookup( + expr: &stmt::Expr, + arg_tys: &[stmt::Type], + depth: usize, +) -> Option<(Vec, eval::Func)> { + // Collect equality terms: single BinaryOp(Eq) or AND of BinaryOp(Eq)s. + let eq_terms: Vec<(&stmt::Expr, &stmt::Expr)> = match expr { + stmt::Expr::BinaryOp(op) if op.op == stmt::BinaryOp::Eq => { + vec![(&op.lhs, &op.rhs)] + } + stmt::Expr::And(and_expr) => { + let mut terms = vec![]; + for operand in and_expr.operands.iter() { + match operand { + stmt::Expr::BinaryOp(op) if op.op == stmt::BinaryOp::Eq => { + terms.push((&*op.lhs, &*op.rhs)); + } + _ => return None, + } + } + terms + } + _ => return None, + }; + + let mut child_projections = vec![]; + let mut lookup_key_exprs = vec![]; + + for (lhs, rhs) in eq_terms { + let (child_proj, parent_expr) = extract_child_parent_eq(lhs, rhs, depth)?; + child_projections.push(child_proj); + lookup_key_exprs.push(parent_expr); + } + + if child_projections.is_empty() { + return None; + } + + // Build the parent key expression. For a single field, use the scalar + // directly. For multiple fields, wrap in a record (evaluates to Value::Record). + let lookup_key_expr = if lookup_key_exprs.len() == 1 { + lookup_key_exprs.remove(0) + } else { + stmt::Expr::record_from_vec(lookup_key_exprs) + }; + + // Parent key args are the ancestor stack types only (not including the + // current child row at position `depth`). + let lookup_key_arg_tys = arg_tys[..depth].to_vec(); + let lookup_key = eval::Func::from_stmt(lookup_key_expr, lookup_key_arg_tys); + + Some((child_projections, lookup_key)) +} + +/// For an equality `lhs == rhs`, determine which side is the child (at `depth`) +/// and which is the parent (at some position < `depth`). +/// +/// Both sides must be simple `arg_project(pos, projection)` expressions. +/// Returns `(child_projection, parent_expr)` or `None` if the pattern doesn't match. +fn extract_child_parent_eq( + lhs: &stmt::Expr, + rhs: &stmt::Expr, + depth: usize, +) -> Option<(stmt::Projection, stmt::Expr)> { + match (as_simple_arg_project(lhs), as_simple_arg_project(rhs)) { + (Some((l_pos, l_proj)), Some((r_pos, _))) if l_pos == depth && r_pos < depth => { + Some((l_proj.clone(), rhs.clone())) + } + (Some((l_pos, _)), Some((r_pos, r_proj))) if r_pos == depth && l_pos < depth => { + Some((r_proj.clone(), lhs.clone())) + } + _ => None, + } +} + +/// Match `Project(Arg { position, nesting: 0 }, projection)` and return +/// `(position, &projection)`. Returns `None` for any other expression shape. +fn as_simple_arg_project(expr: &stmt::Expr) -> Option<(usize, &stmt::Projection)> { + match expr { + stmt::Expr::Project(proj) => match proj.base.as_ref() { + stmt::Expr::Arg(arg) if arg.nesting == 0 => Some((arg.position, &proj.projection)), + _ => None, + }, + _ => None, + } +} diff --git a/packages/toasty/src/engine/plan/statement.rs b/packages/toasty/src/engine/plan/statement.rs new file mode 100644 index 00000000000..113d2bcf46c --- /dev/null +++ b/packages/toasty/src/engine/plan/statement.rs @@ -0,0 +1,2457 @@ +//! Plans a single HIR statement into MIR operations. +//! +//! # Overview +//! +//! Each HIR statement produces one or more MIR nodes that form an execution +//! graph. The planner splits work into two concerns: +//! +//! - **Data loading**: issuing the database operation (SQL query, GetByKey, etc.) +//! - **Output processing**: projecting, filtering, or merging the loaded data +//! into the shape the caller expects +//! +//! The entry point is `PlanStatement::plan()`, which runs these phases in order: +//! +//! 1. Extract columns needed by the returning clause +//! 2. Discover args (references to other statements) in the filter and +//! assignments, registering their MIR node outputs as inputs +//! 3. Rewrite the statement's assignment/value expressions so their `Arg` +//! positions reference `load_data.inputs` indices +//! 4. Plan the data-loading MIR node (SQL or NoSQL path) +//! 5. Create projection nodes for back-references (so child statements can +//! read columns from this statement's results) +//! 6. Plan child/dependent statements +//! 7. Build the output node (projection, eval, or nested merge) +//! +//! # How `Expr::Arg` flows through the planner +//! +//! `Expr::Arg(n)` appears throughout the statement to reference data from +//! other statements. The meaning of `n` changes as the expression moves +//! through the pipeline: +//! +//! ## HIR level (statement enters the planner) +//! +//! `Arg(n)` indexes into `stmt_info.args`, a list of `hir::Arg` entries +//! created during lowering. Each entry is either: +//! +//! - `Arg::Sub { stmt_id, input, .. }` — data from a child statement +//! (e.g., a subquery result used in an `IN` list) +//! - `Arg::Ref { stmt_id, data_load_input, .. }` — a column from a +//! parent/sibling statement (back-reference for nested queries) +//! +//! At this point, `input` and `data_load_input` are unset `Cell`s. +//! +//! ## After `extract_data_load_args` +//! +//! This pass walks the filter and assignments, and for each `Arg(n)`: +//! +//! - Looks up `stmt_info.args[n]` to find the source MIR node +//! - Adds that node to `load_data.inputs` (an `IndexSet`) +//! - Records the index within `load_data.inputs` back into the `hir::Arg` +//! cell (`input.set(Some(index))` or `data_load_input.set(Some(index))`) +//! +//! The `Arg(n)` positions in the statement expressions are unchanged at +//! this point — they still reference HIR arg positions. +//! +//! ## After `rewrite_stmt_*_arg_dependencies` +//! +//! These methods rewrite `Arg` nodes in **assignments and insert values +//! only** (not the filter). They read the cells populated above: +//! +//! - `Arg::Sub { input }` → `Expr::arg(input.get())` — now indexes into +//! `load_data.inputs` +//! - `Arg::Ref { data_load_input, batch_load_index }` → +//! `Expr::arg_project(data_load_input, [batch_load_index, column])` +//! +//! After this, assignment expressions have MIR-level arg positions. Filter +//! expressions still have HIR-level positions. +//! +//! ## During MIR node construction (`rewrite_expr_for_mir`) +//! +//! When building MIR nodes that carry their own expressions (a `Filter` +//! predicate or key expression), the expression may reference a subset of +//! the statement's `load_data.inputs`. `rewrite_expr_for_mir` does two things: +//! +//! 1. Resolves each `Arg(hir_pos)` through `stmt_info.args[hir_pos]` to +//! find the `load_data.inputs` index and MIR node ID +//! 2. Assigns a new compact position, starting at the caller-provided offset, +//! and rewrites the `Arg` node in place +//! +//! It returns the arg types and input node IDs for constructing the MIR +//! node. This is the same resolution as `rewrite_arg_dependencies` but +//! builds a compact, node-specific input set rather than reusing the full +//! `load_data.inputs` list. +//! +//! ## At execution time +//! +//! The executor loads each node's inputs from their `NodeId` variable slots +//! and passes them to `eval::Func`, which resolves `Arg(n)` against the +//! provided input slice. + +use std::mem; + +#[cfg(test)] +mod tests; + +use indexmap::{IndexMap, IndexSet}; +use toasty_core::schema::db; +use toasty_core::stmt::{self, visit_mut}; + +use toasty_core::driver::{Dialect, operation::Pagination}; + +use crate::{ + Result, + engine::{ + SelectItem, SelectItems, eval, exec, + hir::{self}, + index::{self, IndexPlan}, + mir, + plan::HirPlanner, + }, +}; + +#[derive(Debug)] +struct LoadData { + /// MIR node inputs needed to load data associated with the statement + inputs: IndexSet, + + /// Items to select from the database (columns and aggregates like COUNT(*)) + select_items: SelectItems, + + /// When the statement data is batch loaded (single database query to load + /// data for multiple statements), arguments are passed in in batches as + /// well. For SQL, this is done using derived tables. This maps "input" to + /// the derived table index. + batch_load_args: IndexSet, +} + +type Returning = Option; + +#[derive(Debug)] +struct ReturningInfo { + clause: Option, + + /// Nodes the returning expression reads, in reference order. For a + /// `Project` clause these are the other inputs of the per-row `Eval` that + /// evaluates it — the loaded row is not among them; the body + /// references it as `arg(0)` and these as `arg(1 + i)`. + inputs: IndexSet, + + /// `Project` only: the projection references the loaded row (a column or + /// `count(*)`), so it must be evaluated per row of the data load. + reads_row: bool, +} + +struct PaginationInfo { + page_size: i64, + has_previous_page: bool, + cursor_column_indices: Vec, +} + +struct PlanStatement<'a, 'b> { + planner: &'a mut HirPlanner<'b>, + stmt_id: hir::StmtId, + stmt_info: &'b hir::StatementInfo, + + /// Planning information related ot how to load data to satisfy the statement. + load_data: LoadData, + + /// True if the statement's dependencies have been tracked + remaining_deps: Vec, +} + +impl HirPlanner<'_> { + pub(super) fn plan_statement(&mut self, stmt_id: hir::StmtId) -> Result<()> { + let stmt_info = &self.hir[stmt_id]; + + // Check if the statement has already been planned, or is currently + // being planned (a statement-level dependency cycle that is acyclic + // at the operation level — the edge anchors on a reserved data-load + // slot instead of recursing). + if stmt_info.planning.get() { + return Ok(()); + } + stmt_info.planning.set(true); + + // First, plan independent dependency statements. These are statments + // that must run before the current one but do not reference the current + // statement. Effect deps target ancestors, planned by the enclosing + // traversal, so they are never planned from here. + for (&dep_stmt_id, &kind) in &stmt_info.deps { + if kind == hir::DepKind::Statement && self.hir[dep_stmt_id].independent { + self.plan_statement(dep_stmt_id)?; + } + } + + let stmt = stmt_info.stmt.as_deref().unwrap().clone(); + + // Delegate to PlanStatement + let mut planner = PlanStatement { + planner: self, + stmt_id, + stmt_info, + load_data: LoadData { + inputs: IndexSet::new(), + select_items: SelectItems::new(), + batch_load_args: IndexSet::new(), + }, + remaining_deps: stmt_info + .deps + .iter() + .filter(|&(_, &kind)| kind == hir::DepKind::Statement) + .map(|(&stmt_id, _)| stmt_id) + .collect(), + }; + planner.plan(stmt)?; + + Ok(()) + } +} + +impl<'a, 'b> PlanStatement<'a, 'b> { + // ===== Entry point ===== + + fn plan(&mut self, mut stmt: stmt::Statement) -> Result<()> { + let mut returning = stmt.take_returning(); + + // For single VALUES queries (e.g., batch queries), the VALUES body is + // the output expression. Extract it as a returning value so the planner + // can wire up sub-statement dependencies. An empty VALUES body (e.g. an + // optional `belongs_to` whose foreign key is NULL, so simplification + // proved the filter false) has no output expression; leave `returning` + // empty so the query plans as an empty constant and produces zero + // rows, the same shape as any other query that matches nothing. + if returning.is_none() + && let stmt::Statement::Query(query) = &mut stmt + && let stmt::ExprSet::Values(values) = &mut query.body + && !values.rows.is_empty() + { + returning = Some(stmt::Returning::Expr(if query.single { + assert_eq!(1, values.rows.len(), "single query has more than one row"); + values.rows.drain(..).next().unwrap() + } else { + stmt::Expr::list(std::mem::take(&mut values.rows)) + })); + } + + // No queries are single at this point. + match &mut stmt { + stmt::Statement::Query(stmt) => stmt.single = false, + stmt::Statement::Insert(stmt) => stmt.source.single = false, + _ => {} + } + + // Visit the main statement's returning clause to extract needed columns + self.extract_columns_from_returning(&returning); + + // Process any args (sub statements or refs to parent statements) in the query's filter. + self.extract_data_load_args(&mut stmt); + + // For each back ref, include the needed columns + self.collect_back_ref_columns(); + + // If there are any ref args, then the statement might need to be + // rewritten to batch load all records for a NestedMerge operation. + if !self.load_data.batch_load_args.is_empty() { + debug_assert!(stmt.is_query()); + self.rewrite_stmt_for_batch_load(&mut stmt); + } else if let stmt::Statement::Insert(insert) = &mut stmt { + self.rewrite_stmt_insert_arg_dependencies(insert); + } else if let stmt::Statement::Update(update) = &mut stmt { + self.rewrite_stmt_update_arg_dependencies(update); + } else if let stmt::Statement::Query(query) = &mut stmt { + self.rewrite_stmt_query_arg_dependencies(query); + } + + let load_data_node_id = self.plan_data_loading(stmt, &mut returning)?; + + // Track the exec statement operation node. When the slot was + // reserved up front (this statement is the target of an effect dep), + // fill it with a pass-through to the node planning actually produced. + match self.stmt_info.load_data_statement.get() { + Some(reserved) => { + let ty = self.planner.mir[load_data_node_id].ty().clone(); + self.planner.mir.fill( + reserved, + mir::Alias { + input: load_data_node_id, + ty, + }, + ); + } + None => { + self.stmt_info + .load_data_statement + .set(Some(load_data_node_id)); + } + } + + // Order this statement's database operation after every statement + // whose effect must precede it. The target may be an ancestor still + // being planned; its data-load slot is reserved, so the edge resolves + // regardless of planning order. + for (&dep, &kind) in &self.stmt_info.deps { + if kind != hir::DepKind::Effect { + continue; + } + + let anchor = self.planner.hir[dep] + .load_data_statement + .get() + .expect("effect dep target's data-load slot is reserved"); + self.planner.mir[load_data_node_id].deps.insert(anchor); + } + + // Now, for each back ref, we need to project the expression to what the + // next statement expects. + self.process_back_ref_projections(load_data_node_id); + + // Track the selection for later use. + // TODO: Do we actually need to track this on the statement? + self.stmt_info + .load_data_select_items + .set(mem::take(&mut self.load_data.select_items)) + .unwrap(); + + // Plan each child + self.plan_child_statements()?; + + // Track sub-statements referenced in the returning clause as inputs, so their + // results are available when building the return value. + let (inputs, reads_row) = + self.extract_inputs_from_returning(&mut returning, load_data_node_id); + let returning_info = ReturningInfo { + clause: returning, + inputs, + reads_row, + }; + + // Plans a NestedMerge if one is needed + let output_node_id = self.plan_output_node(load_data_node_id, returning_info); + + self.stmt_info.output.set(Some(output_node_id)); + + // Ordinary deps only target siblings and children, all planned by the + // time this statement's output node exists; ancestors are expressed + // as `effect_deps` and were anchored above. + debug_assert!( + self.remaining_deps.is_empty(), + "unresolved statement deps after planning: {:?}", + self.remaining_deps + ); + + Ok(()) + } + + // ===== Setup helpers ===== + + fn extract_inputs_from_returning( + &mut self, + returning: &mut Returning, + load_data_node_id: mir::NodeId, + ) -> (IndexSet, bool) { + let mut inputs = IndexSet::new(); + let mut reads_row = false; + + let is_returning_projection = matches!(returning, Some(stmt::Returning::Project(..))); + debug_assert!( + is_returning_projection || matches!(returning, None | Some(stmt::Returning::Expr(..))) + ); + + match returning { + Some(stmt::Returning::Project(expr)) | Some(stmt::Returning::Expr(expr)) => { + reads_row = self.rewrite_returning_inputs( + expr, + &mut inputs, + load_data_node_id, + is_returning_projection, + ); + } + _ => {} + } + + (inputs, reads_row) + } + + /// Rewrite the returning clause expression so statement-level + /// `Arg`/`Reference`/`Count`/`Project` nodes reference the MIR inputs that + /// supply their data, collecting those inputs into `inputs`. + /// + /// A `Project` clause becomes the body of a per-row `Eval`, so its + /// references follow that operation's convention: the loaded row is + /// `arg(0)` and the collected inputs are `arg(1 + index)`. An `Expr` + /// clause becomes a whole-value `Eval` body, where the collected inputs + /// are `arg(index)` directly. + /// + /// Returns whether the expression references the loaded row (a column or + /// `count(*)`) — for a `Project` clause, whether it must be evaluated per + /// row. + /// + /// Walk scope-aware so that `Arg`/`Reference` nodes nested inside a + /// `Map`/`Let` body (e.g. the via-include projection that strips the + /// linking column with `Map(child, arg(1))`) are not mistaken for + /// statement-level args/columns. Statement-level constructs only appear at + /// the top scope (`scope_depth == 0`); a local arg inside a mapped body has + /// `nesting < scope_depth`. + fn rewrite_returning_inputs( + &self, + expr: &mut stmt::Expr, + inputs: &mut IndexSet, + load_data_node_id: mir::NodeId, + is_returning_projection: bool, + ) -> bool { + // In a per-row `Eval` body, `arg(0)` is the row, so the other inputs + // start at position 1. + let input_offset = if is_returning_projection { 1 } else { 0 }; + let mut reads_row = false; + + visit_mut::walk_expr_scoped_mut(expr, 0, |expr, scope_depth| { + if scope_depth != 0 { + return true; + } + match expr { + stmt::Expr::Arg(expr_arg) => { + match &self.stmt_info.args[expr_arg.position] { + hir::Arg::Ref { + stmt_id: target_id, + returning_input, + batch_load_index, + target_expr_ref, + .. + } => { + let target_stmt_info = &self.planner.hir[target_id]; + let back_ref = &target_stmt_info.back_refs[&self.stmt_id]; + + // Find the column + let column = back_ref.exprs.get_index_of(target_expr_ref).unwrap(); + + if returning_input.get().is_none() { + // Find the node providing the data for the ref + let node_id = back_ref.node_id.get().unwrap(); + + let (index, _) = inputs.insert_full(node_id); + returning_input.set(Some(index)); + } + + let index = returning_input.get().unwrap(); + let row = batch_load_index.get().unwrap(); + + *expr = stmt::Expr::arg_project(input_offset + index, [row, column]); + } + hir::Arg::Sub { + stmt_id: target_id, .. + } => { + let target_stmt_info = &self.planner.hir[target_id]; + let target_node_id = target_stmt_info.output.get().expect("bug"); + + let (index, _) = inputs.insert_full(target_node_id); + + *expr = stmt::Expr::arg(input_offset + index); + } + } + false + } + stmt::Expr::Project(expr_project) if !is_returning_projection => { + // When returning an expression (not projection), + // ExprReference projections need to be handled explicitly. + if let stmt::Expr::Reference(expr_reference) = &*expr_project.base { + let [row] = expr_project.projection.as_slice() else { + todo!("expr_projec{expr_project:#?}") + }; + + let column = self.load_data_expr_reference_position(expr_reference); + let (position, _) = inputs.insert_full(load_data_node_id); + *expr = stmt::Expr::arg_project(position, [*row, column]); + false + } else { + true + } + } + stmt::Expr::Reference(expr_reference) if is_returning_projection => { + let column = self.load_data_expr_reference_position(expr_reference); + reads_row = true; + *expr = stmt::Expr::arg_project(0, [column]); + false + } + stmt::Expr::Func(stmt::ExprFunc::Count(stmt::FuncCount { arg: None, .. })) + if is_returning_projection => + { + let index = self + .stmt_info + .load_data_select_items + .get() + .unwrap() + .get_index_of_count_star(); + reads_row = true; + *expr = stmt::Expr::arg_project(0, [index]); + false + } + _ => true, + } + }); + + reads_row + } + + fn load_data_expr_reference_position(&self, expr_reference: &stmt::ExprReference) -> usize { + assert!( + expr_reference.is_column(), + "TODO: expr_reference = {expr_reference:#?}" + ); + + let Some(column) = self + .stmt_info + .load_data_select_items + .get() + .unwrap() + .try_get_index_of_expr_reference(*expr_reference) + else { + panic!( + "expr_reference={expr_reference:#?}; data_load.select_items={:#?}", + self.load_data.select_items + ) + }; + column + } + + fn extract_columns_from_returning(&mut self, returning: &Returning) { + stmt::visit::for_each_expr(returning, |expr| match expr { + stmt::Expr::Reference(expr_reference) => { + assert!( + expr_reference.is_column(), + "TODO: expr_reference = {expr_reference:#?}" + ); + self.load_data.select_items.insert((*expr_reference).into()); + } + stmt::Expr::Func(stmt::ExprFunc::Count(stmt::FuncCount { arg: None, .. })) => { + self.load_data.select_items.insert(SelectItem::CountStar); + } + _ => {} + }) + } + + /// Extract arguments needed to perform data loading + fn extract_data_load_args(&mut self, stmt: &mut stmt::Statement) { + if let Some(filter) = stmt.filter() { + stmt::visit::for_each_expr(filter, |expr| { + self.extract_data_load_args_from_expr(expr, None); + }); + } + + if let stmt::Statement::Insert(insert) = stmt { + let stmt::ExprSet::Values(values) = &insert.source.body else { + todo!() + }; + + for (i, row) in values.rows.iter().enumerate() { + stmt::visit::for_each_expr(row, |expr| { + self.extract_data_load_args_from_expr(expr, Some(i)); + }); + } + + if let Some(upsert) = &insert.upsert { + for (_, assignment) in upsert.shared.iter() { + stmt::visit::for_each_expr(assignment, |expr| { + self.extract_data_load_args_from_expr(expr, None); + }); + } + for (_, assignment) in upsert.defaults.iter() { + stmt::visit::for_each_expr(assignment, |expr| { + self.extract_data_load_args_from_expr(expr, None); + }); + } + for (_, assignment) in upsert.update_defaults.iter() { + stmt::visit::for_each_expr(assignment, |expr| { + self.extract_data_load_args_from_expr(expr, None); + }); + } + } + } + + if let stmt::Statement::Update(update) = stmt { + for (_, assignment) in update.assignments.iter() { + stmt::visit::for_each_expr(assignment, |expr| { + self.extract_data_load_args_from_expr(expr, None); + }); + } + } + } + + fn extract_data_load_args_from_expr(&mut self, expr: &stmt::Expr, insert_row: Option) { + if let stmt::Expr::Arg(expr_arg) = expr { + match &self.stmt_info.args[expr_arg.position] { + hir::Arg::Sub { + stmt_id: target_id, + returning, + input, + batch_load_index, + .. + } => { + debug_assert!(!returning, "the argument was found in a filter"); + + let target = &self.planner.hir[target_id]; + let Some(node_id) = target.output.get() else { + panic!( + "bug: expected target statement to be planned; curr={:#?}; target={:#?}", + self.stmt_info, target + ); + }; + let (index, _) = self.load_data.inputs.insert_full(node_id); + batch_load_index.set(insert_row); + input.set(Some(index)); + } + hir::Arg::Ref { + stmt_id: target_id, + data_load_input, + batch_load_index, + .. + } => { + // refs can be duplicated in the same statement + if data_load_input.get().is_some() { + return; + } + + let target_stmt_info = &self.planner.hir[target_id]; + let back_ref = &target_stmt_info.back_refs[&self.stmt_id]; + + // TODO: should we just use the data_load node ID? + let node_id = back_ref.node_id.get().unwrap(); + + let (index, _) = self.load_data.inputs.insert_full(node_id); + data_load_input.set(Some(index)); + + // If the target statement is a query, then we are in a batch-load scenario. + if target_stmt_info.stmt().is_query() { + debug_assert!(insert_row.is_none()); + + let (batch_load_table_ref_index, _) = + self.load_data.batch_load_args.insert_full(index); + batch_load_index.set(Some(batch_load_table_ref_index)); + } else if let Some(row) = insert_row { + debug_assert!(target_stmt_info.stmt().is_insert()); + // batch_load_index may already be set during lowering + // (when the parent INSERT's row index was captured via + // scope_statement). In that case, the lowering value is + // the correct parent row index; don't overwrite it. + if batch_load_index.get().is_none() { + batch_load_index.set(Some(row)); + } + } else { + debug_assert!( + batch_load_index.get().is_some(), + "stmt={:#?}; target={:#?}; batch_load_index={:#?}", + self.stmt_info, + target_stmt_info, + batch_load_index.get() + ); + } + } + } + } + } + + fn collect_back_ref_columns(&mut self) { + for back_ref in self.stmt_info.back_refs.values() { + for expr in &back_ref.exprs { + self.load_data.select_items.insert((*expr).into()); + } + } + } + + fn rewrite_stmt_for_batch_load(&mut self, stmt: &mut stmt::Statement) { + if self.planner.engine.capability().sql() { + self.rewrite_stmt_query_for_batch_load_sql(stmt); + } else { + self.rewrite_stmt_query_for_batch_load_nosql(stmt); + } + } + + fn rewrite_stmt_query_for_batch_load_sql(&mut self, stmt: &mut stmt::Statement) { + // If targeting SQL, leverage the SQL query engine to handle most of the rewrite details. + let mut filter = stmt + .filter_mut() + .map(|filter| filter.take()) + .unwrap_or_default(); + + visit_mut::for_each_expr_mut(&mut filter, |expr| { + match expr { + stmt::Expr::Reference(stmt::ExprReference::Column(expr_column)) => { + debug_assert_eq!(0, expr_column.nesting); + // We need to up the nesting to reflect that the filter is moved + // one level deeper. + expr_column.nesting += 1; + } + stmt::Expr::Arg(expr_arg) => { + let hir::Arg::Ref { + stmt_id: target_id, + target_expr_ref, + batch_load_index: batch_load_table_ref_index, + .. + } = &self.stmt_info.args[expr_arg.position] + else { + todo!() + }; + + let back_ref = &self.planner.hir[target_id].back_refs[&self.stmt_id]; + let column = back_ref.exprs.get_index_of(target_expr_ref).unwrap(); + + // Rewrite reference the new `FROM`. + *expr = stmt::Expr::column(stmt::ExprColumn { + nesting: 0, + table: batch_load_table_ref_index.get().unwrap(), + column, + }); + } + _ => {} + } + }); + + let tables: Vec = self + .load_data + .batch_load_args + .iter() + .map(|position| stmt::TableRef::Arg(stmt::ExprArg::new(*position))) + .collect(); + + assert!(tables.len() <= 1, "TODO: handle more complicated cases"); + + let sub_query = stmt::Select { + returning: stmt::Returning::Project(stmt::Expr::record([1])), + source: stmt::Source::Table(stmt::SourceTable { + tables, + from: vec![stmt::TableWithJoins { + relation: stmt::TableFactor::Table(stmt::SourceTableId(0)), + joins: vec![], + }], + }), + filter, + distinct: false, + }; + + stmt.filter_mut_unwrap().set(stmt::Expr::exists(sub_query)); + } + + fn rewrite_stmt_query_for_batch_load_nosql(&mut self, stmt: &mut stmt::Statement) { + let mut filter = stmt.filter_expr_mut(); + visit_mut::for_each_expr_mut(&mut filter, |expr| match expr { + stmt::Expr::Reference(stmt::ExprReference::Column(expr_column)) => { + debug_assert_eq!(0, expr_column.nesting); + } + stmt::Expr::Arg(expr_arg) => { + let hir::Arg::Ref { + stmt_id: target_id, + target_expr_ref, + .. + } = &self.stmt_info.args[expr_arg.position] + else { + todo!() + }; + + let back_ref = &self.planner.hir[target_id].back_refs[&self.stmt_id]; + let column = back_ref.exprs.get_index_of(target_expr_ref).unwrap(); + + *expr = stmt::Expr::arg(column); + } + _ => {} + }); + + assert!( + self.load_data.batch_load_args.len() == 1, + "TODO: handle more complicated cases" + ); + let input = self.load_data.batch_load_args[0]; + + if let Some(filter) = filter { + let expr = filter.take(); + *filter = stmt::Expr::any(stmt::Expr::map(stmt::Expr::arg(input), expr)); + } + } + + fn rewrite_stmt_insert_arg_dependencies(&mut self, stmt: &mut stmt::Insert) { + let stmt::ExprSet::Values(values) = &mut stmt.source.body else { + todo!() + }; + + for row in &mut values.rows { + self.rewrite_arg_dependencies(row); + } + + if let Some(upsert) = &mut stmt.upsert { + for (_, assignment) in upsert.shared.iter_mut() { + self.rewrite_assignment_arg_dependencies(assignment); + } + for (_, assignment) in upsert.defaults.iter_mut() { + self.rewrite_assignment_arg_dependencies(assignment); + } + for (_, assignment) in upsert.update_defaults.iter_mut() { + self.rewrite_assignment_arg_dependencies(assignment); + } + } + } + + fn rewrite_assignment_arg_dependencies(&mut self, assignment: &mut stmt::Assignment) { + match assignment { + stmt::Assignment::Set(expr) + | stmt::Assignment::Insert(expr) + | stmt::Assignment::Remove(expr) + | stmt::Assignment::Append(expr) + | stmt::Assignment::RemoveAt(expr) + | stmt::Assignment::Add(expr) + | stmt::Assignment::Subtract(expr) => self.rewrite_arg_dependencies(expr), + stmt::Assignment::Pop => {} + stmt::Assignment::Batch(assignments) => { + for assignment in assignments { + self.rewrite_assignment_arg_dependencies(assignment); + } + } + } + } + + fn rewrite_stmt_update_arg_dependencies(&mut self, stmt: &mut stmt::Update) { + for (_, assignment) in stmt.assignments.iter_mut() { + let expr = match assignment { + stmt::Assignment::Set(expr) + | stmt::Assignment::Insert(expr) + | stmt::Assignment::Remove(expr) + | stmt::Assignment::Append(expr) + | stmt::Assignment::RemoveAt(expr) + | stmt::Assignment::Add(expr) + | stmt::Assignment::Subtract(expr) => expr, + stmt::Assignment::Pop => continue, + stmt::Assignment::Batch(_) => { + todo!("batch assignments in arg dependency rewriting") + } + }; + self.rewrite_arg_dependencies(expr); + } + } + + /// Rewrite filter args for a query whose parent references bind to a + /// fixed row of the parent statement's output (`batch_load_index` set + /// during lowering — e.g. a `belongs_to` load subquery for one row of an + /// INSERT's returning). Queries that batch-load against a parent *query* + /// take the `rewrite_stmt_for_batch_load` path instead. + fn rewrite_stmt_query_arg_dependencies(&mut self, stmt: &mut stmt::Query) { + if let stmt::ExprSet::Select(select) = &mut stmt.body + && let Some(expr) = &mut select.filter.expr + { + self.rewrite_arg_dependencies(expr); + } + } + + fn rewrite_arg_dependencies(&mut self, expr: &mut stmt::Expr) { + visit_mut::for_each_expr_mut(expr, |expr| { + if let stmt::Expr::Arg(expr_arg) = expr { + match &self.stmt_info.args[expr_arg.position] { + hir::Arg::Ref { + stmt_id: target_id, + target_expr_ref, + data_load_input, + batch_load_index, + .. + } => { + debug_assert!(!self.load_data.inputs.is_empty(), "{:#?}", self.load_data); + + // TODO: this work seems to be duplicated in the returning as well. + let back_ref = &self.planner.hir[target_id].back_refs[&self.stmt_id]; + let column = back_ref.exprs.get_index_of(target_expr_ref).unwrap(); + + *expr = stmt::Expr::arg_project( + data_load_input.get().unwrap(), + [batch_load_index.get().unwrap(), column], + ); + } + hir::Arg::Sub { input, .. } => { + debug_assert!( + !self.load_data.inputs.is_empty(), + "{:#?} | is this needed?", + self.load_data + ); + *expr = stmt::Expr::arg(input.get().unwrap()); + } + } + } + }); + } + + // ===== Plan data loading phase ===== + + fn plan_data_loading( + &mut self, + stmt: stmt::Statement, + returning: &mut Returning, + ) -> Result { + // COUNT(*) is SQL-only + if self.load_data.select_items.contains(&SelectItem::CountStar) + && !self.planner.engine.capability().sql() + { + return Err(toasty_core::Error::unsupported_feature(format!( + "{} does not support count() queries", + self.planner.engine.capability().driver_name + ))); + } + + if let Some(node_id) = self.plan_const_or_empty_statement(&stmt, returning) { + debug_assert!( + stmt.is_query() || stmt.assignments().map(|a| a.is_empty()).unwrap_or(false), + "planned a mutable statement as const; stmt={:#?}", + stmt + ); + Ok(node_id) + } else if stmt.is_insert() { + self.plan_insert(stmt) + } else if self.planner.engine.capability().sql() { + self.plan_data_loading_sql(stmt) + } else { + self.plan_data_loading_nosql(stmt) + } + } + + fn plan_const_or_empty_statement( + &mut self, + stmt: &stmt::Statement, + returning: &mut Returning, + ) -> Option { + if stmt.is_const() { + let stmt::Value::List(rows) = stmt.eval_const().unwrap() else { + todo!() + }; + + return Some( + self.insert_const( + rows, + self.load_data + .select_items + .infer_record_list_ty(&self.planner.engine.expr_cx_for(stmt)), + ), + ); + } + + if stmt.assignments().map(|a| a.is_empty()).unwrap_or(false) { + if returning.is_some() { + return Some(self.insert_const( + vec![stmt::Value::empty_sparse_record()], + stmt::Type::list(stmt::Type::empty_sparse_record()), + )); + } else { + return Some(self.insert_const( + Vec::::new(), + stmt::Type::list(stmt::Type::empty_sparse_record()), + )); + } + } + + None + } + + // ===== Insert execution ===== + + fn plan_insert(&mut self, mut stmt: stmt::Statement) -> Result { + debug_assert!(stmt.is_insert(), "stmt={stmt:#?}"); + + // An upsert's output must reflect the database outcome, so constants + // never replace its RETURNING clause. Back-refs (e.g. an eager + // `belongs_to` load subquery correlating on a returned column) read + // the RETURNING rows: a `DO NOTHING` upsert returns no row on + // conflict, but a back-ref reader only executes when a row exists + // (it is guarded on the insert's output being non-empty). + let const_returning = if stmt.is_upsert() { + None + } else { + self.extract_insert_returning_as_const(&stmt) + }; + + let mut ty = None; + + // When the values the returning clause needs were extracted as + // constants above, the Const node feeds them. Otherwise, ask the + // database to return the remaining columns when it supports that. + // MySQL's exact single-row auto-increment result is represented by an + // Insert operation without a SQL RETURNING clause. + if const_returning.is_none() && !self.load_data.select_items.is_empty() { + if self.planner.engine.capability().sql.is_some() + && !self.planner.engine.capability().returning_from_insert + { + self.verify_insert_returning_without_capability(&stmt)?; + ty = Some( + self.load_data + .select_items + .infer_record_list_ty(&self.planner.engine.expr_cx_for(&stmt)), + ); + } else { + stmt.set_returning_project(stmt::Expr::record( + self.load_data + .select_items + .iter() + .map(|item| item.to_expr()), + )); + } + } + + let input_args: Vec<_> = self + .load_data + .inputs + .iter() + .map(|input| self.planner.mir.ty(*input).clone()) + .collect(); + let ty = ty.unwrap_or_else(|| self.planner.engine.infer_ty(&stmt, &input_args[..])); + let inputs = mem::take(&mut self.load_data.inputs); + + let node = if !self.planner.engine.capability().sql() && stmt.is_upsert() { + mir::Operation::Upsert(Box::new(mir::Upsert { + inputs, + stmt: stmt.into_insert_unwrap(), + ty, + })) + } else { + mir::Operation::ExecStatement(Box::new(mir::ExecStatement { + inputs, + stmt, + ty, + conditional: exec::ConditionalOutput::None, + pagination: None, + })) + }; + + let mut load_data_node = self.insert_mir_with_deps(node); + + if let Some((const_value, const_ty)) = const_returning { + load_data_node = self.planner.mir.insert_with_deps( + mir::Const { + value: const_value, + ty: const_ty, + }, + [load_data_node], + ); + } + + Ok(load_data_node) + } + + /// Accepts the one exact mutation result available without SQL + /// `RETURNING`: MySQL's generated ID for a single-row insert. + fn verify_insert_returning_without_capability(&self, stmt: &stmt::Statement) -> Result<()> { + let insert = stmt.as_insert().expect("plan_insert requires an insert"); + let target = insert.target.as_table_unwrap(); + + let exact_mysql_insert_id = self.planner.engine.capability().sql == Some(Dialect::Mysql) + && insert.upsert.is_none() + && insert + .source + .body + .as_values() + .is_some_and(|values| values.rows.len() == 1) + && self.load_data.select_items.len() == 1 + && self + .load_data + .select_items + .iter() + .next() + .is_some_and(|item| { + let SelectItem::ExprReference(expr_ref) = item else { + return false; + }; + let Some(expr_column) = expr_ref.as_expr_column() else { + return false; + }; + if expr_column.nesting != 0 || expr_column.table != 0 { + return false; + } + + let column_id = db::ColumnId { + table: target.table, + index: expr_column.column, + }; + let column = self.planner.engine.schema.db.column(column_id); + let Some(value_index) = target.columns.iter().position(|id| *id == column_id) + else { + return false; + }; + let row = &insert.source.body.as_values().unwrap().rows[0]; + + column.auto_increment + && row + .entry(value_index) + .is_some_and(|entry| entry.is_expr_default()) + }); + + if exact_mysql_insert_id { + return Ok(()); + } + + let table = self.planner.engine.schema.db.table(target.table); + let columns = self + .load_data + .select_items + .iter() + .map(|item| { + let SelectItem::ExprReference(expr_ref) = item else { + return "the computed value `COUNT(*)`".to_owned(); + }; + let Some(column) = expr_ref.as_expr_column() else { + return "a non-column value".to_owned(); + }; + format!("`{}.{}`", table.name, table.columns[column.column].name) + }) + .collect::>() + .join(", "); + + Err(toasty_core::Error::unsupported_feature(format!( + "{} cannot return database-generated values from INSERT for {columns}; use \ + caller-generated values, insert each row separately, or use a database with \ + mutation RETURNING support", + self.planner.engine.capability().driver_name + ))) + } + + // ===== SQL execution ===== + + fn plan_data_loading_sql(&mut self, mut stmt: stmt::Statement) -> Result { + debug_assert!(self.planner.engine.capability().sql(), "stmt={stmt:#?}"); + debug_assert!(!stmt.is_insert(), "stmt={stmt:#?}"); + + // Phase 1: Detect pagination and add ORDER BY columns to load_data + let pagination_info = self.plan_pagination_sql(&stmt)?; + + // Set returning clause with all columns (including added ORDER BY columns) + if !self.load_data.select_items.is_empty() { + stmt.set_returning_project(stmt::Expr::record( + self.load_data + .select_items + .iter() + .map(|item| item.to_expr()), + )); + } + + let input_args: Vec<_> = self + .load_data + .inputs + .iter() + .map(|input| self.planner.mir.ty(*input).clone()) + .collect(); + + // Infer type after adding all columns + let ty = self.planner.engine.infer_ty(&stmt, &input_args[..]); + + // Phase 2: Build extract_cursor function using the inferred type + let pagination_config = pagination_info.map(|info| self.build_extract_cursor(info, &ty)); + + let node = if stmt.condition().is_some() { + // A conditional UPDATE or DELETE (e.g. an OCC `#[version]` check). + // The condition is checked against the current rows and the write + // only applies when it holds; a mismatch surfaces as an error. Two + // strategies, chosen by capability: a single CTE statement + // (PostgreSQL) or a read-modify-write transaction (SQLite, MySQL). + debug_assert!( + stmt.is_update() || stmt.is_delete(), + "only UPDATE and DELETE carry conditions; stmt={stmt:#?}" + ); + + // A conditional UPDATE or DELETE compiles to a single CTE statement + // on backends that support data-modifying CTEs (PostgreSQL); + // elsewhere it becomes a read-modify-write transaction (which every + // SQL backend supports, locking the probed rows via `SELECT ... FOR + // UPDATE` where available). + if self.planner.engine.capability().cte_with_update { + mir::Operation::ExecStatement(Box::new( + self.plan_conditional_sql_query_as_cte(stmt, ty), + )) + } else { + mir::Operation::ReadModifyWrite(Box::new( + self.plan_conditional_sql_query_as_rmw(stmt, ty), + )) + } + } else { + debug_assert!( + stmt.returning() + .and_then(|returning| returning.as_project()) + .map(|expr| expr.is_record()) + .unwrap_or(true), + "stmt={stmt:#?}" + ); + let inputs = mem::take(&mut self.load_data.inputs); + + // With SQL capability, we can just punt the details of execution to + // the database's query planner. + mir::Operation::ExecStatement(Box::new(mir::ExecStatement { + inputs, + stmt, + ty, + conditional: exec::ConditionalOutput::None, + pagination: pagination_config, + })) + }; + + Ok(self.insert_mir_with_deps(node)) + } + + fn extract_insert_returning_as_const( + &mut self, + stmt: &stmt::Statement, + ) -> Option<(stmt::Value, stmt::Type)> { + let stmt::Statement::Insert(insert) = stmt else { + return None; + }; + + if self.load_data.select_items.is_empty() { + return None; + } + + let target = insert.target.as_table_unwrap(); + let values = insert.source.body.as_values()?; + + let mut indices = vec![]; + + for select_item in &self.load_data.select_items { + let expr_ref = select_item.as_expr_reference_unwrap(); + let expr_col = expr_ref.as_expr_column_unwrap(); + debug_assert!(expr_col.nesting == 0, "expr_column={expr_col:#?}"); + + let Some(index) = target + .columns + .iter() + .enumerate() + .find(|(_, column_id)| column_id.index == expr_col.column) + .map(|(index, _)| index) + else { + todo!("insert returning referencing parent statement"); + // return None; + }; + + indices.push(index); + } + + // Now extract the values for each row + let mut result = Vec::with_capacity(values.rows.len()); + + for row in &values.rows { + // Build a record with only the requested fields + let mut fields = Vec::with_capacity(indices.len()); + + for &index in &indices { + // Try to evaluate the expression to a constant value + let value = row.entry(index)?.eval_const().ok()?; + fields.push(value); + } + + result.push(stmt::Value::record_from_vec(fields)); + } + + let ty = self + .load_data + .select_items + .infer_record_list_ty(&self.planner.engine.expr_cx_for(stmt)); + + Some((stmt::Value::List(result), ty)) + } + + fn plan_pagination_sql(&mut self, stmt: &stmt::Statement) -> Result> { + let stmt::Statement::Query(query) = stmt else { + return Ok(None); + }; + + // Only cursor-based limits trigger pagination planning + let Some(limit) = query.limit.as_ref() else { + return Ok(None); + }; + let stmt::Limit::Cursor(cursor) = limit else { + return Ok(None); + }; + + // SQL cursor pagination requires ORDER BY to produce a deterministic cursor. + let order_by = query.order_by.as_ref().ok_or_else(|| { + toasty_core::Error::unsupported_feature(format!( + "cursor-based pagination on {} requires an ORDER BY clause", + self.planner.engine.capability().driver_name + )) + })?; + + // Extract page_size + let page_size = match &cursor.page_size { + stmt::Expr::Value(stmt::Value::I64(n)) => *n, + _ => return Ok(None), + }; + + // Add ORDER BY columns to load_data so they're available for cursor extraction + let mut cursor_column_indices = Vec::new(); + + for order_expr in &order_by.exprs { + // Try to convert the ORDER BY expression to an ExprReference + if let Some(expr_ref) = order_expr.expr.as_expr_reference().copied() { + // Add to load_data if not already present + let (index, _) = self + .load_data + .select_items + .insert_full(SelectItem::from(expr_ref)); + cursor_column_indices.push(index); + } else { + // Complex expression in ORDER BY - can't handle yet + return Ok(None); + } + } + + Ok(Some(PaginationInfo { + page_size, + has_previous_page: self.stmt_info.has_pagination_cursor, + cursor_column_indices, + })) + } + + /// Builds extract_cursor function using the inferred row type. + /// Called after type inference in plan_data_loading_sql. + fn build_extract_cursor( + &self, + info: PaginationInfo, + ty: &stmt::Type, + ) -> exec::PaginationConfig { + // Extract row type from List + let row_ty = if let stmt::Type::List(item_ty) = ty { + (**item_ty).clone() + } else { + stmt::Type::Unit + }; + + // Build extract_cursor expression: projects ORDER BY column positions from the row + let extract_cursor = stmt::Expr::record( + info.cursor_column_indices + .into_iter() + .map(|index| stmt::Expr::arg_project(0, [index])), + ); + + // Build eval::Func with the actual row type as input + let extract_cursor_func = eval::Func::from_stmt(extract_cursor, vec![row_ty]); + + exec::PaginationConfig { + page_size: info.page_size, + has_previous_page: info.has_previous_page, + extract_cursor: Some(extract_cursor_func), + } + } + + fn plan_conditional_sql_query_as_cte( + &mut self, + stmt: stmt::Statement, + ty: stmt::Type, + ) -> mir::ExecStatement { + let (condition, filter, source, mut write) = self.conditional_write_parts(stmt); + + // `found`: the probe. Projects the OCC condition once per row matching + // the filter, locking those rows (`FOR UPDATE`) so the condition is + // evaluated against the latest committed row version and the rows + // cannot change before the write applies. Without the lock, a stale + // writer blocking on a concurrent committed update would pass the + // write's re-check (the probe is already materialized from the old + // snapshot and the write's own filter is key-only) and silently + // overwrite the newer row — the lost update `#[version]` exists to + // prevent. + let found = stmt::Cte { + query: conditional_probe_query( + condition, + filter.clone(), + source, + self.planner.engine.capability().select_for_update, + ), + }; + + // `counts`: aggregates the probe rows into `[matched, conditioned]`. + // Kept separate from `found` because a locking SELECT cannot carry + // aggregates. `found` sits one scope out (CTE body → WITH). + let counts = stmt::Cte { + query: stmt::Query::builder(stmt::Select { + source: stmt::TableRef::Cte { + nesting: 1, + index: 0, + } + .into(), + filter: stmt::Filter::new(true), + returning: stmt::Returning::Project(conditional_probe_projection(cte_column(0, 0))), + distinct: false, + }) + .build(), + }; + + // `changed`: the write. It applies only when the probe's two counts + // agree, expressed as a scalar subquery over `counts` ANDed onto the + // filter. `counts` sits two scopes out from here (subquery → CTE body → + // WITH). + let counts_agree = stmt::Expr::stmt(stmt::Select { + source: stmt::TableRef::Cte { + nesting: 2, + index: 1, + } + .into(), + filter: true.into(), + returning: stmt::Returning::Project(stmt::Expr::record_from_vec(vec![stmt::Expr::eq( + stmt::ExprColumn { + nesting: 0, + table: 0, + column: 0, + }, + stmt::ExprColumn { + nesting: 0, + table: 0, + column: 1, + }, + )])), + distinct: false, + }); + // The write applies only when the two counts agree — AND that guard + // onto the write's filter. + let guarded_filter = stmt::Filter::new(stmt::Expr::and(filter, counts_agree)); + match &mut write { + stmt::Statement::Update(update) => update.filter = guarded_filter, + stmt::Statement::Delete(delete) => delete.filter = guarded_filter, + _ => unreachable!("conditional write is UPDATE or DELETE; write={write:#?}"), + } + + // A `RETURNING` on the write (e.g. a relative `value = value + 1` read + // back) means the caller wants the changed rows; otherwise the write + // just reports how many rows it touched. Only an UPDATE reads columns + // back — a DELETE never has a returning. + let returning_len = match write.returning() { + Some(stmt::Returning::Project(stmt::Expr::Record(record))) => record.fields.len(), + Some(returning) => todo!("unexpected conditional write returning={returning:#?}"), + None => 0, + }; + + // `changed`: the write, wrapped as a data-modifying CTE body. + let changed_body: stmt::ExprSet = match write { + stmt::Statement::Update(update) => update.into(), + stmt::Statement::Delete(delete) => delete.into(), + _ => unreachable!("conditional write is UPDATE or DELETE"), + }; + let changed = stmt::Cte { + query: stmt::Query::new(changed_body), + }; + + let outer = if returning_len == 0 { + // No columns to read back: select just the two probe counts. The + // `changed` CTE is data-modifying, so PostgreSQL runs it even though + // the outer query does not reference it. + stmt::Query::builder(stmt::Select { + source: stmt::TableRef::Cte { + nesting: 0, + index: 1, + } + .into(), + filter: stmt::Filter::new(true), + returning: stmt::Returning::Project(stmt::Expr::record_from_vec(vec![ + cte_column(0, 0), + cte_column(0, 1), + ])), + distinct: false, + }) + } else { + // Read the probe counts alongside the changed rows. `counts` yields + // exactly one row, so the LEFT JOIN repeats the counts across every + // changed row (and yields a single NULL-padded row when nothing + // changed). + let mut columns = vec![cte_column(0, 0), cte_column(0, 1)]; + for i in 0..returning_len { + columns.push(cte_column(1, i)); + } + + stmt::Query::builder(stmt::Select { + source: stmt::Source::table_with_joins( + vec![ + stmt::TableRef::Cte { + nesting: 0, + index: 1, + }, + stmt::TableRef::Cte { + nesting: 0, + index: 2, + }, + ], + stmt::TableWithJoins { + relation: stmt::TableFactor::Table(stmt::SourceTableId(0)), + joins: vec![stmt::Join { + table: stmt::SourceTableId(1), + constraint: stmt::JoinOp::Left(stmt::Expr::from(true)), + }], + }, + ), + filter: stmt::Filter::new(true), + returning: stmt::Returning::Project(stmt::Expr::record_from_vec(columns)), + distinct: false, + }) + }; + + let stmt = outer.with(vec![found, counts, changed]).build().into(); + + mir::ExecStatement { + inputs: mem::take(&mut self.load_data.inputs), + stmt, + ty, + conditional: if returning_len == 0 { + exec::ConditionalOutput::Count + } else { + exec::ConditionalOutput::Returning + }, + pagination: None, + } + } + + fn plan_conditional_sql_query_as_rmw( + &mut self, + stmt: stmt::Statement, + ty: stmt::Type, + ) -> mir::ReadModifyWrite { + let (condition, filter, source, write) = self.conditional_write_parts(stmt); + + // A conditional write on a backend without data-modifying CTEs (SQLite, + // MySQL) becomes a transaction: first probe the matched rows, then apply + // the write (filter only, no condition) when every matched row satisfies + // the condition. The engine derives the matched and satisfied counts + // from the probe's per-row results. + let read = conditional_probe_query( + condition, + filter, + source, + self.planner.engine.capability().select_for_update, + ); + + mir::ReadModifyWrite { + inputs: mem::take(&mut self.load_data.inputs), + read, + write, + ty, + } + } + + /// Decompose a conditional UPDATE/DELETE into the parts both SQL + /// conditional-write strategies need: the OCC condition, the row filter, a + /// SELECT source over the target table (for the count probe), and the bare + /// write statement with its condition stripped. The filter and any + /// `RETURNING` stay on the write. + fn conditional_write_parts( + &self, + mut stmt: stmt::Statement, + ) -> (stmt::Expr, stmt::Expr, stmt::Source, stmt::Statement) { + let condition = stmt + .condition_mut_unwrap() + .expr + .take() + .expect("conditional write without condition"); + + let filter = stmt + .filter() + .and_then(|filter| filter.expr.clone()) + .expect("conditional write without filter"); + + let source = match &stmt { + stmt::Statement::Update(update) => stmt::Source::table(update.target.as_table_unwrap()), + stmt::Statement::Delete(delete) => delete.from.clone(), + _ => unreachable!("conditional write is UPDATE or DELETE; stmt={stmt:#?}"), + }; + + (condition, filter, source, stmt) + } + + // ===== NoSQL execution ===== + + fn plan_data_loading_nosql(&mut self, stmt: stmt::Statement) -> Result { + if stmt.is_insert() { + debug_assert!(self.load_data.select_items.is_empty()); + } + + // Without SQL capability, we have to plan the execution of the + // statement based on available indices. + let index_plan_opt = self.planner.engine.plan_index_path(&stmt)?; + + if let Some(mut index_plan) = index_plan_opt { + // prepare_post_filter may insert additional columns (e.g. deferred + // fields used only in a post-filter) into select_items, so the + // record type must be computed AFTER this call. + let post_filter = self.prepare_post_filter(&stmt, &mut index_plan); + + let ty = self.infer_nosql_record_ty(&stmt); + + let node_id = if index_plan.index.primary_key { + self.plan_primary_key_execution(stmt, &mut index_plan, &ty) + } else { + self.plan_secondary_index_execution(stmt, &mut index_plan, &ty) + }; + + Ok(self.apply_post_filter(node_id, post_filter, ty)) + } else { + // No index covers the filter — emit a full-table scan. + let ty = self.infer_nosql_record_ty(&stmt); + self.plan_scan_execution(stmt, ty) + } + } + + fn infer_nosql_record_ty(&self, stmt: &stmt::Statement) -> stmt::Type { + if self.load_data.select_items.is_empty() { + if stmt.is_query() { + // Query with no columns selected is an existence check: return + // an empty record for every matching row. + stmt::Type::list(stmt::Type::Record(vec![])) + } else { + stmt::Type::Unit + } + } else { + self.load_data + .select_items + .infer_record_list_ty(&self.planner.engine.expr_cx_for(stmt)) + } + } + + fn plan_scan_execution( + &mut self, + stmt: stmt::Statement, + ty: stmt::Type, + ) -> Result { + let input = if self.load_data.inputs.is_empty() { + None + } else if self.load_data.inputs.len() == 1 { + Some(self.load_data.inputs[0]) + } else { + todo!("scan with multiple inputs") + }; + + let cx = stmt::ExprContext::new(&*self.planner.engine.schema); + let cx = cx.scope(&stmt); + let stmt::ExprTarget::Table(table) = cx.target() else { + return Err(toasty_core::Error::unsupported_feature( + "scan: expected table target", + )); + }; + let table_id = table.id; + + // Reject ORDER BY on drivers whose scan operation returns items in an + // unspecified order (e.g. DynamoDB Scan has no server-side sort). + if let Some(query) = stmt.as_query() + && query.order_by.is_some() + && !self.planner.engine.capability().scan_supports_sort + { + return Err(toasty_core::Error::unsupported_feature(format!( + "{} does not support ORDER BY on full-table scans. Consider adding an index on \ + the sort field or removing the ORDER BY clause.", + self.planner.engine.capability().driver_name + ))); + } + + let mut row_filter = { + let f = stmt.filter_expr_unwrap(); + if f.is_true() { None } else { Some(f.clone()) } + }; + self.legalize_kv_expr(&mut row_filter); + + let limit = extract_pagination(&stmt); + + Ok(self.insert_mir_with_deps(mir::Scan { + input, + table: table_id, + columns: self.load_data.select_items.extract_expr_references(), + row_filter, + limit, + ty, + })) + } + + fn plan_primary_key_execution( + &mut self, + stmt: stmt::Statement, + index_plan: &mut index::IndexPlan, + ty: &stmt::Type, + ) -> mir::NodeId { + if let Some(mut key_expr) = index_plan.key_values.take() { + let (args, input_nodes) = self.rewrite_expr_for_mir(&mut key_expr, 0); + let key_ty = + stmt::Type::list(self.planner.engine.index_key_record_ty(index_plan.index)); + let keys = eval::Func::from_stmt_typed(key_expr, args, key_ty); + let get_by_key_input = + self.build_get_by_key_input(keys, input_nodes, self.index_key_ty(index_plan)); + + self.build_key_operation(&stmt, index_plan, get_by_key_input, ty) + } else { + let input = if self.load_data.inputs.is_empty() { + None + } else if self.load_data.inputs.len() == 1 { + Some(self.load_data.inputs[0]) + } else { + todo!() + }; + + if stmt.is_query() { + let limit = extract_pagination(&stmt); + let order = extract_query_pk_order(&stmt); + + let mut row_filter = index_plan.result_filter.take(); + self.legalize_kv_expr(&mut row_filter); + + // For queries, stream all matching records with the requested columns. + self.insert_mir_with_deps(mir::QueryPk { + input, + table: index_plan.table_id(), + index: None, // Querying primary key + columns: self.load_data.select_items.extract_expr_references(), + pk_filter: index_plan.index_filter.take(), + row_filter, + ty: ty.clone(), + limit, + order, + }) + } else { + // For mutations (UPDATE/DELETE) with a partial primary-key filter, + // first collect the full primary keys of all matching records via + // QueryPk, then apply the mutation to each key. The index key columns + // were pre-populated into load_data.select_items in plan_data_loading_nosql. + let index_key_ty = self.index_key_ty(index_plan); + + let mut columns = self.load_data.select_items.extract_expr_references(); + assert!(columns.is_empty()); + + for index_col in &index_plan.index.columns { + columns.insert(stmt::ExprReference::Column(stmt::ExprColumn { + nesting: 0, + table: 0, + column: index_col.column.index, + })); + } + + let mut row_filter = index_plan.result_filter.take(); + self.legalize_kv_expr(&mut row_filter); + + let query_pk_node = self.insert_mir_with_deps(mir::QueryPk { + input, + table: index_plan.table_id(), + index: None, // Querying primary key + columns, + pk_filter: index_plan.index_filter.take(), + row_filter, + ty: index_key_ty, + limit: None, + order: None, + }); + + self.build_key_operation(&stmt, index_plan, query_pk_node, ty) + } + } + } + + fn plan_secondary_index_execution( + &mut self, + stmt: stmt::Statement, + index_plan: &mut index::IndexPlan, + ty: &stmt::Type, + ) -> mir::NodeId { + let inputs = mem::take(&mut self.load_data.inputs); + assert!(index_plan.post_filter.is_none(), "TODO"); + assert!(inputs.len() <= 1, "TODO: inputs={:#?}", inputs); + + // For queries on NON-UNIQUE indexes, use QueryPk optimization + // Non-unique indexes are created as DynamoDB GSIs with ProjectionType::All, + // meaning they contain all table columns and can return full records. + // Unique indexes are created as separate tables and don't support this optimization. + // This method is already on the nosql path, so we can assume the behavior of unique indexes. + // To support DDB's other projection types, Index will need to track projected columns, not + // just the columns that are part of the index key. + if stmt.is_query() && !index_plan.index.unique { + let input = if inputs.is_empty() { + None + } else { + Some(inputs[0]) + }; + + let limit = extract_pagination(&stmt); + let order = extract_query_pk_order(&stmt); + + let mut row_filter = index_plan.result_filter.take(); + self.legalize_kv_expr(&mut row_filter); + + // Use QueryPk with index to query the secondary index and return full records + // This eliminates the N+1 pattern of FindPkByIndex + GetByKey + return self.insert_mir_with_deps(mir::QueryPk { + input, + table: index_plan.index.on, + index: Some(index_plan.index.id), // Query the secondary index + columns: self.load_data.select_items.extract_expr_references(), // Return full records + pk_filter: index_plan.index_filter.take(), + row_filter, + ty: ty.clone(), // Full record type, not just PKs + limit, + order, + }); + } + + // For mutations, unique indexes, or other cases, use FindPkByIndex + GetByKey + // - Mutations only need primary keys, not full records + // - Unique indexes don't have full column projections in DynamoDB + let primary_key_ty = self.table_primary_key_ty(index_plan.index.on); + + let get_by_key_input = self.insert_mir_with_deps(mir::FindPkByIndex { + inputs, + table: index_plan.index.on, + index: index_plan.index.id, + filter: index_plan.index_filter.take(), + ty: primary_key_ty, + }); + + self.build_key_operation(&stmt, index_plan, get_by_key_input, ty) + } + + fn prepare_post_filter( + &mut self, + stmt: &stmt::Statement, + index_plan: &mut index::IndexPlan, + ) -> Option { + let mut post_filter = index_plan.post_filter.clone(); + + // If fetching rows using GetByKey, some databases do not support + // applying additional filters to the rows before returning results. + // In this case, the result_filter needs to be applied in-memory. + if stmt.is_query() + && (index_plan.has_pk_keys || !index_plan.index.primary_key) + && let Some(result_filter) = index_plan.result_filter.take() + { + post_filter = Some(match post_filter { + Some(post_filter) => stmt::Expr::and(result_filter, post_filter), + None => result_filter, + }); + } + + debug_assert!( + post_filter.is_none() || stmt.is_query(), + "stmt={:#?}; post_filter={post_filter:#?}", + stmt + ); + + // Make sure we are including columns needed to apply the post filter + if let Some(post_filter) = &mut post_filter { + visit_mut::for_each_expr_mut(post_filter, |expr| match expr { + stmt::Expr::Reference(expr_reference) => { + let (index, _) = self + .load_data + .select_items + .insert_full((*expr_reference).into()); + *expr = stmt::Expr::arg_project(0, [index]); + } + stmt::Expr::Arg(_) => todo!("expr={expr:#?}"), + _ => {} + }); + } + + post_filter + } + + fn apply_post_filter( + &mut self, + mut node_id: mir::NodeId, + post_filter: Option, + ty: stmt::Type, + ) -> mir::NodeId { + // If there is a post filter, we need to apply a filter step on the returned rows. + if let Some(post_filter) = post_filter { + let item_ty = ty.as_list_unwrap(); + node_id = self.planner.mir.insert(mir::Filter { + input: node_id, + args: IndexSet::new(), + predicate: eval::Func::from_stmt(post_filter, vec![item_ty.clone()]), + ty, + }); + } + + node_id + } + + fn build_get_by_key_input( + &mut self, + keys: eval::Func, + input_nodes: IndexSet, + index_key_ty: stmt::Type, + ) -> mir::NodeId { + if keys.is_const() { + let keys = keys.eval_const(&self.planner.engine.schema); + self.insert_const(keys, index_key_ty) + } else if keys.is_identity() { + debug_assert_eq!(1, input_nodes.len(), "TODO"); + input_nodes[0] + } else { + // The function maps the referenced inputs' whole values to the + // full key list, same as the `is_const`/`is_identity` cases + // above. + self.planner + .mir + .insert(mir::Eval::compute(input_nodes, keys)) + } + } + + /// Legalize a driver-bound key-value operation expression (a filter or + /// condition the driver compiles, e.g. into a DynamoDB expression): + /// projections into `#[document]` columns become resolved + /// `FuncJsonExtract` name paths, mirroring what legalization does to full + /// statements at the SQL boundary. In-memory expressions (post filters, + /// guards) are deliberately *not* legalized — the interpreter wants the + /// positional form. + fn legalize_kv_expr(&self, expr: &mut Option) { + if let Some(expr) = expr { + self.planner.engine.legalize_table_expr(expr); + } + } + + fn build_key_operation( + &mut self, + stmt: &stmt::Statement, + index_plan: &mut index::IndexPlan, + get_by_key_input: mir::NodeId, + ty: &stmt::Type, + ) -> mir::NodeId { + match stmt { + stmt::Statement::Query(_) => { + debug_assert!(ty.is_list(), "ty={ty:#?}"); + self.insert_mir_with_deps(mir::GetByKey { + input: get_by_key_input, + table: index_plan.table_id(), + columns: self.load_data.select_items.extract_expr_references(), + ty: ty.clone(), + }) + } + stmt::Statement::Delete(delete_stmt) => { + let mut filter = index_plan.result_filter.take(); + self.legalize_kv_expr(&mut filter); + let mut condition = delete_stmt.condition.expr.clone(); + self.legalize_kv_expr(&mut condition); + + self.insert_mir_with_deps(mir::DeleteByKey { + input: get_by_key_input, + table: index_plan.table_id(), + filter, + condition, + ty: stmt::Type::Unit, + }) + } + stmt::Statement::Update(update_stmt) => { + // If there is a pre-filter, filter the key input on it: a + // false pre-filter drops every key, causing UpdateByKey to + // naturally no-op. + let filtered_input = self.apply_pre_filter(get_by_key_input, index_plan); + + let mut filter = index_plan.result_filter.take(); + self.legalize_kv_expr(&mut filter); + let mut condition = update_stmt.condition.expr.clone(); + self.legalize_kv_expr(&mut condition); + + self.insert_mir_with_deps(mir::UpdateByKey { + input: filtered_input, + table: index_plan.table_id(), + // Document values in the assignments are already named: + // the mapping's lowering casts converted them during + // statement lowering/simplification. + assignments: update_stmt.assignments.clone(), + filter, + condition, + columns: self.load_data.select_items.extract_expr_references(), + ty: ty.clone(), + }) + } + _ => todo!("stmt={stmt:#?}"), + } + } + + /// If the index plan has a pre-filter, filter the key input on it: a + /// `Filter` over the key list whose predicate reads only its args (the + /// pre-filter's referenced statement outputs). When the + /// pre-filter is false every key is dropped, so the downstream operation + /// sees no keys and becomes a no-op. Returns the (possibly filtered) + /// input node ID. + fn apply_pre_filter( + &mut self, + input: mir::NodeId, + index_plan: &mut index::IndexPlan, + ) -> mir::NodeId { + let Some(mut pre_filter_expr) = index_plan.pre_filter.take() else { + return input; + }; + + // The predicate's `arg(0)` is the current row, so the pre-filter's + // inputs start at `arg(1)`. + let (arg_tys, args) = self.rewrite_expr_for_mir(&mut pre_filter_expr, 1); + + let ty = self.planner.mir[input].ty().clone(); + let mut func_args = vec![ty.as_list_unwrap().clone()]; + func_args.extend(arg_tys); + + self.planner.mir.insert(mir::Filter { + input, + args, + predicate: eval::Func::from_stmt(pre_filter_expr, func_args), + ty, + }) + } + + /// Rewrite a statement-level expression for use in a MIR node. + /// + /// Statement-level expressions contain `Arg(n)` where `n` is a position in + /// `stmt_info.args` (the HIR arg list). Each HIR arg maps to an entry in + /// `load_data.inputs` via `hir::Arg::Sub { input, .. }`. + /// + /// MIR nodes have their own compact input lists. This method: + /// 1. Resolves each HIR arg to its `load_data.inputs` node ID + /// 2. Assigns a new compact position, starting at `arg_offset` + /// 3. Rewrites the `Arg` position in the expression + /// + /// Only arguments that reference the statement-level scope are rewritten. + /// Arguments bound by nested `Map` or `Let` expressions remain unchanged. + /// + /// Returns `(arg_types, input_node_ids)` for constructing the MIR node. + fn rewrite_expr_for_mir( + &self, + expr: &mut stmt::Expr, + arg_offset: usize, + ) -> (Vec, IndexSet) { + let mut arg_map: IndexMap = IndexMap::new(); + + visit_mut::walk_expr_scoped_mut(expr, 0, |expr, scope_depth| { + if let stmt::Expr::Arg(expr_arg) = expr + && expr_arg.nesting == scope_depth + { + let hir_pos = expr_arg.position; + let new_pos = match arg_map.get_index_of(&hir_pos) { + Some(idx) => idx, + None => { + // Resolve the HIR arg to its load_data input + let input_idx = match &self.stmt_info.args[hir_pos] { + hir::Arg::Sub { input, .. } => input.get().unwrap(), + _ => todo!("rewrite_expr_for_mir with non-Sub arg"), + }; + let node_id = self.load_data.inputs[input_idx]; + let ty = self.planner.mir[node_id].ty().clone(); + let (idx, _) = arg_map.insert_full(hir_pos, (ty, node_id)); + idx + } + }; + expr_arg.position = arg_offset + new_pos; + } + + true + }); + + let mut types = Vec::with_capacity(arg_map.len()); + let mut nodes = IndexSet::with_capacity(arg_map.len()); + for (_, (ty, node_id)) in arg_map { + types.push(ty); + nodes.insert(node_id); + } + (types, nodes) + } + + // ===== Finalization helpers ===== + + fn process_back_ref_projections(&mut self, exec_stmt_node_id: mir::NodeId) { + for (child_stmt_id, back_ref) in &self.stmt_info.back_refs { + let projection = stmt::Expr::record(back_ref.exprs.iter().map(|expr_reference| { + let index = self + .load_data + .select_items + .get_index_of_expr_reference(*expr_reference); + stmt::Expr::arg_project(0, [index]) + })); + + let row_ty = match self.planner.mir[exec_stmt_node_id].ty() { + // Lists are flattened + stmt::Type::List(ty) => (**ty).clone(), + ty => ty.clone(), + }; + + let body = eval::Func::from_stmt(projection, vec![row_ty]); + let eval = + mir::Eval::map_over(&self.planner.mir, exec_stmt_node_id, IndexSet::new(), body); + let project_node_id = self.planner.mir.insert(eval); + let project_node_id = if self.stmt_info.stmt().is_query() + && let Some(predicate) = self.build_parent_only_back_ref_filter( + *child_stmt_id, + back_ref, + project_node_id, + ) { + self.planner.mir.insert(mir::Filter { + input: project_node_id, + args: IndexSet::new(), + predicate, + ty: self.planner.mir[project_node_id].ty().clone(), + }) + } else { + project_node_id + }; + back_ref.node_id.set(Some(project_node_id)); + } + } + + /// Builds a filter over the projected parent rows passed to a child query. + /// + /// Each eligible top-level conjunct is copied from the child query and + /// rewritten to read fields from the projected back-reference row. This + /// limits a variant-scoped include to parent rows from that variant. If + /// every row is removed, the child receives an empty batch. + fn build_parent_only_back_ref_filter( + &self, + child_stmt_id: hir::StmtId, + back_ref: &hir::BackRef, + project_node_id: mir::NodeId, + ) -> Option { + let child_stmt = &self.planner.hir[child_stmt_id]; + let stmt::Statement::Query(child_query) = child_stmt.stmt() else { + return None; + }; + let stmt::ExprSet::Select(child_select) = &child_query.body else { + return None; + }; + let child_filter = child_select.filter.expr.as_ref()?; + let top_level_conjuncts = match child_filter { + stmt::Expr::And(and) => and.operands.as_slice(), + expr => std::slice::from_ref(expr), + }; + let predicates = top_level_conjuncts + .iter() + .filter_map(|conjunct| { + self.rewrite_parent_only_conjunct(child_stmt, back_ref, conjunct.clone()) + }) + .collect::>(); + + if predicates.is_empty() { + return None; + } + + let stmt::Type::List(row_ty) = self.planner.mir[project_node_id].ty() else { + return None; + }; + + Some(eval::Func::from_stmt( + stmt::Expr::and_from_vec(predicates), + vec![(**row_ty).clone()], + )) + } + + /// Rewrites one child-query conjunct for a projected parent row. + /// + /// The conjunct must be evaluable in memory and reference this parent at + /// least once. Every statement-level argument must reference this parent; + /// arguments bound by nested `Map` and `Let` expressions remain local. + fn rewrite_parent_only_conjunct( + &self, + child_stmt: &hir::StatementInfo, + back_ref: &hir::BackRef, + mut conjunct: stmt::Expr, + ) -> Option { + if !conjunct.is_eval() { + return None; + } + + let mut eligible = true; + let mut references_parent = false; + + visit_mut::walk_expr_scoped_mut(&mut conjunct, 0, |expr, scope_depth| { + if !eligible { + return false; + } + if let stmt::Expr::Arg(arg) = expr { + if arg.nesting < scope_depth { + return false; + } + if let Some(hir::Arg::Ref { + stmt_id, + target_expr_ref, + .. + }) = child_stmt.args.get(arg.position) + && *stmt_id == self.stmt_id + && arg.nesting == scope_depth + { + let back_ref_column = back_ref.exprs.get_index_of(target_expr_ref).unwrap(); + *expr = stmt::Expr::arg_project( + stmt::ExprArg { + position: 0, + nesting: scope_depth, + }, + [back_ref_column], + ); + references_parent = true; + } else { + eligible = false; + } + return false; + } + true + }); + + (eligible && references_parent).then_some(conjunct) + } + + fn plan_child_statements(&mut self) -> Result<()> { + // Plan dependent child statements. Effect deps target ancestors, + // planned by the enclosing traversal. + for (&dep_stmt_id, &kind) in &self.stmt_info.deps { + if kind == hir::DepKind::Statement && !self.planner.hir[dep_stmt_id].independent { + self.planner.plan_statement(dep_stmt_id)?; + } + } + + for arg in &self.stmt_info.args { + let hir::Arg::Sub { stmt_id, .. } = arg else { + continue; + }; + + self.planner.plan_statement(*stmt_id)?; + } + + Ok(()) + } + + fn plan_output_node( + &mut self, + data_load_node_id: mir::NodeId, + returning: ReturningInfo, + ) -> mir::NodeId { + // First check for nested merge + if let Some(node_id) = self.planner.plan_nested_merge(self.stmt_id) { + // `plan_nested_merge` builds its own MIR node directly on + // `self.planner.mir`, so the parent statement's remaining HIR + // deps (e.g. a sibling `Disassociate` DELETE in an + // `stmt::apply([insert, remove])` batch) are never consumed. + // Attach them to the NestedMerge so the dep statements stay + // reachable from `completion` and end up in the exec plan. + self.apply_dependencies_to_node(node_id); + return node_id; + } + + let returning_arg_tys = |row_ty: Option| -> Vec { + row_ty + .into_iter() + .chain( + returning + .inputs + .iter() + .map(|input| self.planner.mir[input].ty().clone()), + ) + .collect() + }; + + // Then handle returning clause + if let Some(clause) = returning.clause { + match clause { + stmt::Returning::Expr(expr) => { + // Value variant contains a constant expression that can be evaluated + if let Ok(value) = expr.eval_const() { + let ty = value.infer_ty(); + + self.planner + .mir + .insert_with_deps(mir::Const { value, ty }, [data_load_node_id]) + } else { + let body = eval::Func::from_stmt(expr, returning_arg_tys(None)); + + let node_id = + self.insert_mir_with_deps(mir::Eval::compute(returning.inputs, body)); + + if !self.stmt().is_query() { + self.planner.mir[node_id].deps.insert(data_load_node_id); + } + + node_id + } + } + stmt::Returning::Project(projection) => { + if returning.reads_row { + let row_ty = match self.planner.mir[data_load_node_id].ty() { + stmt::Type::List(ty) => (**ty).clone(), + ty => panic!("per-row returning over a rowless data load; ty={ty:#?}"), + }; + let body = + eval::Func::from_stmt(projection, returning_arg_tys(Some(row_ty))); + + let eval = mir::Eval::map_over( + &self.planner.mir, + data_load_node_id, + returning.inputs, + body, + ); + self.insert_mir_with_deps(eval) + } else { + // The projection is fully constant — the constantize + // pipeline substituted every value, so nothing was + // requested from the database and the data load may + // yield only an affected-row count. Repeat the value + // once per row. + let projection = eval::Func::from_stmt(projection, vec![]); + let ty = stmt::Type::list(projection.ret.clone()); + let value = projection.eval_const(&self.planner.engine.schema); + + self.insert_mir_with_deps(mir::Repeat { + input: data_load_node_id, + value, + ty, + }) + } + } + returning => panic!("unexpected `stmt::Returning` kind; returning={returning:#?}"), + } + } else { + self.apply_dependencies_to_node(data_load_node_id); + data_load_node_id + } + } + + // ===== MIR/utility helpers ===== + + #[track_caller] + fn insert_const(&mut self, value: impl Into, ty: stmt::Type) -> mir::NodeId { + let value = value.into(); + + // Type check + debug_assert!( + ty.is_list(), + "const types must be of type `stmt::Type::List`" + ); + debug_assert!( + value.is_a(&self.planner.engine.schema.app, &ty), + "const type mismatch; expected={ty:#?}; actual={value:#?}", + ); + + self.planner.mir.insert(mir::Const { value, ty }) + } + + fn insert_mir_with_deps(&mut self, node: impl Into) -> mir::NodeId { + let node_id = self.planner.mir.insert(node); + self.apply_dependencies_to_node(node_id); + node_id + } + + fn apply_dependencies_to_node(&mut self, node_id: mir::NodeId) { + let node = &mut self.planner.mir[node_id]; + + self.remaining_deps.retain(|stmt_id| { + let dep_info = &self.planner.hir[stmt_id]; + + // A dep not yet planned (e.g. a child sub-statement planned after + // this statement's data loading) stays retained and attaches at a + // later node — ultimately the output node, by which point every + // dep is planned. Ancestors are never in this set; they are + // `effect_deps`, anchored on reserved data-load slots. + if let Some(dep_id) = dep_info.output.get() { + node.deps.insert(dep_id); + false + } else { + true + } + }); + } + + fn index_key_ty(&self, index_plan: &IndexPlan) -> stmt::Type { + // Type of the index key. Value for single index keys, record for + // composite. + stmt::Type::list(self.planner.engine.index_key_record_ty(index_plan.index)) + } + + fn table_primary_key_ty(&self, table: db::TableId) -> stmt::Type { + let table = self.planner.engine.schema.db.table(table); + let primary_key = self.planner.engine.schema.db.index(table.primary_key.index); + + stmt::Type::list(self.planner.engine.index_key_record_ty(primary_key)) + } + + fn stmt(&self) -> &stmt::Statement { + self.stmt_info.stmt.as_deref().unwrap() + } +} + +/// A reference to column `column` of CTE `table` in the outer query's `FROM` +/// (both at zero nesting). +fn cte_column(table: usize, column: usize) -> stmt::Expr { + stmt::Expr::column(stmt::ExprColumn { + nesting: 0, + table, + column, + }) +} + +/// The probe query shared by both SQL conditional-write strategies: +/// `SELECT FROM WHERE [FOR UPDATE]`. +/// +/// The condition is projected once per matched row rather than as an +/// aggregate: a per-row projection can carry `FOR UPDATE` while `SELECT +/// count(*) ... FOR UPDATE` is rejected. Locking the matched rows makes the +/// probe authoritative — the condition is evaluated against the latest +/// committed row version and the rows cannot change before the write applies. +fn conditional_probe_query( + condition: stmt::Expr, + filter: stmt::Expr, + source: stmt::Source, + lock: bool, +) -> stmt::Query { + stmt::Query::builder(stmt::Select { + source, + filter: stmt::Filter::new(filter), + returning: stmt::Returning::Project(stmt::Expr::record_from_vec(vec![condition])), + distinct: false, + }) + .locks(if lock { + vec![stmt::Lock::Update] + } else { + vec![] + }) + .build() +} + +/// The aggregate projection over the probe's per-row results: +/// `[count(*), count(*) FILTER (WHERE )]`. The first column counts rows +/// matching the filter; the second counts those that also satisfy the OCC +/// condition. The write is safe to apply exactly when the two agree. Used by +/// the CTE strategy's `counts` CTE; the read-modify-write strategy derives the +/// same counts client-side. +fn conditional_probe_projection(condition: stmt::Expr) -> stmt::Expr { + stmt::Expr::record_from_vec(vec![ + stmt::Expr::count_star(), + stmt::FuncCount { + arg: None, + filter: Some(Box::new(condition)), + } + .into(), + ]) +} + +/// Extract limit/pagination bounds from a query statement for use with +/// `QueryPk` on NoSQL drivers. Returns `None` when the statement has no limit +/// clause. +/// +/// Assumes `page_size`, `limit`, and `offset` fields are `I64` literals. Runtime +/// values use `Expr::Value`; the fixed limit emitted by `.first()` uses +/// `Expr::Static`. `verify::verify_limit_is_integer_literal` enforces this +/// invariant on the AST, so any other shape reaching here is a bug upstream. +fn extract_pagination(stmt: &stmt::Statement) -> Option { + let query = stmt.as_query()?; + match query.limit.as_ref()? { + stmt::Limit::Cursor(c) => { + let page_size = as_i64_literal(&c.page_size); + let after = c.after.as_ref().and_then(|e| match e { + stmt::Expr::Value(v) => Some(v.clone()), + _ => None, + }); + Some(Pagination::Cursor { page_size, after }) + } + stmt::Limit::Offset(lo) => { + let limit = as_i64_literal(&lo.limit); + let offset = lo.offset.as_ref().map(as_i64_literal); + Some(Pagination::Offset { limit, offset }) + } + } +} + +/// Extracts an `i64` from a bound or static `I64` literal expression. Panics on +/// any other shape — an invariant violation that `verify` should have caught. +fn as_i64_literal(expr: &stmt::Expr) -> i64 { + match expr { + stmt::Expr::Value(stmt::Value::I64(n)) | stmt::Expr::Static(stmt::Value::I64(n)) => *n, + _ => panic!("limit/offset must be an i64 literal; got {expr:#?}"), + } +} + +/// Extract the sort direction from a query statement's ORDER BY clause for use +/// with `QueryPk` on NoSQL drivers. +fn extract_query_pk_order(stmt: &stmt::Statement) -> Option { + let query = stmt.as_query()?; + query.order_by.as_ref().and_then(|ob| { + ob.exprs.first().map(|e| match e.order { + Some(stmt::Direction::Desc) => stmt::Direction::Desc, + _ => stmt::Direction::Asc, + }) + }) +} diff --git a/packages/toasty/src/engine/plan/statement/tests.rs b/packages/toasty/src/engine/plan/statement/tests.rs new file mode 100644 index 00000000000..200b412f24c --- /dev/null +++ b/packages/toasty/src/engine/plan/statement/tests.rs @@ -0,0 +1,129 @@ +use super::*; +use crate::engine::{Engine, HirStatement, test_util::test_schema}; +use toasty_core::driver::Capability; +use toasty_core::stmt::{Expr, ExprArg, ExprLet, Type, Value}; + +fn rewrite(expr: Expr) -> Option { + let engine = Engine::new(test_schema().into(), &Capability::SQLITE); + let mut hir = HirStatement::new(); + let parent = hir.new_statement_info(IndexMap::new()); + let other = hir.new_statement_info(IndexMap::new()); + let column = |column| { + stmt::ExprReference::Column(stmt::ExprColumn { + nesting: 0, + table: 0, + column, + }) + }; + let back_ref = hir::BackRef { + exprs: [column(3), column(7)].into(), + ..Default::default() + }; + let mut child = hir::StatementInfo::new(IndexMap::new()); + for stmt_id in [parent, other] { + child.args.push(hir::Arg::Ref { + target_expr_ref: column(7), + stmt_id, + nesting: 1, + data_load_input: Default::default(), + returning_input: Default::default(), + batch_load_index: Default::default(), + }); + } + child.args.push(hir::Arg::Sub { + stmt_id: other, + returning: false, + input: Default::default(), + batch_load_index: Default::default(), + }); + let mut planner = HirPlanner { + engine: &engine, + hir: &hir, + mir: mir::Store::new(), + }; + PlanStatement { + planner: &mut planner, + stmt_id: parent, + stmt_info: &hir[parent], + load_data: LoadData { + inputs: IndexSet::new(), + select_items: SelectItems::new(), + batch_load_args: IndexSet::new(), + }, + remaining_deps: vec![], + } + .rewrite_parent_only_conjunct(&child, &back_ref, expr) +} + +fn assert_parent_filter(expr: Expr) { + let expr = rewrite(expr).expect("parent-only predicate"); + let func = eval::Func::from_stmt(expr, vec![Type::Record(vec![Type::I64, Type::I64])]); + let schema = test_schema(); + for (value, expected) in [(42i64, true), (9, false)] { + let row = Value::record_from_vec(vec![0i64.into(), value.into()]); + assert_eq!(func.eval_bool(&schema, [row]).unwrap(), expected); + } +} + +fn scoped_arg(position: usize, nesting: usize) -> Expr { + Expr::arg(ExprArg { position, nesting }) +} + +#[test] +fn parent_filter_supports_in_list() { + assert_parent_filter(Expr::in_list(Expr::arg(0), Expr::list([42i64]))); +} + +#[test] +fn parent_filter_preserves_map_and_let_scopes() { + assert_parent_filter( + ExprLet { + bindings: vec![Expr::arg(0)], + body: Box::new(Expr::any(Expr::map( + Expr::list([42i64]), + Expr::and( + Expr::eq(Expr::arg(0), scoped_arg(0, 1)), + Expr::eq(Expr::arg(0), scoped_arg(0, 2)), + ), + ))), + } + .into(), + ); +} + +#[test] +fn parent_filter_finds_parent_only_in_map_body() { + assert_parent_filter(Expr::any(Expr::map( + Expr::list([42i64]), + Expr::eq(Expr::arg(0), scoped_arg(0, 1)), + ))); +} + +#[test] +fn parent_filter_rejects_other_dependencies() { + for dependency in [ + Expr::arg(1), + Expr::arg(2), + Expr::arg(3), + scoped_arg(0, 1), + Expr::Reference(stmt::ExprReference::Model { nesting: 0 }), + Expr::any(Expr::map(Expr::list([true]), scoped_arg(1, 1))), + ] { + assert!(rewrite(Expr::and(Expr::arg(0), dependency)).is_none()); + } +} + +#[test] +fn local_arguments_do_not_count_as_parent_references() { + assert!(rewrite(Expr::any(Expr::map(Expr::list([true]), Expr::arg(0)))).is_none()); + assert!( + rewrite( + ExprLet { + bindings: vec![Expr::from(false), Expr::from(true)], + body: Box::new(Expr::arg(1)), + } + .into() + ) + .is_none() + ); +} diff --git a/packages/toasty/src/engine/select_item.rs b/packages/toasty/src/engine/select_item.rs new file mode 100644 index 00000000000..3baa44b1e6d --- /dev/null +++ b/packages/toasty/src/engine/select_item.rs @@ -0,0 +1,126 @@ +use std::ops::{Deref, DerefMut}; + +use toasty_core::stmt; + +/// Represents an item that the engine can select from the database. +/// +/// This generalizes `ExprReference` to support both column references and +/// computed expressions like `COUNT(*)`. Using an enum that derives `Hash` and +/// `Eq` allows the planner to continue deduplicating select items via +/// `SelectItems`. +#[derive(Debug, Clone, Copy, PartialEq, Hash, Eq)] +pub(crate) enum SelectItem { + /// A reference to a column or field — the traditional case. + ExprReference(stmt::ExprReference), + + /// The `COUNT(*)` aggregate. SQL-only. + CountStar, +} + +impl SelectItem { + /// Returns the inner `ExprReference`, panicking if this is not one. + #[track_caller] + pub(crate) fn as_expr_reference_unwrap(&self) -> &stmt::ExprReference { + match self { + SelectItem::ExprReference(r) => r, + other => panic!("expected ExprReference, got {other:?}"), + } + } + + /// Infer the type of this select item given the expression context. + pub(crate) fn infer_ty(&self, cx: &stmt::ExprContext<'_>) -> stmt::Type { + match self { + SelectItem::ExprReference(expr_reference) => cx.infer_expr_reference_ty(expr_reference), + SelectItem::CountStar => stmt::Type::U64, + } + } + + /// Convert this select item into the corresponding expression. + pub(crate) fn to_expr(self) -> stmt::Expr { + match self { + SelectItem::ExprReference(expr_reference) => stmt::Expr::from(expr_reference), + SelectItem::CountStar => stmt::Expr::count_star(), + } + } +} + +impl From for SelectItem { + fn from(value: stmt::ExprReference) -> Self { + SelectItem::ExprReference(value) + } +} + +/// A set of [`SelectItem`]s backed by an [`IndexSet`](indexmap::IndexSet) for +/// deduplication and index-based lookup. +#[derive(Debug, Default, Clone)] +pub(crate) struct SelectItems(indexmap::IndexSet); + +impl SelectItems { + pub(crate) fn new() -> Self { + Self(indexmap::IndexSet::new()) + } + + /// Find the index of an `ExprReference` item, returning `None` if not + /// present. + pub(crate) fn try_get_index_of_expr_reference( + &self, + expr_reference: impl Into, + ) -> Option { + let item = SelectItem::ExprReference(expr_reference.into()); + self.0.get_index_of(&item) + } + + /// Find the index of an `ExprReference` item. + pub(crate) fn get_index_of_expr_reference( + &self, + expr_reference: impl Into, + ) -> usize { + self.try_get_index_of_expr_reference(expr_reference) + .unwrap() + } + + /// Find the index of the `CountStar` item. + pub(crate) fn get_index_of_count_star(&self) -> usize { + self.0.get_index_of(&SelectItem::CountStar).unwrap() + } + + /// Returns `Type::List(Type::Record(field_tys))` where each `field_ty` is + /// inferred from the corresponding select item. + pub(crate) fn infer_record_list_ty(&self, cx: &stmt::ExprContext<'_>) -> stmt::Type { + let field_tys = self.0.iter().map(|item| item.infer_ty(cx)).collect(); + stmt::Type::list(stmt::Type::Record(field_tys)) + } + + /// Extract only the `ExprReference` items from this set. + /// + /// Used by the NoSQL path where only column references are valid. + pub(crate) fn extract_expr_references(&self) -> indexmap::IndexSet { + self.0 + .iter() + .map(|item| *item.as_expr_reference_unwrap()) + .collect() + } +} + +impl<'a> IntoIterator for &'a SelectItems { + type Item = &'a SelectItem; + type IntoIter = indexmap::set::Iter<'a, SelectItem>; + + fn into_iter(self) -> Self::IntoIter { + self.0.iter() + } +} + +impl Deref for SelectItems { + type Target = indexmap::IndexSet; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for SelectItems { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} diff --git a/packages/toasty/src/engine/simplify.rs b/packages/toasty/src/engine/simplify.rs new file mode 100644 index 00000000000..d1d467e7309 --- /dev/null +++ b/packages/toasty/src/engine/simplify.rs @@ -0,0 +1,351 @@ +mod expr_and; +mod expr_any; +mod expr_binary_op; +mod expr_cast; +mod expr_exists; +mod expr_in_list; +mod expr_intersects; +mod expr_is_null; +mod expr_is_superset; +mod expr_list; +mod expr_map; +mod expr_or; +mod expr_project; +mod merge_in_subqueries; +mod stmt_query; + +use toasty_core::{ + driver::Capability, + schema::*, + stmt::{self, Expr, IntoExprTarget, Node, VisitMut}, +}; + +use crate::engine::{Engine, fold}; + +/// Statement and expression simplifier. +/// +/// [`Simplify`] implements the [`VisitMut`] trait to traverse and transform +/// statement ASTs. It applies optimization and normalization rules defined in +/// submodules of [`engine::simplify`](self). +/// +/// Simplification runs twice during query compilation: once before lowering +/// (to normalize the input) and once after (to clean up generated expressions). +pub(crate) struct Simplify<'a> { + /// Expression context providing schema access and type information. + cx: stmt::ExprContext<'a>, + /// Driver capabilities, consulted by passes that emit driver-specific shapes. + capability: &'a Capability, + /// Only whether the expression is true matters in a positive filter. + positive_filter: bool, +} + +impl Engine { + /// Simplifies a statement or expression in place. + pub(crate) fn simplify_stmt(&self, stmt: &mut T) { + Simplify::new(&self.schema, self.capability).visit_mut(stmt); + } +} + +/// Simplifies an expression in place using the given context and capability. +pub(crate) fn simplify_expr( + cx: stmt::ExprContext<'_>, + capability: &Capability, + expr: &mut stmt::Expr, +) { + Simplify::with_context(cx, capability).visit_expr_mut(expr); +} + +impl VisitMut for Simplify<'_> { + fn visit_expr_mut(&mut self, i: &mut stmt::Expr) { + // Only AND/OR propagate positive filtering to their operands. Under + // NOT, IS NULL, or a returned expression, false and null must remain + // distinguishable. Nested statement filters establish their own context. + let positive_filter = self.positive_filter; + self.positive_filter &= matches!(i, Expr::And(_) | Expr::Or(_)); + stmt::visit_mut::visit_expr_mut(self, i); + self.positive_filter = positive_filter; + + // Fold this node bottom-up so heavyweight rules see canonical input. + // Children are already canonical (post-order recursion), so this is + // effectively local. + fold::fold_stmt(i); + + let maybe_expr = match i { + Expr::Any(expr) => self.simplify_expr_any(expr), + Expr::And(expr) => self.simplify_expr_and(expr), + Expr::BinaryOp(expr) => { + self.simplify_expr_binary_op(expr.op, &mut expr.lhs, &mut expr.rhs) + } + Expr::Cast(expr) => self.simplify_expr_cast(expr), + Expr::Exists(expr) => self.simplify_expr_exists(expr), + Expr::InList(expr) => self.simplify_expr_in_list(expr), + Expr::Intersects(expr) => self.simplify_expr_intersects(expr), + Expr::IsSuperset(expr) => self.simplify_expr_is_superset(expr), + Expr::List(expr) => self.simplify_expr_list(expr), + Expr::Map(_) => self.simplify_expr_map(i), + Expr::Or(expr) => self.simplify_expr_or(expr), + Expr::IsNull(expr) => self.simplify_expr_is_null(expr), + Expr::Project(expr) => self.simplify_expr_project(expr), + _ => None, + }; + + if let Some(mut expr) = maybe_expr { + // Heavyweight rules may emit new fold-eligible structure + // (e.g., match elimination produces ANDs containing constants + // that need short-circuiting). + fold::fold_stmt(&mut expr); + *i = expr; + } + } + + fn visit_filter_mut(&mut self, filter: &mut stmt::Filter) { + let previous = std::mem::replace(&mut self.positive_filter, true); + stmt::visit_mut::visit_filter_mut(self, filter); + self.positive_filter = previous; + } + + fn visit_expr_match_mut(&mut self, i: &mut stmt::ExprMatch) { + // Simplify the subject first. + self.visit_expr_mut(&mut i.subject); + + // If the subject simplified to a constant, only simplify the matching arm. + // Skipping dead-code arms avoids panics on expressions like + // `project([1], Record([I64(disc)]))` that would be invalid to evaluate. + if let Expr::Value(ref value) = *i.subject { + let value = value.clone(); + for arm in &mut i.arms { + if arm.pattern == value { + self.visit_expr_mut(&mut arm.expr); + return; + } + } + } else { + for arm in &mut i.arms { + self.visit_expr_mut(&mut arm.expr); + } + } + } + + fn visit_expr_set_mut(&mut self, i: &mut stmt::ExprSet) { + match i { + stmt::ExprSet::SetOp(expr_set_op) if expr_set_op.operands.is_empty() => { + todo!("is there anything we do here?"); + } + stmt::ExprSet::SetOp(expr_set_op) if expr_set_op.operands.len() == 1 => { + let operand = expr_set_op.operands.drain(..).next().unwrap(); + *i = operand; + } + stmt::ExprSet::SetOp(expr_set_op) if expr_set_op.is_union() => { + // First, simplify each sub-query in the union, then rewrite the + // query as a single disjuntive query. + let mut operands = vec![]; + + Self::flatten_nested_unions(expr_set_op, &mut operands); + + expr_set_op.operands = operands; + } + _ => {} + } + + stmt::visit_mut::visit_expr_set_mut(self, i); + } + + fn visit_stmt_delete_mut(&mut self, stmt: &mut stmt::Delete) { + // Visit and simplify source first before pushing a new scope + self.visit_source_mut(&mut stmt.from); + + let mut s = self.scope(&stmt.from); + + s.visit_filter_mut(&mut stmt.filter); + + if let Some(returning) = &mut stmt.returning { + s.visit_returning_mut(returning); + } + } + + fn visit_stmt_insert_mut(&mut self, stmt: &mut stmt::Insert) { + // Visit target first before pushing a new scope. + self.visit_insert_target_mut(&mut stmt.target); + + // Create a new scope for the insert target + let mut s = self.scope(&stmt.target); + + // First, simplify the source + s.visit_stmt_query_mut(&mut stmt.source); + + if let Some(upsert) = &mut stmt.upsert { + s.visit_assignments_mut(&mut upsert.shared); + s.visit_assignments_mut(&mut upsert.defaults); + s.visit_assignments_mut(&mut upsert.update_defaults); + s.visit_assignments_mut(&mut upsert.create); + s.visit_assignments_mut(&mut upsert.update); + } + + if let Some(returning) = &mut stmt.returning { + s.visit_returning_mut(returning); + } + } + + fn visit_stmt_query_mut(&mut self, stmt: &mut stmt::Query) { + stmt::visit_mut::visit_stmt_query_mut(self, stmt); + + self.simplify_stmt_query_when_empty(stmt); + } + + fn visit_source_mut(&mut self, source: &mut stmt::Source) { + // A `Source::Table` JOIN constraint references columns in the + // source's own table list (`tbl_d_n.col`), so it must be simplified + // with that source in scope — not the outer query scope active here. + // Take the `from` items out so we can hold an immutable scope onto + // the (still table-list-bearing) source while simplifying the join + // constraint expressions, then put them back. + let from = match source { + stmt::Source::Table(table) if table.from.iter().any(|twj| !twj.joins.is_empty()) => { + Some(std::mem::take(&mut table.from)) + } + _ => None, + }; + + if let Some(mut from) = from { + { + let mut scoped = self.scope(&*source); + for twj in &mut from { + for join in &mut twj.joins { + let (stmt::JoinOp::Inner(expr) | stmt::JoinOp::Left(expr)) = + &mut join.constraint; + scoped.visit_expr_mut(expr); + } + } + } + + if let stmt::Source::Table(table) = source { + table.from = from; + } + return; + } + + stmt::visit_mut::visit_source_mut(self, source); + } + + fn visit_stmt_select_mut(&mut self, stmt: &mut stmt::Select) { + if let stmt::Source::Model(model) = &mut stmt.source + && let Some(via) = model.via.take() + { + todo!("via={via:#?}"); + } + + // Simplify the source first + self.visit_source_mut(&mut stmt.source); + + // Create a new scope for the insert target + let mut s = self.scope(&stmt.source); + + s.visit_filter_mut(&mut stmt.filter); + s.visit_returning_mut(&mut stmt.returning); + } + + fn visit_stmt_update_mut(&mut self, stmt: &mut stmt::Update) { + // `UpdateTarget::Query` is lifted into `UpdateTarget::Model` by the + // pre-lowering `lower::lift_update_query::LiftUpdateQuery` pass, not + // here. + self.visit_update_target_mut(&mut stmt.target); + + let mut s = self.scope(&stmt.target); + s.visit_assignments_mut(&mut stmt.assignments); + + s.visit_filter_mut(&mut stmt.filter); + + if let Some(expr) = &mut stmt.condition.expr { + s.visit_expr_mut(expr); + } + + if let Some(returning) = &mut stmt.returning { + s.visit_returning_mut(returning); + } + } +} + +impl<'a> Simplify<'a> { + pub(crate) fn new(schema: &'a Schema, capability: &'a Capability) -> Self { + Simplify::with_context(stmt::ExprContext::new(schema), capability) + } + + pub(crate) fn with_context(cx: stmt::ExprContext<'a>, capability: &'a Capability) -> Self { + Simplify { + cx, + capability, + positive_filter: false, + } + } + + /// Return a new `Simplify` instance that operates on a nested scope + /// targeting the provided relation. + pub(crate) fn scope<'scope>( + &'scope self, + target: impl IntoExprTarget<'scope>, + ) -> Simplify<'scope> { + Simplify { + cx: self.cx.scope(target), + capability: self.capability, + positive_filter: false, + } + } + + /// Returns the source model + fn flatten_nested_unions(expr_set_op: &mut stmt::ExprSetOp, operands: &mut Vec) { + assert!(expr_set_op.is_union()); + + for expr_set in &mut expr_set_op.operands { + match expr_set { + stmt::ExprSet::SetOp(nested_set_op) if nested_set_op.is_union() => { + Self::flatten_nested_unions(nested_set_op, operands) + } + // Just drop empty values + stmt::ExprSet::Values(values) if values.is_empty() => {} + stmt::ExprSet::Select(select) => { + if let Some(stmt::ExprSet::Select(tail)) = operands.last_mut() { + todo!("merge select={:#?} tail={:#?}", select, tail); + } + + operands.push(std::mem::take(expr_set)); + } + stmt::ExprSet::Values(values) => { + if let Some(stmt::ExprSet::Values(tail)) = operands.last_mut() { + tail.rows.append(&mut values.rows); + continue; + } + + operands.push(std::mem::take(expr_set)); + } + _ => todo!("expr={:#?}", expr_set), + } + } + } +} + +// Equivalence preserves independent evaluations of non-deterministic expressions. +fn dedup_operands(operands: &mut Vec) { + let mut seen: Vec = Vec::new(); + operands.retain(|operand| { + if seen.iter().any(|e| e.is_equivalent_to(operand)) { + false + } else { + seen.push(operand.clone()); + true + } + }); +} + +// Nullable operands do not satisfy the complement law in three-valued logic. +fn has_complement(operands: &[Expr]) -> bool { + operands.iter().any(|operand| { + !matches!(operand, Expr::Not(_)) + && operand.is_always_non_nullable() + && operands + .iter() + .any(|other| matches!(other, Expr::Not(not) if not.expr.is_equivalent_to(operand))) + }) +} + +#[cfg(test)] +mod tests; diff --git a/packages/toasty/src/engine/simplify/expr_and.rs b/packages/toasty/src/engine/simplify/expr_and.rs new file mode 100644 index 00000000000..5b282ddfa24 --- /dev/null +++ b/packages/toasty/src/engine/simplify/expr_and.rs @@ -0,0 +1,234 @@ +use super::{Simplify, dedup_operands, has_complement}; +use std::mem; +use toasty_core::stmt::{self, BinaryOp, Expr}; + +impl Simplify<'_> { + /// Heavyweight AND rewrites. Cheap canonicalization (flatten, drop unit + /// literals, null propagation, single/empty collapse on canonical input) + /// runs in `fold::expr_and` before this is reached. + pub(super) fn simplify_expr_and(&mut self, expr: &mut stmt::ExprAnd) -> Option { + self.merge_in_subqueries(&mut expr.operands); + dedup_operands(&mut expr.operands); + + // Absorption law, `x and (x or y)` → `x` + // If an operand is an OR that contains another operand of the AND, remove the OR. + let non_or_operands: Vec<_> = expr + .operands + .iter() + .filter(|op| !matches!(op, stmt::Expr::Or(_))) + .cloned() + .collect(); + + expr.operands.retain(|operand| { + if let stmt::Expr::Or(or_expr) = operand { + // Remove this OR if any of its operands appears as a direct operand of the AND + !or_expr + .operands + .iter() + .any(|op| non_or_operands.iter().any(|e| e.is_equivalent_to(op))) + } else { + true + } + }); + + // Complement law, `a and not(a)` → `false` (only if `a` is non-nullable) + if has_complement(&expr.operands) { + return Some(false.into()); + } + + // Range to equality: `a >= c and a <= c` → `a = c` + self.try_range_to_equality(expr); + + // Contradicting equality: `a == 1 AND a == 2` → false, + // `a == 1 AND a != 1` → false + if has_self_contradiction(&expr.operands) { + return Some(false.into()); + } + + // OR branch pruning: `AND(x == 1, OR(AND(x != 1, b), ...))` → prune + // branches whose eq/ne constraints contradict the outer AND constraints. + if let Some(result) = prune_or_branches(expr) { + return Some(result); + } + + if expr.operands.is_empty() { + Some(true.into()) + } else if expr.operands.len() == 1 { + Some(expr.operands.remove(0)) + } else { + None + } + } + + /// Finds pairs of range comparisons that collapse to equality. + /// + /// `a >= c and a <= c` → `a = c` + /// + /// When a pair is found, all other bounds on the same (lhs, rhs) are also + /// removed since equality implies them. + /// + /// NOTE: This assumes comparisons are already canonicalized with literals + /// on the right-hand side (e.g., `a >= 5` not `5 <= a`). + fn try_range_to_equality(&mut self, expr: &mut stmt::ExprAnd) { + for i in 0..expr.operands.len() { + let Expr::BinaryOp(op_i) = &expr.operands[i] else { + continue; + }; + + if !matches!(op_i.op, BinaryOp::Ge | BinaryOp::Le) { + continue; + } + + for j in (i + 1)..expr.operands.len() { + let Expr::BinaryOp(op_j) = &expr.operands[j] else { + continue; + }; + + if !matches!( + (op_i.op, op_j.op), + (BinaryOp::Ge, BinaryOp::Le) | (BinaryOp::Le, BinaryOp::Ge) + ) { + continue; + } + + if op_i.lhs.is_equivalent_to(&op_j.lhs) && op_i.rhs.is_equivalent_to(&op_j.rhs) { + let lhs = op_i.lhs.clone(); + let rhs = op_i.rhs.clone(); + + // Replace the first operand with equality + expr.operands[i] = Expr::eq(lhs.as_ref().clone(), rhs.as_ref().clone()); + + // Mark all other `Ge`/`Le` bounds on the same (lhs, rhs) + // for removal + for k in (i + 1)..expr.operands.len() { + if let Expr::BinaryOp(op_k) = &expr.operands[k] + && matches!(op_k.op, BinaryOp::Ge | BinaryOp::Le) + && op_k.lhs.is_equivalent_to(&lhs) + && op_k.rhs.is_equivalent_to(&rhs) + { + expr.operands[k] = true.into(); + } + } + break; + } + } + } + + expr.operands.retain(|e| !e.is_true()); + } +} + +/// Checks for contradicting equality constraints within a single operand +/// list: `a == 1 AND a == 2` → true, `a == 1 AND a != 1` → true. +fn has_self_contradiction(operands: &[Expr]) -> bool { + for i in 0..operands.len() { + if is_contradicting_eq_constraints(&operands[i..=i], &operands[i + 1..]) { + return true; + } + } + false +} + +/// Prunes OR branches whose eq/ne constraints contradict the outer AND's +/// non-OR constraints. Returns `Some(false)` if pruning produces a false +/// operand; otherwise mutates `expr` in place and returns `None`. +fn prune_or_branches(expr: &mut stmt::ExprAnd) -> Option { + // Separate OR operands from non-OR constraints. + let mut or_operands: Vec = Vec::new(); + for op in mem::take(&mut expr.operands) { + if matches!(&op, Expr::Or(_)) { + or_operands.push(op); + } else { + expr.operands.push(op); + } + } + + if or_operands.is_empty() { + return None; + } + + // Prune OR branches that contradict the outer constraints. + for or_op in &mut or_operands { + let Expr::Or(or_expr) = or_op else { + unreachable!() + }; + + or_expr.operands.retain(|branch| { + let branch_ops: &[Expr] = match branch { + Expr::And(and) => &and.operands, + other => std::slice::from_ref(other), + }; + !is_contradicting_eq_constraints(&expr.operands, branch_ops) + }); + + match or_expr.operands.len() { + 0 => *or_op = false.into(), + 1 => *or_op = or_expr.operands.remove(0), + _ => {} + } + } + + // Put OR operands back, flattening surviving AND branches. + for op in or_operands { + match op { + Expr::And(and) => expr.operands.extend(and.operands), + other => expr.operands.push(other), + } + } + + // All OR branches pruned → false. + if expr.operands.iter().any(|e| e.is_false()) { + return Some(false.into()); + } + + // Flattening can reintroduce operands already present in the outer AND. + dedup_operands(&mut expr.operands); + + None +} + +/// Returns `true` if any eq/ne constraint in `a` contradicts any +/// eq/ne constraint in `b`. +fn is_contradicting_eq_constraints(a: &[Expr], b: &[Expr]) -> bool { + for outer_op in a { + let Some((o_lhs, o_op, o_val)) = extract_eq_ne(outer_op) else { + continue; + }; + + for branch_op in b { + let Some((b_lhs, b_op, b_val)) = extract_eq_ne(branch_op) else { + continue; + }; + + // Only consider the two lhs sides "the same value" when they are + // syntactically equal AND stable. Otherwise `f() == 1 AND f() == 2` + // would be rewritten to `false`, but two evaluations of a + // non-deterministic `f()` can produce 1 and 2. + if !o_lhs.is_equivalent_to(b_lhs) { + continue; + } + + match (o_op, b_op) { + (BinaryOp::Eq, BinaryOp::Eq) if o_val != b_val => return true, + (BinaryOp::Eq, BinaryOp::Ne) | (BinaryOp::Ne, BinaryOp::Eq) if o_val == b_val => { + return true; + } + _ => {} + } + } + } + + false +} + +/// Extracts `(lhs, op, rhs_value)` from an `Expr::BinaryOp` if it is an +/// `==` or `!=` with a constant value on the right. +fn extract_eq_ne(expr: &Expr) -> Option<(&Expr, BinaryOp, &stmt::Value)> { + if let Expr::BinaryOp(binop) = expr + && let Expr::Value(val) = binop.rhs.as_ref() + && matches!(binop.op, BinaryOp::Eq | BinaryOp::Ne) + { + return Some((binop.lhs.as_ref(), binop.op, val)); + } + None +} diff --git a/packages/toasty/src/engine/simplify/expr_any.rs b/packages/toasty/src/engine/simplify/expr_any.rs new file mode 100644 index 00000000000..039994643a5 --- /dev/null +++ b/packages/toasty/src/engine/simplify/expr_any.rs @@ -0,0 +1,37 @@ +use super::Simplify; +use toasty_core::stmt::{self, Expr}; + +impl Simplify<'_> { + pub(super) fn simplify_expr_any(&self, expr_any: &stmt::ExprAny) -> Option { + // Only simplify when the inner expression is a Map with a constant base + let stmt::Expr::Map(expr_map) = &*expr_any.expr else { + return None; + }; + + if !expr_map.base.is_const() { + return None; + } + + let Ok(base) = expr_map.base.eval_const() else { + return None; + }; + let stmt::Value::List(items) = base else { + todo!() + }; + + let mut operands = vec![]; + + for item in items { + let mut operand = (*expr_map.map).clone(); + + match item { + stmt::Value::Record(value_record) => operand.substitute(&value_record.fields[..]), + item => operand.substitute([item]), + } + + operands.push(operand); + } + + Some(Expr::or_from_vec(operands)) + } +} diff --git a/packages/toasty/src/engine/simplify/expr_binary_op.rs b/packages/toasty/src/engine/simplify/expr_binary_op.rs new file mode 100644 index 00000000000..00412cc4274 --- /dev/null +++ b/packages/toasty/src/engine/simplify/expr_binary_op.rs @@ -0,0 +1,418 @@ +use super::Simplify; +use toasty_core::stmt::{self, Expr, ResolvedRef, VisitMut}; + +impl Simplify<'_> { + /// Heavyweight binary-op rewrites. Cheap canonicalization (constant + /// folding, null propagation, boolean-constant simplification, + /// literal-on-right swap) runs in `fold::expr_binary_op` before this + /// is reached, so heavyweight rules see operands in canonical form + /// (no `(Value, Value)`, no `(Value, _)` ahead of `(_, Value)`). + /// + /// App-level rewrites on eq/ne operands (`Reference::Model` → + /// primary-key field, `BelongsTo` → foreign-key field) fire in the + /// pre-lowering `lower::expr_eq_operand::RewriteEqOperand` pass, not + /// here. + pub(super) fn simplify_expr_binary_op( + &mut self, + op: stmt::BinaryOp, + lhs: &mut stmt::Expr, + rhs: &mut stmt::Expr, + ) -> Option { + if (op.is_eq() || op.is_ne()) + && let (Some(lhs_len), Some(rhs_len)) = (lhs.record_len(), rhs.record_len()) + && lhs_len != rhs_len + { + return Some(op.is_ne().into()); + } + + let result = match (&mut *lhs, &mut *rhs) { + // Self-comparison, e.g., + // + // - `x = x` → `true` + // - `x != x` → `false` + // + // Only applied for non-nullable field references. + (Expr::Reference(lhs), Expr::Reference(rhs)) + if lhs == rhs && (op.is_eq() || op.is_ne()) => + { + if lhs.is_field() { + let field = self.cx.resolve_expr_reference(lhs).as_field_unwrap(); + if !field.nullable() { + return Some(op.is_eq().into()); + } + } + None + } + // Tuple decomposition, + // + // - `(a, b) = (x, y)` → `a = x and b = y` + // - `(a, b) != (x, y)` → `a != x or b != y` + (Expr::Record(lhs_rec), Expr::Record(rhs_rec)) + if (op.is_eq() || op.is_ne()) && lhs_rec.len() == rhs_rec.len() => + { + let comparisons: Vec<_> = std::mem::take(&mut lhs_rec.fields) + .into_iter() + .zip(std::mem::take(&mut rhs_rec.fields)) + .map(|(l, r)| record_field_comparison(l, op, r)) + .collect(); + + let mut comparison = if op.is_eq() { + Expr::and_from_vec(comparisons) + } else { + Expr::or_from_vec(comparisons) + }; + self.visit_expr_mut(&mut comparison); + Some(comparison) + } + // Tuple decomposition with a Value::Record on one side, + // + // - `(a, b) = Value::Record([x, y])` → `a = x and b = y` + // + // This arises after match elimination produces `Record([col1, col2]) == Value::Record([1, "alice"])`. + (Expr::Record(rec), Expr::Value(stmt::Value::Record(val_rec))) + | (Expr::Value(stmt::Value::Record(val_rec)), Expr::Record(rec)) + if (op.is_eq() || op.is_ne()) && rec.len() == val_rec.len() => + { + let comparisons: Vec<_> = std::mem::take(&mut rec.fields) + .into_iter() + .zip(std::mem::take(&mut val_rec.fields)) + .map(|(expr, val)| record_field_comparison(expr, op, Expr::from(val))) + .collect(); + + let mut comparison = if op.is_eq() { + Expr::and_from_vec(comparisons) + } else { + Expr::or_from_vec(comparisons) + }; + self.visit_expr_mut(&mut comparison); + Some(comparison) + } + // Match elimination: distribute binary op into match arms as OR + // + // Match(subj, [p1 => e1, p2 => e2]) rhs + // → OR(subj == p1 AND e1 rhs, subj == p2 AND e2 rhs) + // + // Each arm is fully simplified inline. Arms that fold to false/null + // are pruned. Comparison ops only — distributing arithmetic this + // way would produce a malformed boolean from a non-boolean term. + (Expr::Match(m), _) if !op.is_arithmetic() && m.subject.is_stable() => { + let match_expr = lhs.take(); + let other = rhs.take(); + Some(self.eliminate_match_in_binary_op(op, match_expr, other, true)) + } + (_, Expr::Match(m)) if !op.is_arithmetic() && m.subject.is_stable() => { + let other = lhs.take(); + let match_expr = rhs.take(); + Some(self.eliminate_match_in_binary_op(op, match_expr, other, false)) + } + // Decode-cast stripping on comparisons with a constant. A stored + // column whose type differs from the model type decodes through + // `cast(col, )` (see `map_table_column_to_model`). When + // such a cast surfaces in a comparison — e.g. out of an enum + // decode `Match` arm, which only unfolds via match elimination + // after lowering has run — move the conversion onto the constant + // side so the driver compares the stored form directly. Other + // cast shapes are handled during lowering + // (`lower_expr_binary_op`); this rule fires only on the + // post-lower column shape. + (Expr::Cast(cast), Expr::Value(value)) | (Expr::Value(value), Expr::Cast(cast)) + if (op.is_eq() || op.is_ne()) && cast.from.is_none() && cast.expr.is_column() => + { + self.strip_decode_cast_comparison(op, cast, value) + } + // Decode-cast stripping on column-versus-column comparisons, such + // as one embedded relation compared to another after both sides + // substitute to key columns. The eligibility check proves that + // comparing the stored forms preserves decoded equality. + (Expr::Cast(lhs_cast), Expr::Cast(rhs_cast)) + if self.can_strip_decode_cast_column_comparison(op, lhs_cast, rhs_cast) => + { + self.strip_decode_cast_column_comparison(op, lhs_cast, rhs_cast) + } + // Self-comparison with projections, e.g., + // + // - `address.city = address.city` → `true` + // - `address.city != address.city` → `false` + // + // By this point, constant projections and record projections have been simplified. + // What remains are projections with opaque bases (e.g., field references). + // `lhs.base.is_stable()` keeps this sound: a projection through a + // non-deterministic base would evaluate the base twice and could + // yield different values each time. + (Expr::Project(lhs), Expr::Project(rhs)) + if lhs == rhs && lhs.base.is_stable() && (op.is_eq() || op.is_ne()) => + { + // TODO: Check if the projected value is nullable + Some(Expr::from(op.is_eq())) + } + _ => None, + }; + + if result.is_some() { + return result; + } + + // Null propagation for derived VALUES columns. + // + // If either operand is a column reference into a derived VALUES + // table where every row has NULL at that column position, the + // binary op can never produce a non-null result. + if self.is_always_null_derived_column(lhs) || self.is_always_null_derived_column(rhs) { + return Some(Expr::null()); + } + + // Relation-path-comparison and IN-subquery lifting fire in the + // pre-lowering `lower::lift_in_subquery::*` pass, not here. + None + } + + /// Rewrites `cast(col, T) const` so the conversion happens on + /// the constant instead of the column: converts `const` to `col`'s + /// stored type once, here, and emits `col const'` so the driver + /// compares the bare column against the converted constant rather than + /// casting the column on every row. + /// + /// The target type comes from the referenced column, not from + /// `Capability::native_type_for(T)`: a `#[column(type = ...)]` override + /// can store the value in a different type than the backend's default + /// (e.g. a text-stored UUID on SQLite, whose default UUID storage is a + /// blob). Bails on anything that does not resolve to a physical column. + fn strip_decode_cast_comparison( + &mut self, + op: stmt::BinaryOp, + cast: &mut stmt::ExprCast, + value: &mut stmt::Value, + ) -> Option { + let expr_reference = cast.expr.as_expr_reference()?; + + let ResolvedRef::Column(column) = self.cx.resolve_expr_reference(expr_reference) else { + return None; + }; + + let value = column + .ty + .cast(self.cx.schema(), value.take()) + .expect("failed to cast value"); + Some(Expr::binary_op(cast.expr.take(), op, value)) + } + + /// Returns whether `cast(col_a, T) cast(col_b, T)` can compare the + /// stored columns directly without changing the result. + fn can_strip_decode_cast_column_comparison( + &self, + op: stmt::BinaryOp, + lhs: &stmt::ExprCast, + rhs: &stmt::ExprCast, + ) -> bool { + if !(op.is_eq() || op.is_ne()) + || lhs.from.is_some() + || rhs.from.is_some() + || !lhs.expr.is_column() + || !rhs.expr.is_column() + || lhs.ty != rhs.ty + { + return false; + } + + let Some(lhs_reference) = lhs.expr.as_expr_reference() else { + return false; + }; + let Some(rhs_reference) = rhs.expr.as_expr_reference() else { + return false; + }; + + let ResolvedRef::Column(lhs_column) = self.cx.resolve_expr_reference(lhs_reference) else { + return false; + }; + let ResolvedRef::Column(rhs_column) = self.cx.resolve_expr_reference(rhs_reference) else { + return false; + }; + + lhs_column.ty == rhs_column.ty && lhs_column.ty.cast_preserves_equality(&lhs.ty) + } + + /// Rewrites `cast(col_a, T) cast(col_b, T)` to + /// `col_a col_b` after + /// [`Self::can_strip_decode_cast_column_comparison`] proves the rewrite is + /// valid. + fn strip_decode_cast_column_comparison( + &mut self, + op: stmt::BinaryOp, + lhs: &mut stmt::ExprCast, + rhs: &mut stmt::ExprCast, + ) -> Option { + Some(Expr::binary_op(lhs.expr.take(), op, rhs.expr.take())) + } + + /// Returns `true` if `expr` is a column reference that resolves to a + /// derived VALUES table where every row has NULL at the referenced column. + fn is_always_null_derived_column(&self, expr: &Expr) -> bool { + let Expr::Reference(expr_ref) = expr else { + return false; + }; + + match self.cx.resolve_expr_reference(expr_ref) { + ResolvedRef::Derived(derived_ref) => derived_ref.is_column_always_null(), + _ => false, + } + } + + /// Distributes a binary op over match arms, producing an OR of guarded + /// comparisons. Each arm becomes `(subject == pattern) AND (arm_expr other)`. + /// Dead branches (false/null) are pruned after inline simplification. + fn eliminate_match_in_binary_op( + &mut self, + op: stmt::BinaryOp, + match_expr: Expr, + other: Expr, + match_on_lhs: bool, + ) -> Expr { + self.eliminate_match(match_expr, |arm| { + if match_on_lhs { + Expr::binary_op(arm, op, other.clone()) + } else { + Expr::binary_op(other.clone(), op, arm) + } + }) + } + + /// Distributes a predicate over match arms, producing an OR of guarded + /// terms. `term` builds the predicate for one arm from that arm's + /// expression; each arm becomes `(subject == pattern) AND term(arm_expr)`, + /// and the else branch is guarded by the negation of every pattern. Dead + /// branches (false/null/error) are pruned after inline simplification. + pub(super) fn eliminate_match( + &mut self, + match_expr: Expr, + term: impl Fn(Expr) -> Expr, + ) -> Expr { + let Expr::Match(match_expr) = match_expr else { + unreachable!() + }; + + let mut operands = Vec::new(); + + // Collect arm patterns before consuming the arms (needed for the else guard). + let patterns: Vec<_> = match_expr.arms.iter().map(|a| a.pattern.clone()).collect(); + + for arm in match_expr.arms { + // Unit enum variants return the discriminant subject. The arm's + // pattern is the value of that subject within this branch. + let arm_expr = if arm.expr == *match_expr.subject { + Expr::from(arm.pattern.clone()) + } else { + arm.expr + }; + let guard = Expr::binary_op( + (*match_expr.subject).clone(), + stmt::BinaryOp::Eq, + Expr::from(arm.pattern), + ); + + let mut term = Expr::and_from_vec(vec![guard, term(arm_expr)]); + self.visit_expr_mut(&mut term); + + // Prune dead branches + if is_dead_filter_term(&term) { + continue; + } + + operands.push(term); + } + + // Include the else branch with a guard that negates all arm patterns. + { + let guards: Vec = patterns + .into_iter() + .map(|pattern| { + Expr::not(Expr::binary_op( + (*match_expr.subject).clone(), + stmt::BinaryOp::Eq, + Expr::from(pattern), + )) + }) + .collect(); + + let mut else_operands = guards; + else_operands.push(term(*match_expr.else_expr)); + let mut term = Expr::and_from_vec(else_operands); + self.visit_expr_mut(&mut term); + + // Prune dead branches + if !is_dead_filter_term(&term) { + operands.push(term); + } + } + + Expr::or_from_vec(operands) + } +} + +/// Returns `true` when a distributed-comparison term can never be `true`, so it +/// adds nothing to the surrounding `OR` and can be dropped. +/// +/// Three shapes qualify: +/// +/// 1. The term is `false` or a bare `NULL`. +/// +/// 2. The term is an `AND` with a `NULL` conjunct, such as `x AND NULL`. This +/// comes from comparing against a decode `Match`'s `null` else branch — for +/// example `eq(option_embed, Some(..))` on a `None` row. `x AND NULL` never +/// holds in three-valued logic, and DynamoDB rejects the bare `NULL` +/// placeholder it would otherwise serialize to. +/// +/// 3. The term compares against an `Expr::Error`, the placeholder a total enum +/// decode puts in its unreachable else branch. For a two-variant enum, +/// distributing `field == "x"` over the decode produces a term like +/// `disc != 1 AND disc != 2 AND Error == "x"`. A surrounding variant gate +/// usually contradicts the `disc != k` guards and folds this away, but +/// factoring a shared predicate out from under its gates removes that gate +/// (issue #1061). Since `Error` marks a branch that never runs, the +/// comparison can never hold. Dropping the term also keeps `Error` out of +/// the SQL serializer, which cannot render it. +fn is_dead_filter_term(term: &Expr) -> bool { + if term.is_unsatisfiable() { + return true; + } + if contains_error(term) { + return true; + } + matches!( + term, + Expr::And(and) if and.operands.iter().any(Expr::is_value_null) + ) +} + +/// Returns `true` if `expr` contains an `Expr::Error` node anywhere in its +/// subtree — the unreachable-branch placeholder produced by enum decode. +fn contains_error(expr: &Expr) -> bool { + use toasty_core::stmt::Visit; + + struct FindError(bool); + + impl Visit for FindError { + fn visit_expr_error(&mut self, _: &stmt::ExprError) { + self.0 = true; + } + } + + let mut find = FindError(false); + find.visit_expr(expr); + find.0 +} + +/// Uses null checks for record fields, including empty relation slots. +fn record_field_comparison(lhs: Expr, op: stmt::BinaryOp, rhs: Expr) -> Expr { + let other = if lhs.is_value_null() { + rhs + } else if rhs.is_value_null() { + lhs + } else { + return Expr::binary_op(lhs, op, rhs); + }; + if op.is_eq() { + Expr::is_null(other) + } else { + Expr::is_not_null(other) + } +} diff --git a/packages/toasty/src/engine/simplify/expr_cast.rs b/packages/toasty/src/engine/simplify/expr_cast.rs new file mode 100644 index 00000000000..5f235eb53d3 --- /dev/null +++ b/packages/toasty/src/engine/simplify/expr_cast.rs @@ -0,0 +1,61 @@ +use super::Simplify; +use toasty_core::schema::app::FieldTy; +use toasty_core::stmt; + +impl Simplify<'_> { + /// Heavyweight `CAST` rewrites. Cheap canonicalization (constant + /// folding) runs in `fold::expr_cast` before this is reached. + pub(super) fn simplify_expr_cast(&self, expr: &mut stmt::ExprCast) -> Option { + // Typed model records retain relation slots until relation planning + // in lowering has assigned their foreign keys. + if expr.from.is_none() + && let stmt::Type::Model(model) = expr.ty + && expr.expr.record_len().is_some() + { + let model = self.cx.schema().app.model(model); + if model.is_root() || model.fields().iter().any(|field| field.ty.is_belongs_to()) { + return None; + } + } + + // Schema-directed constant folding: a `#[document]` cast (marked by a + // source type or a model-typed target) resolves the embed's field + // names through the schema, so the schema-free fold pass skips it and + // it folds here instead. The operand may be a constant expression + // *tree* (an insert row's document value is an `Expr::Record` of + // per-leaf values), so it is folded to a value first. + if expr.from.is_some() || expr.ty.contains_model() { + if !expr.expr.is_const() { + return None; + } + let value = expr + .expr + .eval_const() + .expect("constant expression failed to evaluate"); + let value = expr + .ty + .cast_from(self.cx.schema(), expr.from.as_ref(), value) + .expect("failed to cast value"); + return Some(value.into()); + } + + // Redundant cast elimination: `cast(x as T) → x` when x is already T. + // + // If the inner expression is a field reference whose primitive type + // matches the cast target, the cast is a no-op and can be removed. + let stmt::Expr::Reference(f @ stmt::ExprReference::Field { .. }) = &*expr.expr else { + return None; + }; + + let field = self.cx.resolve_expr_reference(f).as_field_unwrap(); + let FieldTy::Primitive(primitive) = &field.ty else { + return None; + }; + + if primitive.ty == expr.ty { + return Some(expr.expr.take()); + } + + None + } +} diff --git a/packages/toasty/src/engine/simplify/expr_exists.rs b/packages/toasty/src/engine/simplify/expr_exists.rs new file mode 100644 index 00000000000..bda96e3b973 --- /dev/null +++ b/packages/toasty/src/engine/simplify/expr_exists.rs @@ -0,0 +1,17 @@ +use toasty_core::stmt; + +use crate::engine::simplify::Simplify; + +impl Simplify<'_> { + pub(super) fn simplify_expr_exists( + &self, + expr_exists: &stmt::ExprExists, + ) -> Option { + // `exists(empty_query)` → `false` + if self.stmt_query_is_empty(&expr_exists.subquery) { + return Some(stmt::Expr::FALSE); + } + + None + } +} diff --git a/packages/toasty/src/engine/simplify/expr_in_list.rs b/packages/toasty/src/engine/simplify/expr_in_list.rs new file mode 100644 index 00000000000..7265003fdd3 --- /dev/null +++ b/packages/toasty/src/engine/simplify/expr_in_list.rs @@ -0,0 +1,65 @@ +use super::Simplify; +use toasty_core::stmt::{self, Expr, ResolvedRef}; + +impl Simplify<'_> { + /// Post-lower rewrites of ` IN (list)` whose subject is a column + /// decode: the shapes an embedded relation resolves to once its key + /// projection lowers to storage. + pub(super) fn simplify_expr_in_list(&mut self, expr: &mut stmt::ExprInList) -> Option { + match &mut *expr.expr { + // An enum decode `Match` as the subject: distribute membership + // over the arms, as `simplify_expr_binary_op` does for + // comparisons. + Expr::Match(m) if m.subject.is_stable() => { + let match_expr = expr.expr.take(); + let list = expr.list.take(); + Some(self.eliminate_match(match_expr, |arm| Expr::in_list(arm, list.clone()))) + } + // Decode-cast stripping: `cast(col, T) IN (values)` compares the + // stored form directly, with the values converted to the column + // type — the list-membership form of the cast-versus-value rule. + Expr::Cast(cast) if cast.from.is_none() && cast.expr.is_column() => { + self.strip_decode_cast_in_list(cast, &mut expr.list) + } + _ => None, + } + } + + fn strip_decode_cast_in_list( + &mut self, + cast: &mut stmt::ExprCast, + list: &mut Expr, + ) -> Option { + let expr_reference = cast.expr.as_expr_reference()?; + + let ResolvedRef::Column(column) = self.cx.resolve_expr_reference(expr_reference) else { + return None; + }; + + let schema = self.cx.schema(); + match list { + Expr::Value(stmt::Value::List(values)) => { + for value in values { + *value = column + .ty + .cast(schema, value.take()) + .expect("failed to cast value"); + } + } + Expr::List(expr_list) => { + for item in &mut expr_list.items { + let Expr::Value(value) = item else { + return None; + }; + *value = column + .ty + .cast(schema, value.take()) + .expect("failed to cast value"); + } + } + _ => return None, + } + + Some(Expr::in_list(cast.expr.take(), list.take())) + } +} diff --git a/packages/toasty/src/engine/simplify/expr_intersects.rs b/packages/toasty/src/engine/simplify/expr_intersects.rs new file mode 100644 index 00000000000..15a1c9caec5 --- /dev/null +++ b/packages/toasty/src/engine/simplify/expr_intersects.rs @@ -0,0 +1,32 @@ +use super::Simplify; +use toasty_core::stmt; + +impl Simplify<'_> { + /// Rewrite `Intersects(path, [v1, v2, …])` into `OR(any_op(v1, Eq, path), …)` + /// for drivers lacking `native_array_set_predicates`. The bottom-up fold + /// pass has already collapsed a literal rhs into `Value::List` by the time + /// this runs; a non-list rhs cannot be expanded and is left for the + /// capability check to reject. + pub(super) fn simplify_expr_intersects( + &self, + expr: &mut stmt::ExprIntersects, + ) -> Option { + if self.capability.native_array_set_predicates { + return None; + } + + let stmt::Expr::Value(stmt::Value::List(values)) = expr.rhs.as_ref() else { + return None; + }; + + // Empty-rhs has been folded to `false` upstream by `fold::expr_intersects`. + debug_assert!(!values.is_empty()); + + let path = (*expr.lhs).clone(); + let operands: Vec = values + .iter() + .map(|v| stmt::Expr::any_op(v.clone(), stmt::BinaryOp::Eq, path.clone())) + .collect(); + Some(stmt::Expr::or_from_vec(operands)) + } +} diff --git a/packages/toasty/src/engine/simplify/expr_is_null.rs b/packages/toasty/src/engine/simplify/expr_is_null.rs new file mode 100644 index 00000000000..8e4197b8b33 --- /dev/null +++ b/packages/toasty/src/engine/simplify/expr_is_null.rs @@ -0,0 +1,23 @@ +use super::Simplify; +use toasty_core::stmt; + +impl Simplify<'_> { + /// Heavyweight `IS NULL` rewrites. Cheap canonicalization (constant + /// folding, cast stripping) runs in `fold::expr_is_null` before this is + /// reached. + pub(super) fn simplify_expr_is_null(&self, expr: &mut stmt::ExprIsNull) -> Option { + let stmt::Expr::Reference(f @ stmt::ExprReference::Field { .. }) = &mut *expr.expr else { + return None; + }; + + let field = self.cx.resolve_expr_reference(f).as_field_unwrap(); + + if !field.nullable() { + // `is_null` on a non-nullable field evaluates to `false`, and + // `is_not_null` to `true`. + return Some(stmt::Expr::Value(stmt::Value::Bool(expr.negated))); + } + + None + } +} diff --git a/packages/toasty/src/engine/simplify/expr_is_superset.rs b/packages/toasty/src/engine/simplify/expr_is_superset.rs new file mode 100644 index 00000000000..9b6d238f61f --- /dev/null +++ b/packages/toasty/src/engine/simplify/expr_is_superset.rs @@ -0,0 +1,32 @@ +use super::Simplify; +use toasty_core::stmt; + +impl Simplify<'_> { + /// Rewrite `IsSuperset(path, [v1, v2, …])` into `AND(any_op(v1, Eq, path), …)` + /// for drivers lacking `native_array_set_predicates`. The bottom-up fold + /// pass has already collapsed a literal rhs into `Value::List` by the time + /// this runs; a non-list rhs cannot be expanded and is left for the + /// capability check to reject. + pub(super) fn simplify_expr_is_superset( + &self, + expr: &mut stmt::ExprIsSuperset, + ) -> Option { + if self.capability.native_array_set_predicates { + return None; + } + + let stmt::Expr::Value(stmt::Value::List(values)) = expr.rhs.as_ref() else { + return None; + }; + + // Empty-rhs has been folded to `true` upstream by `fold::expr_is_superset`. + debug_assert!(!values.is_empty()); + + let path = (*expr.lhs).clone(); + let operands: Vec = values + .iter() + .map(|v| stmt::Expr::any_op(v.clone(), stmt::BinaryOp::Eq, path.clone())) + .collect(); + Some(stmt::Expr::and_from_vec(operands)) + } +} diff --git a/packages/toasty/src/engine/simplify/expr_list.rs b/packages/toasty/src/engine/simplify/expr_list.rs new file mode 100644 index 00000000000..fac45446765 --- /dev/null +++ b/packages/toasty/src/engine/simplify/expr_list.rs @@ -0,0 +1,93 @@ +use super::Simplify; +use toasty_core::stmt; + +impl Simplify<'_> { + /// Heavyweight `List` rewrites. Cheap canonicalization (collapsing a + /// list of literal values into `Value::List`) runs in + /// `fold::expr_list` before this is reached. + pub(super) fn simplify_expr_list(&mut self, expr: &mut stmt::ExprList) -> Option { + self.simplify_expr_list_insert_stmt(expr) + } + + // TODO: rewrite this + fn simplify_expr_list_insert_stmt(&mut self, expr: &mut stmt::ExprList) -> Option { + // Check if all items are Expr::Stmt with single-row Insert statements + for item in &expr.items { + let stmt::Expr::Stmt(expr_stmt) = item else { + return None; + }; + + let insert = expr_stmt.stmt.as_insert()?; + + // Must be single-row insert + if !insert.source.single { + return None; + } + + // Must have Returning::Model + match &insert.returning { + Some(stmt::Returning::Model { .. }) => {} + _ => return None, + } + + // Must target a Model (not Table or Scope) + if !insert.target.is_model() { + return None; + } + } + + // Extract the first insert to get the target model and returning clause + let first_insert = match &expr.items[0] { + stmt::Expr::Stmt(s) => s.stmt.as_insert().unwrap(), + _ => unreachable!(), + }; + + let first_target_model = match &first_insert.target { + stmt::InsertTarget::Model(model_id) => model_id, + _ => unreachable!(), + }; + + let first_returning = first_insert.returning.as_ref().unwrap(); + + // Check all inserts target the same model and have the same returning clause + for item in &expr.items[1..] { + let insert = match item { + stmt::Expr::Stmt(s) => s.stmt.as_insert().unwrap(), + _ => unreachable!(), + }; + + let target_model = match &insert.target { + stmt::InsertTarget::Model(model_id) => model_id, + _ => unreachable!(), + }; + + if target_model != first_target_model { + return None; + } + + if insert.returning.as_ref().unwrap() != first_returning { + return None; + } + } + + // All inserts are compatible, merge them into a single batch insert + let mut items = std::mem::take(&mut expr.items); + let mut merged_insert = match items.remove(0) { + stmt::Expr::Stmt(s) => s.stmt.into_insert_unwrap(), + _ => unreachable!(), + }; + + for item in items { + let insert = match item { + stmt::Expr::Stmt(s) => s.stmt.into_insert_unwrap(), + _ => unreachable!(), + }; + merged_insert.merge(insert); + } + + // Set single = false since we're now returning a list of records + merged_insert.source.single = false; + + Some(stmt::Expr::stmt(stmt::Statement::Insert(merged_insert))) + } +} diff --git a/packages/toasty/src/engine/simplify/expr_map.rs b/packages/toasty/src/engine/simplify/expr_map.rs new file mode 100644 index 00000000000..c895c0d9456 --- /dev/null +++ b/packages/toasty/src/engine/simplify/expr_map.rs @@ -0,0 +1,16 @@ +use super::Simplify; +use crate::engine::eval::Func; +use toasty_core::stmt; + +impl Simplify<'_> { + pub(super) fn simplify_expr_map(&self, expr: &mut stmt::Expr) -> Option { + if expr.as_map_unwrap().base.is_value() { + let eval = Func::try_from_stmt(expr, vec![])?; + + let ret = eval.eval_const(self.cx.schema()); + Some(stmt::Expr::Value(ret)) + } else { + None + } + } +} diff --git a/packages/toasty/src/engine/simplify/expr_or.rs b/packages/toasty/src/engine/simplify/expr_or.rs new file mode 100644 index 00000000000..001f6f4a072 --- /dev/null +++ b/packages/toasty/src/engine/simplify/expr_or.rs @@ -0,0 +1,180 @@ +use super::{Simplify, dedup_operands, has_complement}; +use std::mem; +use toasty_core::stmt; + +impl Simplify<'_> { + /// Heavyweight OR rewrites. Cheap canonicalization (flatten, true + /// short-circuit, drop false, null propagation) runs in `fold::expr_or` + /// before this is reached. + pub(super) fn simplify_expr_or(&mut self, expr: &mut stmt::ExprOr) -> Option { + dedup_operands(&mut expr.operands); + + // Absorption law, `x or (x and y)` → `x` + // If an operand is an AND that contains another operand of the OR, remove the AND. + let non_and_operands: Vec<_> = expr + .operands + .iter() + .filter(|op| !matches!(op, stmt::Expr::And(_))) + .cloned() + .collect(); + + expr.operands.retain(|operand| { + if let stmt::Expr::And(and_expr) = operand { + // Remove this AND if any of its operands appears as a direct operand of the OR + !and_expr + .operands + .iter() + .any(|op| non_and_operands.iter().any(|e| e.is_equivalent_to(op))) + } else { + true + } + }); + + // Factoring, `(a and b) or (a and c)` → `a and (b or c)` + // Find common factors across all AND operands and factor them out. + if let Some(factored) = self.try_factor_or(expr) { + return Some(factored); + } + + // Complement law, `a or not(a)` → `true` (only if `a` is non-nullable) + if has_complement(&expr.operands) { + return Some(true.into()); + } + + // The variant-tautology rewrite (`is_variant(x, 0) or is_variant(x, 1)` + // covering all variants → `true`) fires in the pre-lowering + // `LowerStatement::visit_expr_mut` `Expr::Or` arm, not here. + + // OR-to-IN conversion, `a = 1 or a = 2 or a = 3` → `a in (1, 2, 3)` + if let Some(in_list) = self.try_or_to_in_list(expr) { + return Some(in_list); + } + + if expr.operands.is_empty() { + Some(false.into()) + } else if expr.operands.len() == 1 { + Some(expr.operands.remove(0)) + } else { + None + } + } + + /// Attempts to factor common terms from AND expressions within an OR. + /// `(a and b) or (a and c)` → `a and (b or c)` + /// `(a and b and c) or (a and b and d)` → `a and b and (c or d)` + fn try_factor_or(&self, expr: &mut stmt::ExprOr) -> Option { + // Need at least 2 operands, all must be ANDs + if expr.operands.len() < 2 { + return None; + } + + if !expr + .operands + .iter() + .all(|op| matches!(op, stmt::Expr::And(_))) + { + return None; + } + + // Find all common factors by checking which operands from the first AND + // appear in all other ANDs + let first_and = match &expr.operands[0] { + stmt::Expr::And(and) => and, + _ => unreachable!(), + }; + + let common: Vec<_> = first_and + .operands + .iter() + .filter(|op| { + expr.operands[1..].iter().all(|other| { + if let stmt::Expr::And(other_and) = other { + other_and.operands.iter().any(|e| e.is_equivalent_to(op)) + } else { + false + } + }) + }) + .cloned() + .collect(); + + if common.is_empty() { + return None; + } + + // Remove all common factors from each AND + for operand in &mut expr.operands { + if let stmt::Expr::And(and) = operand { + and.operands + .retain(|op| !common.iter().any(|c| c.is_equivalent_to(op))); + // If only one operand left, unwrap the AND + if and.operands.len() == 1 { + *operand = and.operands.pop().unwrap(); + } else if and.operands.is_empty() { + *operand = true.into(); + } + } + } + + // Common factors AND (the modified OR) + let mut result = common; + let or_expr = stmt::ExprOr { + operands: mem::take(&mut expr.operands), + }; + result.push(stmt::Expr::Or(or_expr)); + Some(stmt::Expr::and_from_vec(result)) + } + + /// Converts disjunctive equality chains to IN lists. + /// + /// `a = 1 or a = 2 or b = 3` → `a in (1, 2) or b = 3` + /// `a = 1 or a = 2 or b = 3 or b = 4` → `a in (1, 2) or b in (3, 4)` + /// + /// Groups equality comparisons by their LHS and converts groups with 2+ + /// values into IN lists. Non-equality operands are preserved. + fn try_or_to_in_list(&self, expr: &mut stmt::ExprOr) -> Option { + let mut groups: Vec<(stmt::Expr, Vec)> = Vec::new(); + let mut other_operands: Vec = Vec::new(); + + for operand in mem::take(&mut expr.operands) { + if let stmt::Expr::BinaryOp(bin_op) = &operand + && bin_op.op.is_eq() + && let stmt::Expr::Value(value) = bin_op.rhs.as_ref() + { + // Keep non-deterministic expressions in separate groups so an + // IN list cannot collapse independent evaluations into one. + if let Some((_, values)) = groups + .iter_mut() + .find(|(lhs, _)| lhs.is_equivalent_to(&bin_op.lhs)) + { + values.push(value.clone()); + } else { + groups.push(((*bin_op.lhs).clone(), vec![value.clone()])); + } + continue; + } + + // Non-equality or non-constant RHS - keep as is + other_operands.push(operand); + } + + let has_conversion = groups.iter().any(|(_, values)| values.len() >= 2); + let mut result_operands = other_operands; + + for (lhs, mut values) in groups { + let operand = if values.len() >= 2 { + stmt::Expr::in_list(lhs, stmt::Expr::list(values)) + } else { + stmt::Expr::eq(lhs, values.pop().unwrap()) + }; + result_operands.push(operand); + } + + if has_conversion && result_operands.len() == 1 { + Some(result_operands.remove(0)) + } else { + expr.operands = result_operands; + None + } + } +} diff --git a/packages/toasty/src/engine/simplify/expr_project.rs b/packages/toasty/src/engine/simplify/expr_project.rs new file mode 100644 index 00000000000..3d48e1bee6f --- /dev/null +++ b/packages/toasty/src/engine/simplify/expr_project.rs @@ -0,0 +1,63 @@ +use toasty_core::stmt::{self, Project}; + +use super::Simplify; + +impl Simplify<'_> { + pub(super) fn simplify_expr_project( + &mut self, + expr: &mut stmt::ExprProject, + ) -> Option { + // Constant evaluation: if the base is an Expr::Value, we can evaluate + // the projection at compile time. + // + // Examples: + // - `project(Value::I64(42), [])` → `Value::I64(42)` + // - `project(Value::Record([1, 2, 3]), [0])` → `Value::I64(1)` + // + // Note: This only handles Expr::Value, not Expr::Record or Expr::List. + // Those variants represent expressions that will be evaluated later, + // not constant values that can be folded now. + if let stmt::Expr::Value(value) = &*expr.base { + // Use the value's project method to follow the projection path. + // Projecting into a `Null` base (e.g. the `Some`-arm body of an + // `Option` whose value is `None`) yields `Null` — see + // `Value::entry`. + if let Some(result) = value.project(&expr.projection) { + return Some(result); + } + } + + // Handle projections through records (embedded fields lower to records of columns). + // After lowering, embedded field references become records where each field is a column. + // Uses `entry` to support arbitrary-depth projections (e.g., [1, 1] for nested embedded). + // Examples: + // project([street_col, city_col, zip_col], [1]) → city_col + // project([name_col, record([street_col, city_col])], [1, 1]) → city_col + if let stmt::Expr::Record(_) = &*expr.base + && let Some(entry) = expr.base.entry(&expr.projection) + { + return Some(entry.to_expr()); + } + + // Project into Match: distribute the projection into each arm's expression. + // Example: project(Match(d, [1 => Record([d, a]), 2 => Record([d, n])]), [0]) + // → Match(d, [1 => project(Record([d, a]), [0]), 2 => project(Record([d, n]), [0])]) + // → Match(d, [1 => d, 2 => d]) (after recursive simplification) + if let stmt::Expr::Match(match_expr) = &mut *expr.base { + for arm in &mut match_expr.arms { + arm.expr = stmt::Expr::project(arm.expr.take(), expr.projection.clone()); + } + *match_expr.else_expr = + stmt::Expr::project(match_expr.else_expr.take(), expr.projection.clone()); + return Some(expr.base.take()); + } + + // A path into a `#[document]` column lowers to a plain `ExprProject` + // and stays one — positional projection over a structured value, the + // same shape a column-expanded embed uses. The SQL serializer renders + // it to a JSON path extraction at the edge; the in-memory interpreter + // evaluates it directly via `Value::entry`. The engine itself never + // deals in JSON. + None + } +} diff --git a/packages/toasty/src/engine/simplify/merge_in_subqueries.rs b/packages/toasty/src/engine/simplify/merge_in_subqueries.rs new file mode 100644 index 00000000000..895e4123bab --- /dev/null +++ b/packages/toasty/src/engine/simplify/merge_in_subqueries.rs @@ -0,0 +1,275 @@ +use super::Simplify; +use toasty_core::stmt::{self, Expr, ExprReference, Source, Visit, VisitMut}; + +impl Simplify<'_> { + /// Combine `IN` subqueries joined by `AND` when they test the same row. + /// + /// For example, if `users.id` is a primary key, this filter: + /// + /// ```sql + /// user_id IN (SELECT id FROM users WHERE name = 'alice') + /// AND user_id IN (SELECT id FROM users WHERE active) + /// ``` + /// becomes: + /// + /// ```sql + /// user_id IN (SELECT id FROM users WHERE name = 'alice' AND active) + /// ``` + /// Both checks refer to the same user. That user must satisfy both + /// filters. The subqueries can come from relation paths, explicit queries, + /// or a mix of the two. Their operands need not be next to each other. + /// + /// The filters can contain `OR` or nested subqueries. For example, combining + /// a filter for either Alice or Bob with a filter for active users gives: + /// + /// ```sql + /// user_id IN ( + /// SELECT id FROM users + /// WHERE (name = 'alice' OR name = 'bob') AND active + /// ) + /// ``` + /// + /// Both queries must test the same value and read the same columns from + /// the same source. The returned columns must include a complete unique + /// key, and none may be nullable. A secondary unique key works too. For a + /// key made of `(tenant_id, id)`, returning only `tenant_id` is not enough. + /// + /// A query that returns a non-unique value stays separate. For example: + /// + /// ```sql + /// parent_id IN (SELECT parent_id FROM children WHERE color = 'red') + /// AND parent_id IN (SELECT parent_id FROM children WHERE color = 'blue') + /// ``` + /// + /// A parent can have one red child and another blue child. Combining the + /// filters would require one child to be both red and blue. This is why + /// separate `has_many` checks cannot usually be combined. + /// + /// Queries with limits also stay separate. For example, adding `active` + /// to `WHERE name = 'alice' LIMIT 1` can select a different user from the + /// one chosen before the merge. This rule also skips ordering, locks, + /// CTEs, joins, set operations, and queries marked as returning one row. + /// Those query restrictions also apply to nested subqueries. Queries that + /// contain function calls or references to an outer query stay separate + /// too. This rule combines only positive `IN` checks, not `NOT IN` checks. + /// + /// SQL nulls need care. If the left value is null, each `IN` check can be + /// null even when the subqueries have no rows in common. The merged query + /// then returns no rows, making `IN` false. Both results are rejected by a + /// `WHERE` filter, including through `AND` or `OR`. They differ under `NOT` or when returned + /// as a value, so those cases require a proven non-null left value. NoSQL + /// membership always returns a boolean and does not need this restriction. + /// + /// This runs after relation lifting and before lowering extracts NoSQL + /// subqueries into separate statements. Newly combined filters are + /// simplified too, so matching subqueries inside them can also combine. + pub(super) fn merge_in_subqueries(&self, operands: &mut Vec) { + for i in 0..operands.len() { + // Earlier merges can shorten the operand list. + if i >= operands.len() { + break; + } + let Expr::InSubquery(first) = &operands[i] else { + continue; + }; + if !self.can_merge_membership(first) { + continue; + } + + let mut j = i + 1; + while j < operands.len() { + let Expr::InSubquery(first) = &operands[i] else { + unreachable!(); + }; + let compatible = match &operands[j] { + Expr::InSubquery(other) if self.can_merge_membership(other) => { + let a = first.query.body.as_select_unwrap(); + let b = other.query.body.as_select_unwrap(); + first.expr.is_equivalent_to(&other.expr) + && a.source == b.source + && a.returning == b.returning + && a.distinct == b.distinct + } + _ => false, + }; + if !compatible { + j += 1; + continue; + } + + let Expr::InSubquery(other) = operands.remove(j) else { + unreachable!(); + }; + let Expr::InSubquery(first) = &mut operands[i] else { + unreachable!(); + }; + let select = first.query.body.as_select_mut_unwrap(); + select.add_filter(other.query.body.as_select_unwrap().filter.clone()); + // Combining filters can expose further memberships at a deeper + // relation hop. Simplify them before subquery extraction. + self.scope(&select.source) + .visit_filter_mut(&mut select.filter); + } + } + } + + fn can_merge_membership(&self, membership: &stmt::ExprInSubquery) -> bool { + if membership.negated || !membership.expr.is_stable() { + return false; + } + // In SQL, NULL IN two nonempty disjoint sets is NULL AND NULL, while + // NULL IN their intersection is false. They select the same rows in + // a positive filter, but are not interchangeable as values or under NOT. + // NoSQL membership uses value equality and always returns a boolean. + if self.capability.sql() + && !self.positive_filter + && !self.non_nullable_key(&membership.expr) + { + return false; + } + let mut safe = SafeQuery(true); + safe.visit_stmt_query(&membership.query); + if !safe.0 { + return false; + } + let select = membership.query.body.as_select_unwrap(); + let stmt::Returning::Project(projection) = &select.returning else { + return false; + }; + let fields = projection + .as_record() + .map_or(std::slice::from_ref(projection), |record| &record.fields); + + match &select.source { + Source::Model(source) => { + let model = self.cx.schema().app.model(source.id).as_root_unwrap(); + let Some(indices) = fields + .iter() + .map(|expr| match expr { + Expr::Reference(ExprReference::Field { nesting: 0, index }) + if !model.fields[*index].nullable => + { + Some(*index) + } + _ => None, + }) + .collect::>>() + else { + return false; + }; + model.indices.iter().any(|index| { + index.unique + && !index.fields.is_empty() + && index + .fields + .iter() + .all(|field| indices.contains(&field.field.index)) + }) + } + Source::Table(source) => { + let stmt::TableRef::Table(id) = source.tables[0] else { + unreachable!(); + }; + let table = self.cx.schema().db.table(id); + let Some(indices) = fields + .iter() + .map(|expr| match expr { + Expr::Reference(ExprReference::Column(column)) + if column.nesting == 0 + && column.table == 0 + && !table.columns[column.column].nullable => + { + Some(column.column) + } + _ => None, + }) + .collect::>>() + else { + return false; + }; + table.indices.iter().any(|index| { + index.unique + && !index.columns.is_empty() + && index + .columns + .iter() + .all(|column| indices.contains(&column.column.index)) + }) + } + } + } + + fn non_nullable_key(&self, expr: &Expr) -> bool { + match expr { + Expr::Reference(reference @ ExprReference::Field { .. }) => !self + .cx + .resolve_expr_reference(reference) + .as_field_unwrap() + .nullable(), + Expr::Record(record) => record + .fields + .iter() + .all(|field| self.non_nullable_key(field)), + Expr::Value(value) => !value.is_null() && !matches!(value, stmt::Value::Record(_)), + // Columns may be nullable through an outer join even when their + // schema declaration is not. Other expressions need a nullability proof. + _ => false, + } + } +} + +/// Restrict merging to plain reads. Walk nested subqueries too: their filters +/// may contain arbitrary boolean expressions, but must not introduce effects, +/// unstable functions, or references to an outer query scope. +struct SafeQuery(bool); + +impl Visit for SafeQuery { + fn visit_stmt_query(&mut self, query: &stmt::Query) { + if query.with.is_some() + || query.limit.is_some() + || query.order_by.is_some() + || !query.locks.is_empty() + || query.single + { + self.0 = false; + return; + } + let Some(select) = query.body.as_select() else { + self.0 = false; + return; + }; + let plain_source = match &select.source { + Source::Model(source) => source.via.is_none(), + Source::Table(source) => { + matches!(source.tables.as_slice(), [stmt::TableRef::Table(_)]) + && matches!(source.from.as_slice(), [from] + if matches!(from.relation, stmt::TableFactor::Table(stmt::SourceTableId(0))) + && from.joins.is_empty()) + } + }; + if !plain_source || !matches!(select.returning, stmt::Returning::Project(_)) { + self.0 = false; + return; + } + stmt::visit::visit_stmt_query(self, query); + } + + fn visit_expr(&mut self, expr: &Expr) { + if matches!( + expr, + Expr::Func(_) | Expr::Stmt(_) | Expr::Default | Expr::Ident(_) | Expr::Error(_) + ) { + self.0 = false; + } else { + stmt::visit::visit_expr(self, expr); + } + } + + fn visit_expr_reference(&mut self, reference: &ExprReference) { + let nesting = match reference { + ExprReference::Field { nesting, .. } | ExprReference::Model { nesting } => *nesting, + ExprReference::Column(column) => column.nesting, + }; + self.0 &= nesting == 0; + } +} diff --git a/packages/toasty/src/engine/simplify/stmt_query.rs b/packages/toasty/src/engine/simplify/stmt_query.rs new file mode 100644 index 00000000000..b06b602404d --- /dev/null +++ b/packages/toasty/src/engine/simplify/stmt_query.rs @@ -0,0 +1,85 @@ +use toasty_core::stmt; + +use crate::engine::simplify::Simplify; + +impl Simplify<'_> { + pub(super) fn simplify_stmt_query_when_empty(&mut self, stmt: &mut stmt::Query) { + if stmt.with.is_some() { + // Just skip if there are any CTEs for now + return; + } + + if self.stmt_query_is_empty(stmt) { + stmt.body = stmt::ExprSet::Values(Default::default()); + stmt.order_by = None; + stmt.limit = None; + } + } + + pub(super) fn stmt_query_is_empty(&self, query: &stmt::Query) -> bool { + match &query.body { + stmt::ExprSet::Values(values) => values.is_empty(), + stmt::ExprSet::Select(select) => self.stmt_select_is_empty(select), + _ => false, + } + } + + pub(super) fn stmt_select_is_empty(&self, select: &stmt::Select) -> bool { + // Only check table sources (lowered queries) + let stmt::Source::Table(source_table) = &select.source else { + return false; + }; + + // SQL-style predicates retain only true rows. A constant NULL + // comparison (for example, a cleared relation key) also matches none. + if select.filter.is_false() || select.filter.as_expr().is_value_null() { + return true; + } + + // Check if the source guarantees empty results + self.source_table_is_empty(source_table) + } + + fn source_table_is_empty(&self, source_table: &stmt::SourceTable) -> bool { + if source_table.from.is_empty() { + todo!( + "this case is not handled yet at this level; \ + Do we want to handle this or transform the statement \ + to something different first?" + ); + } + + // Get the primary relation from the first table + let stmt::TableFactor::Table(primary_table_id) = &source_table.from[0].relation; + + // Check if primary source is empty VALUES + if self.table_ref_is_empty_values(&source_table.tables[primary_table_id.0]) { + // Primary source is empty - result will be empty unless there are + // LEFT JOINs that could still produce results + return !source_table + .from + .iter() + .flat_map(|twj| &twj.joins) + .any(|join| matches!(join.constraint, stmt::JoinOp::Left(_))); + } + + // Check if any non-optional join has empty VALUES + // Currently only LEFT joins exist, so any other join would be INNER + // For now, this is conservative - we'd add INNER/CROSS join detection here + false + } + + fn table_ref_is_empty_values(&self, table_ref: &stmt::TableRef) -> bool { + match table_ref { + stmt::TableRef::Derived(derived) => { + // Check if the derived table is empty VALUES. Inner queries + // would have already been simplified, so we only need to check + // for empty values here. + matches!(&derived.subquery.body, + stmt::ExprSet::Values(values) if values.is_empty() + ) + } + _ => false, + } + } +} diff --git a/packages/toasty/src/engine/simplify/tests.rs b/packages/toasty/src/engine/simplify/tests.rs new file mode 100644 index 00000000000..c455e9c1ae4 --- /dev/null +++ b/packages/toasty/src/engine/simplify/tests.rs @@ -0,0 +1,17 @@ +pub use crate::engine::test_util::*; + +mod expr_and; +mod expr_any; +mod expr_binary_op; +mod expr_cast; +mod expr_exists; +mod expr_intersects; +mod expr_is_null; +mod expr_is_superset; +mod expr_map; +mod expr_match; +mod expr_or; +mod expr_project; +mod merge_in_subqueries; +mod prop_const; +mod stmt_query; diff --git a/packages/toasty/src/engine/simplify/tests/expr_and.rs b/packages/toasty/src/engine/simplify/tests/expr_and.rs new file mode 100644 index 00000000000..333a1ca1fb0 --- /dev/null +++ b/packages/toasty/src/engine/simplify/tests/expr_and.rs @@ -0,0 +1,708 @@ +use super::test_schema; +use crate::engine::simplify::Simplify; +use toasty_core::stmt::{BinaryOp, Expr, ExprAnd, ExprOr}; + +// Cheap canonicalization (flatten, drop unit literals, null propagation, +// single/empty collapse on canonical input) is exercised in +// `engine::fold::tests::expr_and`. Tests here cover the heavyweight rules +// only: idempotent law, absorption, complement, range-to-equality, +// contradiction detection, and OR-branch pruning. + +#[test] +fn idempotent_two_identical() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `and(a, a) → a` + let mut expr = ExprAnd { + operands: vec![Expr::arg(0), Expr::arg(0)], + }; + let result = simplify.simplify_expr_and(&mut expr); + + assert!(result.is_some()); + assert_eq!(result.unwrap(), Expr::arg(0)); +} + +#[test] +fn idempotent_three_identical() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `and(a, a, a) → a` + let mut expr = ExprAnd { + operands: vec![Expr::arg(0), Expr::arg(0), Expr::arg(0)], + }; + let result = simplify.simplify_expr_and(&mut expr); + + assert!(result.is_some()); + assert_eq!(result.unwrap(), Expr::arg(0)); +} + +#[test] +fn idempotent_with_different() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `and(a, b, a) → and(a, b)` + let mut expr = ExprAnd { + operands: vec![Expr::arg(0), Expr::arg(1), Expr::arg(0)], + }; + let result = simplify.simplify_expr_and(&mut expr); + + assert!(result.is_none()); + assert_eq!(expr.operands.len(), 2); + assert_eq!(expr.operands[0], Expr::arg(0)); + assert_eq!(expr.operands[1], Expr::arg(1)); +} + +#[test] +fn absorption_and_or() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `and(a, or(a, b))` → `a` + let mut expr = ExprAnd { + operands: vec![ + Expr::arg(0), + Expr::Or(ExprOr { + operands: vec![Expr::arg(0), Expr::arg(1)], + }), + ], + }; + let result = simplify.simplify_expr_and(&mut expr); + + assert!(result.is_some()); + assert_eq!(result.unwrap(), Expr::arg(0)); +} + +#[test] +fn absorption_with_multiple_operands() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `and(a, b, or(a, c))` → `and(a, b)` + let mut expr = ExprAnd { + operands: vec![ + Expr::arg(0), + Expr::arg(1), + Expr::Or(ExprOr { + operands: vec![Expr::arg(0), Expr::arg(2)], + }), + ], + }; + let result = simplify.simplify_expr_and(&mut expr); + + assert!(result.is_none()); + assert_eq!(expr.operands.len(), 2); + assert_eq!(expr.operands[0], Expr::arg(0)); + assert_eq!(expr.operands[1], Expr::arg(1)); +} + +#[test] +fn absorption_two_and_three_or() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `and(a, b, or(a, c, d))` → `and(a, b)` + let mut expr = ExprAnd { + operands: vec![ + Expr::arg(0), + Expr::arg(1), + Expr::Or(ExprOr { + operands: vec![Expr::arg(0), Expr::arg(2), Expr::arg(3)], + }), + ], + }; + let result = simplify.simplify_expr_and(&mut expr); + + assert!(result.is_none()); + assert_eq!(expr.operands.len(), 2); + assert_eq!(expr.operands[0], Expr::arg(0)); + assert_eq!(expr.operands[1], Expr::arg(1)); +} + +#[test] +fn complement_basic() { + use toasty_core::stmt::ExprNot; + + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `a and not(a)` → `false` (where a is a non-nullable comparison) + let a = Expr::eq(Expr::arg(0), Expr::arg(1)); + let mut expr = ExprAnd { + operands: vec![a.clone(), Expr::Not(ExprNot { expr: Box::new(a) })], + }; + let result = simplify.simplify_expr_and(&mut expr); + + assert!(result.is_some()); + assert!(result.unwrap().is_false()); +} + +#[test] +fn complement_with_other_operands() { + use toasty_core::stmt::ExprNot; + + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `a and b and not(a)` → `false` + let a = Expr::eq(Expr::arg(0), Expr::arg(1)); + let mut expr = ExprAnd { + operands: vec![ + a.clone(), + Expr::arg(2), + Expr::Not(ExprNot { expr: Box::new(a) }), + ], + }; + let result = simplify.simplify_expr_and(&mut expr); + + assert!(result.is_some()); + assert!(result.unwrap().is_false()); +} + +#[test] +fn complement_nullable_not_simplified() { + use toasty_core::stmt::ExprNot; + + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `a and not(a)` where `a` is an arg (nullable) → no change + let a = Expr::arg(0); + let mut expr = ExprAnd { + operands: vec![a.clone(), Expr::Not(ExprNot { expr: Box::new(a) })], + }; + let result = simplify.simplify_expr_and(&mut expr); + + assert!(result.is_none()); +} + +#[test] +fn complement_multiple_repetitions() { + use toasty_core::stmt::ExprNot; + + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `a and a and not(a) and not(a)` → `false` + let a = Expr::eq(Expr::arg(0), Expr::arg(1)); + let mut expr = ExprAnd { + operands: vec![ + a.clone(), + a.clone(), + Expr::Not(ExprNot { + expr: Box::new(a.clone()), + }), + Expr::Not(ExprNot { expr: Box::new(a) }), + ], + }; + let result = simplify.simplify_expr_and(&mut expr); + + assert!(result.is_some()); + assert!(result.unwrap().is_false()); +} + +#[test] +fn range_to_equality_ge_le() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `a >= 5 and a <= 5` → `a = 5` + let mut expr = ExprAnd { + operands: vec![ + Expr::binary_op(Expr::arg(0), BinaryOp::Ge, 5i64), + Expr::binary_op(Expr::arg(0), BinaryOp::Le, 5i64), + ], + }; + let result = simplify.simplify_expr_and(&mut expr); + + let Some(Expr::BinaryOp(bin_op)) = result else { + panic!("expected binary op"); + }; + assert!(bin_op.op.is_eq()); +} + +#[test] +fn range_to_equality_le_ge() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `a <= 5 and a >= 5` → `a = 5` (opposite order) + let mut expr = ExprAnd { + operands: vec![ + Expr::binary_op(Expr::arg(0), BinaryOp::Le, 5i64), + Expr::binary_op(Expr::arg(0), BinaryOp::Ge, 5i64), + ], + }; + let result = simplify.simplify_expr_and(&mut expr); + + let Some(Expr::BinaryOp(bin_op)) = result else { + panic!("expected binary op"); + }; + assert!(bin_op.op.is_eq()); +} + +#[test] +fn range_to_equality_different_bounds_not_simplified() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `a >= 5 and a <= 10` is not simplified (different bounds) + let mut expr = ExprAnd { + operands: vec![ + Expr::binary_op(Expr::arg(0), BinaryOp::Ge, 5i64), + Expr::binary_op(Expr::arg(0), BinaryOp::Le, 10i64), + ], + }; + let result = simplify.simplify_expr_and(&mut expr); + + assert!(result.is_none()); + assert_eq!(expr.operands.len(), 2); +} + +#[test] +fn range_to_equality_different_exprs_not_simplified() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `a >= 5 and b <= 5` is not simplified (different expressions) + let mut expr = ExprAnd { + operands: vec![ + Expr::binary_op(Expr::arg(0), BinaryOp::Ge, 5i64), + Expr::binary_op(Expr::arg(1), BinaryOp::Le, 5i64), + ], + }; + let result = simplify.simplify_expr_and(&mut expr); + + assert!(result.is_none()); + assert_eq!(expr.operands.len(), 2); +} + +#[test] +fn range_to_equality_with_other_operands() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `x and a >= 5 and a <= 5` → `x and a = 5` + let mut expr = ExprAnd { + operands: vec![ + Expr::arg(0), + Expr::binary_op(Expr::arg(1), BinaryOp::Ge, 5i64), + Expr::binary_op(Expr::arg(1), BinaryOp::Le, 5i64), + ], + }; + let result = simplify.simplify_expr_and(&mut expr); + + assert!(result.is_none()); // Still has multiple operands + assert_eq!(expr.operands.len(), 2); + + // One should be arg(0), the other should be the equality + let has_equality = expr + .operands + .iter() + .any(|e| matches!(e, Expr::BinaryOp(op) if op.op.is_eq())); + assert!(has_equality); +} + +#[test] +fn range_to_equality_uneven_repetitions() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `a >= 5 and a >= 5 and a <= 5` → `a = 5` + let mut expr = ExprAnd { + operands: vec![ + Expr::binary_op(Expr::arg(0), BinaryOp::Ge, 5i64), + Expr::binary_op(Expr::arg(0), BinaryOp::Ge, 5i64), + Expr::binary_op(Expr::arg(0), BinaryOp::Le, 5i64), + ], + }; + let result = simplify.simplify_expr_and(&mut expr); + + // All bounds collapse to a single equality + let Some(Expr::BinaryOp(bin_op)) = result else { + panic!("expected binary op"); + }; + assert!(bin_op.op.is_eq()); +} + +// --- AND-over-OR branch pruning tests --- + +/// The core enum variant+field pattern: disc == 1 in the outer AND +/// contradicts disc != 1 in the else branch of the OR. +/// +/// AND(disc == 1, OR(AND(disc == 1, addr == "alice"), AND(disc != 1, Error == "alice"))) +/// → AND(disc == 1, addr == "alice") +#[test] +fn prune_or_branch_contradicting_outer_eq() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + let disc_eq_1 = Expr::eq(Expr::arg(0), Expr::from(1i64)); + let addr_eq_alice = Expr::eq(Expr::arg(1), Expr::from("alice")); + let disc_ne_1 = Expr::ne(Expr::arg(0), Expr::from(1i64)); + let error_eq_alice = Expr::eq(Expr::error("unreachable"), Expr::from("alice")); + + let mut expr = ExprAnd { + operands: vec![ + disc_eq_1.clone(), + Expr::Or(ExprOr { + operands: vec![ + Expr::and(disc_eq_1.clone(), addr_eq_alice.clone()), + Expr::and(disc_ne_1, error_eq_alice), + ], + }), + ], + }; + + let result = simplify.simplify_expr_and(&mut expr); + + // Should simplify: OR collapses to single branch, then AND flattens + // and deduplicates disc == 1. + assert!(result.is_none()); // Still has 2 operands + assert_eq!(expr.operands.len(), 2); + + // The two remaining operands should be disc == 1 and addr == "alice" + assert!(expr.operands.contains(&disc_eq_1)); + assert!(expr.operands.contains(&addr_eq_alice)); +} + +/// Multiple OR branches where only the matching one survives. +/// +/// AND(x == 1, OR(AND(x == 1, a), AND(x == 2, b), AND(x == 3, c))) +/// → AND(x == 1, a) +#[test] +fn prune_or_multiple_contradicting_branches() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + let x_eq_1 = Expr::eq(Expr::arg(0), Expr::from(1i64)); + let x_eq_2 = Expr::eq(Expr::arg(0), Expr::from(2i64)); + let x_eq_3 = Expr::eq(Expr::arg(0), Expr::from(3i64)); + + let mut expr = ExprAnd { + operands: vec![ + x_eq_1.clone(), + Expr::Or(ExprOr { + operands: vec![ + Expr::and(x_eq_1.clone(), Expr::arg(1)), + Expr::and(x_eq_2, Expr::arg(2)), + Expr::and(x_eq_3, Expr::arg(3)), + ], + }), + ], + }; + + let result = simplify.simplify_expr_and(&mut expr); + + assert!(result.is_none()); + assert_eq!(expr.operands.len(), 2); + assert!(expr.operands.contains(&x_eq_1)); + assert!(expr.operands.contains(&Expr::arg(1))); +} + +/// When no OR branch contradicts the outer constraint, nothing is pruned. +#[test] +fn prune_or_no_contradiction_preserved() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // AND(x == 1, OR(AND(y == 2, a), AND(y == 3, b))) + // x == 1 doesn't contradict y == 2 or y == 3 — no pruning. + let mut expr = ExprAnd { + operands: vec![ + Expr::eq(Expr::arg(0), Expr::from(1i64)), + Expr::Or(ExprOr { + operands: vec![ + Expr::and(Expr::eq(Expr::arg(1), Expr::from(2i64)), Expr::arg(2)), + Expr::and(Expr::eq(Expr::arg(1), Expr::from(3i64)), Expr::arg(3)), + ], + }), + ], + }; + + let result = simplify.simplify_expr_and(&mut expr); + + // No simplification — 2 operands remain, OR still has 2 branches. + assert!(result.is_none()); + assert_eq!(expr.operands.len(), 2); + assert!(matches!(&expr.operands[1], Expr::Or(or) if or.operands.len() == 2)); +} + +/// All OR branches pruned → OR becomes false → AND becomes false. +#[test] +fn prune_or_all_branches_contradicted() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // AND(x == 1, OR(AND(x == 2, a), AND(x == 3, b))) + // Both branches contradict x == 1, so OR → false → AND → false. + let mut expr = ExprAnd { + operands: vec![ + Expr::eq(Expr::arg(0), Expr::from(1i64)), + Expr::Or(ExprOr { + operands: vec![ + Expr::and(Expr::eq(Expr::arg(0), Expr::from(2i64)), Expr::arg(1)), + Expr::and(Expr::eq(Expr::arg(0), Expr::from(3i64)), Expr::arg(2)), + ], + }), + ], + }; + + let result = simplify.simplify_expr_and(&mut expr); + + assert!(result.is_some()); + assert!(result.unwrap().is_false()); +} + +/// A non-AND branch in the OR is tested as a single-element constraint. +/// +/// AND(x == 1, OR(x == 2, AND(x == 1, a))) +/// → prune `x == 2` (contradicts x == 1) +/// → AND(x == 1, a) +#[test] +fn prune_or_non_and_branch() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + let x_eq_1 = Expr::eq(Expr::arg(0), Expr::from(1i64)); + let x_eq_2 = Expr::eq(Expr::arg(0), Expr::from(2i64)); + + let mut expr = ExprAnd { + operands: vec![ + x_eq_1.clone(), + Expr::Or(ExprOr { + operands: vec![ + x_eq_2, // bare expr, not wrapped in AND + Expr::and(x_eq_1.clone(), Expr::arg(1)), + ], + }), + ], + }; + + let result = simplify.simplify_expr_and(&mut expr); + + assert!(result.is_none()); + assert_eq!(expr.operands.len(), 2); + assert!(expr.operands.contains(&x_eq_1)); + assert!(expr.operands.contains(&Expr::arg(1))); +} + +/// Multiple non-OR operands form the constraint set. +/// +/// AND(x == 1, y == 2, OR(AND(x == 1, y == 3, a), AND(x == 1, y == 2, b))) +/// → prune first OR branch (y == 2 contradicts y == 3) +/// → AND(x == 1, y == 2, b) +#[test] +fn prune_or_multiple_constraints() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + let x_eq_1 = Expr::eq(Expr::arg(0), Expr::from(1i64)); + let y_eq_2 = Expr::eq(Expr::arg(1), Expr::from(2i64)); + let y_eq_3 = Expr::eq(Expr::arg(1), Expr::from(3i64)); + + let mut expr = ExprAnd { + operands: vec![ + x_eq_1.clone(), + y_eq_2.clone(), + Expr::Or(ExprOr { + operands: vec![ + Expr::and_from_vec(vec![x_eq_1.clone(), y_eq_3, Expr::arg(2)]), + Expr::and_from_vec(vec![x_eq_1.clone(), y_eq_2.clone(), Expr::arg(3)]), + ], + }), + ], + }; + + let result = simplify.simplify_expr_and(&mut expr); + + // After pruning + flatten + dedup: AND(x == 1, y == 2, arg(3)) + assert!(result.is_none()); + assert_eq!(expr.operands.len(), 3); + assert!(expr.operands.contains(&x_eq_1)); + assert!(expr.operands.contains(&y_eq_2)); + assert!(expr.operands.contains(&Expr::arg(3))); +} + +/// When the AND has no non-OR operands, no pruning occurs. +#[test] +fn prune_or_no_constraints_no_change() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // AND(OR(a, b), OR(c, d)) — no non-OR constraints to propagate. + let mut expr = ExprAnd { + operands: vec![ + Expr::Or(ExprOr { + operands: vec![Expr::arg(0), Expr::arg(1)], + }), + Expr::Or(ExprOr { + operands: vec![Expr::arg(2), Expr::arg(3)], + }), + ], + }; + + let result = simplify.simplify_expr_and(&mut expr); + + assert!(result.is_none()); + assert_eq!(expr.operands.len(), 2); +} + +/// End-to-end through visit_expr_mut: the full enum variant+field pattern. +/// +/// AND(disc == 1, eq(Match(disc, [1 => addr, 2 => arg(2)], else: Error), "alice")) +/// → match elimination produces OR +/// → AND-over-OR pruning removes else branch +/// → AND(disc == 1, addr == "alice") +#[test] +fn prune_or_end_to_end_via_visit() { + use toasty_core::stmt::{ExprMatch, MatchArm, Value, VisitMut}; + + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + let disc = Expr::arg(0); + let addr = Expr::arg(1); + + let mut expr = Expr::and( + Expr::eq(disc.clone(), Expr::from(1i64)), + Expr::eq( + Expr::Match(ExprMatch { + subject: Box::new(disc.clone()), + arms: vec![ + MatchArm { + pattern: Value::from(1i64), + expr: addr.clone(), + }, + MatchArm { + pattern: Value::from(2i64), + expr: Expr::arg(2), + }, + ], + else_expr: Box::new(Expr::error("unreachable")), + }), + Expr::from("alice"), + ), + ); + + simplify.visit_expr_mut(&mut expr); + + // Result should be AND(disc == 1, addr == "alice") + let Expr::And(and) = &expr else { + panic!("expected AND, got: {expr:?}"); + }; + assert_eq!(and.operands.len(), 2); + assert!(and.operands.contains(&Expr::eq(disc, Expr::from(1i64)))); + assert!(and.operands.contains(&Expr::eq(addr, Expr::from("alice")))); +} + +// --------------------------------------------------------------------------- +// Determinism-aware simplification (issue #236). +// +// `is_equivalent_to` gates every "syntactic identity implies semantic +// equivalence" rewrite on `Expr::is_stable`. Non-deterministic expressions +// (e.g. `LAST_INSERT_ID()`) are re-evaluated on each occurrence and must not +// be folded even when two occurrences are textually identical. +// --------------------------------------------------------------------------- + +#[test] +fn idempotent_not_simplified_for_non_deterministic() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `LAST_INSERT_ID() AND LAST_INSERT_ID()` must retain both operands. + let mut expr = ExprAnd { + operands: vec![Expr::last_insert_id(), Expr::last_insert_id()], + }; + let result = simplify.simplify_expr_and(&mut expr); + + assert!(result.is_none()); + assert_eq!(expr.operands.len(), 2); + assert_eq!(expr.operands[0], Expr::last_insert_id()); + assert_eq!(expr.operands[1], Expr::last_insert_id()); +} + +#[test] +fn absorption_not_simplified_for_non_deterministic() { + use toasty_core::stmt::ExprOr; + + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `x AND (x OR y)` would absorb to `x` under PartialEq, but with a + // non-deterministic `x` the two occurrences are independent draws, so + // absorption must not fire. + let x = Expr::eq(Expr::last_insert_id(), 1i64); + let mut expr = ExprAnd { + operands: vec![ + x.clone(), + Expr::Or(ExprOr { + operands: vec![x, Expr::arg(0)], + }), + ], + }; + let result = simplify.simplify_expr_and(&mut expr); + + assert!(result.is_none()); + assert_eq!(expr.operands.len(), 2); +} + +#[test] +fn complement_not_simplified_for_non_deterministic() { + use toasty_core::stmt::ExprNot; + + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `f() = 1 AND NOT (f() = 1)` — both `f()` calls are independent + // evaluations, so complement must NOT fire. (Compare with + // `complement_basic`, which uses `arg` — stable.) + let a = Expr::eq(Expr::last_insert_id(), 1i64); + let mut expr = ExprAnd { + operands: vec![a.clone(), Expr::Not(ExprNot { expr: Box::new(a) })], + }; + let result = simplify.simplify_expr_and(&mut expr); + + assert!(result.is_none()); + assert_eq!(expr.operands.len(), 2); +} + +#[test] +fn range_to_equality_not_simplified_for_non_deterministic() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `f() >= 5 AND f() <= 5` — two independent draws of `f()` bracketed + // by the same constant do not imply the draws are equal to 5. + let mut expr = ExprAnd { + operands: vec![ + Expr::binary_op(Expr::last_insert_id(), BinaryOp::Ge, 5i64), + Expr::binary_op(Expr::last_insert_id(), BinaryOp::Le, 5i64), + ], + }; + let result = simplify.simplify_expr_and(&mut expr); + + assert!(result.is_none()); + assert_eq!(expr.operands.len(), 2); +} + +#[test] +fn contradicting_eq_not_simplified_for_non_deterministic() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `f() == 1 AND f() == 2` — two independent draws can produce 1 and 2 + // respectively, so this is NOT a contradiction. + let mut expr = ExprAnd { + operands: vec![ + Expr::eq(Expr::last_insert_id(), 1i64), + Expr::eq(Expr::last_insert_id(), 2i64), + ], + }; + let result = simplify.simplify_expr_and(&mut expr); + + assert!(result.is_none()); + assert_eq!(expr.operands.len(), 2); +} diff --git a/packages/toasty/src/engine/simplify/tests/expr_any.rs b/packages/toasty/src/engine/simplify/tests/expr_any.rs new file mode 100644 index 00000000000..f40968daa30 --- /dev/null +++ b/packages/toasty/src/engine/simplify/tests/expr_any.rs @@ -0,0 +1,92 @@ +use super::test_schema; +use crate::engine::simplify::Simplify; +use toasty_core::stmt::{Expr, ExprAny, ExprMap, ExprOr, Value}; + +/// Helper to construct an "any of" expression. +fn any_of(expr: Expr) -> ExprAny { + ExprAny { + expr: Box::new(expr), + } +} + +/// Helper for making a map expression. +fn map_expr(base: Expr, map: Expr) -> Expr { + Expr::Map(ExprMap { + base: Box::new(base), + map: Box::new(map), + }) +} + +/// Helper for making a value list. +fn value_list(values: Vec) -> Expr { + Expr::Value(Value::List(values)) +} + +#[test] +fn non_map_returns_none() { + let schema = test_schema(); + let simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + let expr = any_of(Expr::arg(0)); + let result = simplify.simplify_expr_any(&expr); + + assert!(result.is_none()); +} + +#[test] +fn non_const_base_returns_none() { + let schema = test_schema(); + let simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // Map with non-constant base (arg(0)) + let expr = any_of(map_expr(Expr::arg(0), Expr::arg(0))); + let result = simplify.simplify_expr_any(&expr); + + assert!(result.is_none()); +} + +#[test] +fn empty_const_list_becomes_false() { + let schema = test_schema(); + let simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `any(map([], x => x)) → false` + let expr = any_of(map_expr(value_list(vec![]), Expr::arg(0))); + let result = simplify.simplify_expr_any(&expr); + + assert!(result.is_some()); + assert!(result.unwrap().is_false()); +} + +#[test] +fn single_item_unwrapped() { + let schema = test_schema(); + let simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `any(map([42], x => x)) → 42` + let expr = any_of(map_expr(value_list(vec![Value::from(42i64)]), Expr::arg(0))); + let result = simplify.simplify_expr_any(&expr); + + assert!(matches!(result, Some(Expr::Value(Value::I64(42))))); +} + +#[test] +fn multiple_items_become_or() { + let schema = test_schema(); + let simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `any(map([1, 2], x => x)) → or(1, 2)` + let expr = any_of(map_expr( + value_list(vec![Value::from(1i64), Value::from(2i64)]), + Expr::arg(0), + )); + let result = simplify.simplify_expr_any(&expr); + + assert!(matches!( + result, + Some(Expr::Or(ExprOr { operands })) + if operands.len() == 2 + && operands[0] == Expr::Value(Value::from(1i64)) + && operands[1] == Expr::Value(Value::from(2i64)) + )); +} diff --git a/packages/toasty/src/engine/simplify/tests/expr_binary_op.rs b/packages/toasty/src/engine/simplify/tests/expr_binary_op.rs new file mode 100644 index 00000000000..f02fee6910a --- /dev/null +++ b/packages/toasty/src/engine/simplify/tests/expr_binary_op.rs @@ -0,0 +1,488 @@ +use crate as toasty; +use crate::engine::simplify::Simplify; +use crate::schema::Model; +use toasty_core::{ + driver::Capability, + schema::{Builder, app}, + stmt::{BinaryOp, Expr, ExprCast, ExprReference, MatchArm, Type, Value, ValueRecord, VisitMut}, +}; + +#[derive(toasty::Model)] +struct User { + #[key] + id: String, + + #[allow(dead_code)] + name: Option, +} + +#[derive(toasty::Model)] +struct UuidColumns { + #[key] + id: String, + + lhs: uuid::Uuid, + rhs: uuid::Uuid, +} + +#[cfg(feature = "rust_decimal")] +#[derive(toasty::Model)] +struct DecimalColumns { + #[key] + id: String, + + lhs: rust_decimal::Decimal, + rhs: rust_decimal::Decimal, +} + +fn test_schema() -> toasty_core::Schema { + let app_schema = + app::Schema::from_macro([User::schema()]).expect("schema should build from macro"); + + Builder::new() + .build(app_schema, &Capability::SQLITE) + .expect("schema should build") +} + +#[test] +fn non_id_cast_not_unwrapped() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `eq(cast(arg(0), String), "test")`, non-Id cast is not unwrapped + let mut lhs = Expr::Cast(ExprCast { + from: None, + expr: Box::new(Expr::arg(0)), + ty: Type::String, + }); + let mut rhs = Expr::Value(Value::from("test")); + + let result = simplify.simplify_expr_binary_op(BinaryOp::Eq, &mut lhs, &mut rhs); + + assert!(result.is_none()); + assert!(matches!(lhs, Expr::Cast(_))); +} + +#[test] +fn equality_preserving_column_casts_are_stripped() { + let schema = crate::engine::test_util::test_schema_with(&[UuidColumns::schema()]); + let model = schema.app.model(UuidColumns::id()); + let simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + let mut simplify = simplify.scope(model.as_root_unwrap()); + let mut lhs = Expr::cast(Expr::column(ExprReference::column(0, 1)), Type::Uuid); + let mut rhs = Expr::cast(Expr::column(ExprReference::column(0, 2)), Type::Uuid); + + let result = simplify.simplify_expr_binary_op(BinaryOp::Eq, &mut lhs, &mut rhs); + + let Some(Expr::BinaryOp(result)) = result else { + panic!("expected binary op"); + }; + assert!(result.lhs.is_column()); + assert!(result.rhs.is_column()); +} + +#[cfg(feature = "rust_decimal")] +#[test] +fn non_equality_preserving_column_casts_are_not_stripped() { + let schema = crate::engine::test_util::test_schema_with(&[DecimalColumns::schema()]); + let model = schema.app.model(DecimalColumns::id()); + let simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + let mut simplify = simplify.scope(model.as_root_unwrap()); + let mut lhs = Expr::cast(Expr::column(ExprReference::column(0, 1)), Type::Decimal); + let mut rhs = Expr::cast(Expr::column(ExprReference::column(0, 2)), Type::Decimal); + + let result = simplify.simplify_expr_binary_op(BinaryOp::Eq, &mut lhs, &mut rhs); + + assert!(result.is_none()); + assert!(lhs.is_cast()); + assert!(rhs.is_cast()); +} + +#[test] +fn self_comparison_eq_non_nullable_becomes_true() { + let schema = test_schema(); + let model = schema.app.model(User::id()); + let simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + let mut simplify = simplify.scope(model.as_root_unwrap()); + + // `id = id` → `true` (non-nullable field) + let mut lhs = Expr::Reference(ExprReference::Field { + nesting: 0, + index: 0, + }); + let mut rhs = Expr::Reference(ExprReference::Field { + nesting: 0, + index: 0, + }); + + let result = simplify.simplify_expr_binary_op(BinaryOp::Eq, &mut lhs, &mut rhs); + + assert!(matches!(result, Some(Expr::Value(Value::Bool(true))))); +} + +#[test] +fn self_comparison_ne_non_nullable_becomes_false() { + let schema = test_schema(); + let model = schema.app.model(User::id()); + let simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + let mut simplify = simplify.scope(model.as_root_unwrap()); + + // `id != id` → `false` (non-nullable field) + let mut lhs = Expr::Reference(ExprReference::Field { + nesting: 0, + index: 0, + }); + let mut rhs = Expr::Reference(ExprReference::Field { + nesting: 0, + index: 0, + }); + + let result = simplify.simplify_expr_binary_op(BinaryOp::Ne, &mut lhs, &mut rhs); + + assert!(matches!(result, Some(Expr::Value(Value::Bool(false))))); +} + +#[test] +fn self_comparison_nullable_not_simplified() { + let schema = test_schema(); + let model = schema.app.model(User::id()); + let simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + let mut simplify = simplify.scope(model.as_root_unwrap()); + + // `name = name` is not simplified (nullable field) + let mut lhs = Expr::Reference(ExprReference::Field { + nesting: 0, + index: 1, + }); + let mut rhs = Expr::Reference(ExprReference::Field { + nesting: 0, + index: 1, + }); + + let result = simplify.simplify_expr_binary_op(BinaryOp::Eq, &mut lhs, &mut rhs); + + assert!(result.is_none()); +} + +#[test] +fn different_fields_not_simplified() { + let schema = test_schema(); + let model = schema.app.model(User::id()); + let simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + let mut simplify = simplify.scope(model.as_root_unwrap()); + + // `id = name` is not simplified (different fields) + let mut lhs = Expr::Reference(ExprReference::Field { + nesting: 0, + index: 0, + }); + let mut rhs = Expr::Reference(ExprReference::Field { + nesting: 0, + index: 1, + }); + + let result = simplify.simplify_expr_binary_op(BinaryOp::Eq, &mut lhs, &mut rhs); + + assert!(result.is_none()); +} + +#[test] +fn tuple_eq_decomposition_two_elements() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `(a, b) = (x, y)` → `a = x and b = y` + let mut lhs = Expr::record([Expr::arg(0), Expr::arg(1)]); + let mut rhs = Expr::record([Expr::arg(2), Expr::arg(3)]); + + let result = simplify.simplify_expr_binary_op(BinaryOp::Eq, &mut lhs, &mut rhs); + + let Some(Expr::And(and_expr)) = result else { + panic!("expected And expression"); + }; + assert_eq!(and_expr.len(), 2); + assert!(matches!(&and_expr[0], Expr::BinaryOp(op) if op.op.is_eq())); + assert!(matches!(&and_expr[1], Expr::BinaryOp(op) if op.op.is_eq())); +} + +#[test] +fn tuple_eq_decomposition_three_elements() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `(a, b, c) = (x, y, z)` → `a = x and b = y and c = z` + let mut lhs = Expr::record([Expr::arg(0), Expr::arg(1), Expr::arg(2)]); + let mut rhs = Expr::record([Expr::arg(3), Expr::arg(4), Expr::arg(5)]); + + let result = simplify.simplify_expr_binary_op(BinaryOp::Eq, &mut lhs, &mut rhs); + + let Some(Expr::And(and_expr)) = result else { + panic!("expected And expression"); + }; + assert_eq!(and_expr.len(), 3); +} + +#[test] +fn tuple_ne_decomposition() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `(a, b) != (x, y)` → `a != x or b != y` + let mut lhs = Expr::record([Expr::arg(0), Expr::arg(1)]); + let mut rhs = Expr::record([Expr::arg(2), Expr::arg(3)]); + + let result = simplify.simplify_expr_binary_op(BinaryOp::Ne, &mut lhs, &mut rhs); + + let Some(Expr::Or(or_expr)) = result else { + panic!("expected Or expression"); + }; + assert_eq!(or_expr.len(), 2); + assert!(matches!(&or_expr[0], Expr::BinaryOp(op) if op.op.is_ne())); + assert!(matches!(&or_expr[1], Expr::BinaryOp(op) if op.op.is_ne())); +} + +#[test] +fn single_element_tuple_eq() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `(a) = (x)` → `a = x` + let mut lhs = Expr::record([Expr::arg(0)]); + let mut rhs = Expr::record([Expr::arg(1)]); + + let result = simplify.simplify_expr_binary_op(BinaryOp::Eq, &mut lhs, &mut rhs); + assert!(matches!(result, Some(Expr::BinaryOp(op)) if op.op.is_eq())); +} + +// --- Match elimination tests --- + +#[test] +fn match_eq_constant_value() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // Match(col, [1 => Record([col, addr]), 2 => Record([col, num])], + // else: Record([col, Error])) == Value(Record([I64(1), "alice"])) + // → col == 1 AND addr == "alice" + // + // The else branch uses Record([col, Error]) matching the real-world + // data-carrying enum pattern. Tuple decomposition produces col == I64(1) + // which contradicts the NOT(col == 1) guard, so the complement law + // folds the else term to false. + let mut expr = Expr::binary_op( + Expr::match_expr( + Expr::arg(0), + vec![ + MatchArm { + pattern: Value::from(1i64), + expr: Expr::record([Expr::arg(0), Expr::arg(1)]), + }, + MatchArm { + pattern: Value::from(2i64), + expr: Expr::record([Expr::arg(0), Expr::arg(2)]), + }, + ], + Expr::record([Expr::arg(0), Expr::error("unreachable")]), + ), + BinaryOp::Eq, + Expr::from(Value::Record(ValueRecord::from_vec(vec![ + Value::from(1i64), + Value::from("alice"), + ]))), + ); + + simplify.visit_expr_mut(&mut expr); + + // Should be: arg(0) == 1 AND arg(1) == "alice" + let Expr::And(and) = &expr else { + panic!("expected And, got {expr:?}"); + }; + assert_eq!(and.len(), 2); +} + +#[test] +fn match_eq_scalar_folds_matching_arm() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // Match(arg(0), [1 => "a", 2 => "b"], else: "__") == "a" → arg(0) == 1 + // The else value "__" != "a" folds to false, pruning the else term. + let mut expr = Expr::binary_op( + Expr::match_expr( + Expr::arg(0), + vec![ + MatchArm { + pattern: Value::from(1i64), + expr: Expr::from("a"), + }, + MatchArm { + pattern: Value::from(2i64), + expr: Expr::from("b"), + }, + ], + Expr::from("__"), + ), + BinaryOp::Eq, + Expr::from("a"), + ); + + simplify.visit_expr_mut(&mut expr); + + // Only arm 1 survives (arm 2: "b" == "a" → false, pruned) + // Result: arg(0) == 1 + let Expr::BinaryOp(binop) = &expr else { + panic!("expected BinaryOp, got {expr:?}"); + }; + assert!(binop.op.is_eq()); + assert!(matches!(*binop.lhs, Expr::Arg(_))); + assert!(matches!(*binop.rhs, Expr::Value(Value::I64(1)))); +} + +#[test] +fn match_eq_no_matching_arm_folds_to_false() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // Match(arg(0), [1 => "a", 2 => "b"], else: "__") == "c" → false (all arms pruned) + // The else value "__" != "c" folds to false, pruning the else term too. + let mut expr = Expr::binary_op( + Expr::match_expr( + Expr::arg(0), + vec![ + MatchArm { + pattern: Value::from(1i64), + expr: Expr::from("a"), + }, + MatchArm { + pattern: Value::from(2i64), + expr: Expr::from("b"), + }, + ], + Expr::from("__"), + ), + BinaryOp::Eq, + Expr::from("c"), + ); + + simplify.visit_expr_mut(&mut expr); + + assert!(expr.is_false(), "expected false, got {expr:?}"); +} + +#[test] +fn match_on_rhs() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // "a" == Match(arg(0), [1 => "a", 2 => "b"], else: "__") → arg(0) == 1 + let mut expr = Expr::binary_op( + Expr::from("a"), + BinaryOp::Eq, + Expr::match_expr( + Expr::arg(0), + vec![ + MatchArm { + pattern: Value::from(1i64), + expr: Expr::from("a"), + }, + MatchArm { + pattern: Value::from(2i64), + expr: Expr::from("b"), + }, + ], + Expr::from("__"), + ), + ); + + simplify.visit_expr_mut(&mut expr); + + // Only arm 1 survives + let Expr::BinaryOp(binop) = &expr else { + panic!("expected BinaryOp, got {expr:?}"); + }; + assert!(binop.op.is_eq()); + assert!(matches!(*binop.lhs, Expr::Arg(_))); + assert!(matches!(*binop.rhs, Expr::Value(Value::I64(1)))); +} + +#[test] +fn match_ne_preserves_non_matching_arms() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // Match(arg(0), [1 => "a", 2 => "b"], else: "a") != "a" + // arm 1: arg(0) == 1 AND "a" != "a" → false → pruned + // arm 2: arg(0) == 2 AND "b" != "a" → arg(0) == 2 + // else: NOT(arg(0)==1) AND NOT(arg(0)==2) AND "a" != "a" → false → pruned + let mut expr = Expr::binary_op( + Expr::match_expr( + Expr::arg(0), + vec![ + MatchArm { + pattern: Value::from(1i64), + expr: Expr::from("a"), + }, + MatchArm { + pattern: Value::from(2i64), + expr: Expr::from("b"), + }, + ], + Expr::from("a"), + ), + BinaryOp::Ne, + Expr::from("a"), + ); + + simplify.visit_expr_mut(&mut expr); + + // Only arm 2 survives → arg(0) == 2 + let Expr::BinaryOp(binop) = &expr else { + panic!("expected BinaryOp, got {expr:?}"); + }; + assert!(binop.op.is_eq()); + assert!(matches!(*binop.lhs, Expr::Arg(_))); + assert!(matches!(*binop.rhs, Expr::Value(Value::I64(2)))); +} + +#[test] +fn match_with_non_constant_subject() { + let schema = test_schema(); + let model = schema.app.model(User::id()); + let simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + let mut simplify = simplify.scope(model.as_root_unwrap()); + + // Match over a column reference (the real-world case) + // Match(field[0], [1 => "a", 2 => "b"], else: "__") == "a" + let subject = Expr::Reference(ExprReference::Field { + nesting: 0, + index: 0, + }); + + let mut expr = Expr::binary_op( + Expr::match_expr( + subject, + vec![ + MatchArm { + pattern: Value::from(1i64), + expr: Expr::from("a"), + }, + MatchArm { + pattern: Value::from(2i64), + expr: Expr::from("b"), + }, + ], + Expr::from("__"), + ), + BinaryOp::Eq, + Expr::from("a"), + ); + + simplify.visit_expr_mut(&mut expr); + + // Only arm 1 survives. The guard becomes field[0] == 1. + // The exact shape depends on canonicalization, but there should be no Match left. + assert!( + !matches!(&expr, Expr::Match(_)), + "Match should be eliminated, got {expr:?}" + ); +} diff --git a/packages/toasty/src/engine/simplify/tests/expr_cast.rs b/packages/toasty/src/engine/simplify/tests/expr_cast.rs new file mode 100644 index 00000000000..731decef397 --- /dev/null +++ b/packages/toasty/src/engine/simplify/tests/expr_cast.rs @@ -0,0 +1,192 @@ +use super::test_schema_with; +use crate::engine::simplify::Simplify; +use toasty_core::stmt::{Expr, ExprCast, ExprReference, Type, Value, ValueObject}; + +/// Schema fixture for the document-cast tests: a model with a `#[document]` +/// embed, returning the schema and the embed's model-level type +/// (`Type::Model`). +fn document_schema() -> (toasty_core::Schema, Type) { + use crate as toasty; + use crate::schema::{Embed, Model}; + + #[allow(dead_code)] + #[derive(toasty::Embed)] + struct Profile { + name: String, + age: i64, + } + + #[allow(dead_code)] + #[derive(toasty::Model)] + struct User { + #[key] + id: String, + #[document] + profile: Profile, + } + + let schema = test_schema_with(&[User::schema(), Profile::schema()]); + let (_, profile_ty) = schema + .mapping + .document_columns + .first() + .expect("the schema has one document column"); + let profile_ty = profile_ty.clone(); + assert!(matches!(profile_ty, Type::Model(_))); + + (schema, profile_ty) +} + +#[test] +fn document_lowering_cast_folds_to_named_object() { + let (schema, profile_ty) = document_schema(); + let simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `cast(("Alice", 30) as Object, from Model)` → `{name: "Alice", age: 30}` + let mut expr = ExprCast { + from: Some(profile_ty), + expr: Box::new(Expr::Value(Value::record_from_vec(vec![ + Value::from("Alice"), + Value::I64(30), + ]))), + ty: Type::Object, + }; + let result = simplify.simplify_expr_cast(&mut expr); + + let Some(Expr::Value(Value::Object(object))) = result else { + panic!("expected the result to be a `Value::Object`"); + }; + assert_eq!( + object.entries, + vec![ + ("name".to_owned(), Value::from("Alice")), + ("age".to_owned(), Value::I64(30)), + ] + ); +} + +#[test] +fn document_raising_cast_folds_to_positional_record() { + let (schema, profile_ty) = document_schema(); + let simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // A wire object raises to the embed's positional record: an absent key + // (`name`) decodes to Null, a key unknown to the schema is dropped. + let mut expr = ExprCast { + from: None, + expr: Box::new(Expr::Value(Value::Object(ValueObject::from_vec(vec![ + ("age".to_owned(), Value::I64(30)), + ("unknown".to_owned(), Value::from("dropped")), + ])))), + ty: profile_ty, + }; + let result = simplify.simplify_expr_cast(&mut expr); + + assert_eq!( + result, + Some(Expr::Value(Value::record_from_vec(vec![ + Value::Null, + Value::I64(30), + ]))) + ); +} + +#[test] +fn document_raising_cast_is_idempotent() { + let (schema, profile_ty) = document_schema(); + let simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // An engine-computed positional record passes through the raising cast. + let record = Value::record_from_vec(vec![Value::from("Alice"), Value::I64(30)]); + let mut expr = ExprCast { + from: None, + expr: Box::new(Expr::Value(record.clone())), + ty: profile_ty, + }; + let result = simplify.simplify_expr_cast(&mut expr); + + assert_eq!(result, Some(Expr::Value(record))); +} + +#[test] +fn document_cast_on_non_const_kept() { + let (schema, profile_ty) = document_schema(); + let simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // A non-constant operand cannot fold; the cast survives for a later, + // post-substitution simplify. + let mut expr = ExprCast { + from: Some(profile_ty), + expr: Box::new(Expr::arg(0)), + ty: Type::Object, + }; + let result = simplify.simplify_expr_cast(&mut expr); + + assert!(result.is_none()); +} + +#[test] +fn redundant_cast_on_field_eliminated() { + use crate as toasty; + use crate::schema::Model; + + #[allow(dead_code)] + #[derive(toasty::Model)] + struct User { + #[key] + id: String, + name: String, + } + + let schema = test_schema_with(&[User::schema()]); + let model = schema.app.model(User::id()); + let simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + let simplify = simplify.scope(model.as_root_unwrap()); + + // `cast(name_field, String) → name_field` + // The `name` field (index 1) is already String, so the cast is a no-op. + let field_ref = Expr::Reference(ExprReference::Field { + nesting: 0, + index: 1, + }); + let mut expr = ExprCast { + from: None, + expr: Box::new(field_ref.clone()), + ty: Type::String, + }; + let result = simplify.simplify_expr_cast(&mut expr); + + assert_eq!(result, Some(field_ref)); +} + +#[test] +fn non_redundant_cast_on_field_kept() { + use crate as toasty; + use crate::schema::Model; + + #[allow(dead_code)] + #[derive(toasty::Model)] + struct User { + #[key] + id: String, + name: String, + } + + let schema = test_schema_with(&[User::schema()]); + let model = schema.app.model(User::id()); + let simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + let simplify = simplify.scope(model.as_root_unwrap()); + + // `cast(name_field, I64)` — String field cast to I64, not redundant. + let mut expr = ExprCast { + from: None, + expr: Box::new(Expr::Reference(ExprReference::Field { + nesting: 0, + index: 1, + })), + ty: Type::I64, + }; + let result = simplify.simplify_expr_cast(&mut expr); + + assert!(result.is_none()); +} diff --git a/packages/toasty/src/engine/simplify/tests/expr_exists.rs b/packages/toasty/src/engine/simplify/tests/expr_exists.rs new file mode 100644 index 00000000000..9030147a29d --- /dev/null +++ b/packages/toasty/src/engine/simplify/tests/expr_exists.rs @@ -0,0 +1,72 @@ +use super::test_schema; +use crate::engine::simplify::Simplify; +use toasty_core::stmt::{self, Expr, ExprExists, Query, Values, VisitMut as _}; + +/// Creates a query with no rows (empty VALUES clause). +fn empty_query() -> Box { + Box::new(Query::values(Values::default())) +} + +/// Creates a query with one row containing a single value. +fn non_empty_query() -> Box { + let mut values = Values::default(); + values.rows.push(stmt::Expr::Value(stmt::Value::from(1i64))); + Box::new(Query::values(values)) +} + +#[test] +fn exists_empty_query_becomes_false() { + let schema = test_schema(); + let simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `exists(empty_query)` → `false` + let expr = ExprExists { + subquery: empty_query(), + }; + let result = simplify.simplify_expr_exists(&expr); + + assert!(result.is_some()); + assert!(result.unwrap().is_false()); +} + +#[test] +fn exists_non_empty_query_not_simplified() { + let schema = test_schema(); + let simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `exists(non_empty_query)`, non-empty, not simplified + let expr = ExprExists { + subquery: non_empty_query(), + }; + let result = simplify.simplify_expr_exists(&expr); + + assert!(result.is_none()); +} + +#[test] +fn not_exists_empty_query_becomes_true() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `not(exists(empty_query))` → `not(false)` → `true` + let mut expr = Expr::not(Expr::Exists(ExprExists { + subquery: empty_query(), + })); + simplify.visit_expr_mut(&mut expr); + assert!(expr.is_true()); +} + +#[test] +fn not_exists_non_empty_query_not_simplified() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `not(exists(non_empty_query))`, non-empty, not simplified + let mut expr = Expr::not(Expr::Exists(ExprExists { + subquery: non_empty_query(), + })); + simplify.visit_expr_mut(&mut expr); + + // Should remain as `not(exists(...))` + assert!(matches!(expr, Expr::Not(_))); +} diff --git a/packages/toasty/src/engine/simplify/tests/expr_intersects.rs b/packages/toasty/src/engine/simplify/tests/expr_intersects.rs new file mode 100644 index 00000000000..ab3517b1a68 --- /dev/null +++ b/packages/toasty/src/engine/simplify/tests/expr_intersects.rs @@ -0,0 +1,43 @@ +use super::test_schema; +use crate::engine::simplify::Simplify; +use toasty_core::driver::Capability; +use toasty_core::stmt::{BinaryOp, Expr, ExprIntersects, Value}; + +fn intersects(rhs: Expr) -> ExprIntersects { + ExprIntersects { + lhs: Box::new(Expr::arg(0)), + rhs: Box::new(rhs), + } +} + +#[test] +fn rewrites_value_list_into_or_of_any_op_when_not_native() { + let schema = test_schema(); + let simplify = Simplify::new(&schema, &Capability::DYNAMODB); + + let mut expr = intersects(Expr::Value(Value::List(vec![Value::I64(1), Value::I64(2)]))); + let result = simplify + .simplify_expr_intersects(&mut expr) + .expect("expected rewrite"); + + let Expr::Or(or) = result else { + panic!("expected Or, got {result:#?}"); + }; + assert_eq!(or.operands.len(), 2); + for operand in &or.operands { + let Expr::AnyOp(any) = operand else { + panic!("expected AnyOp, got {operand:#?}"); + }; + assert!(matches!(any.op, BinaryOp::Eq)); + } +} + +#[test] +fn leaves_node_alone_when_driver_is_native() { + let schema = test_schema(); + let simplify = Simplify::new(&schema, &Capability::SQLITE); + + let mut expr = intersects(Expr::Value(Value::List(vec![Value::I64(1)]))); + let result = simplify.simplify_expr_intersects(&mut expr); + assert!(result.is_none()); +} diff --git a/packages/toasty/src/engine/simplify/tests/expr_is_null.rs b/packages/toasty/src/engine/simplify/tests/expr_is_null.rs new file mode 100644 index 00000000000..fa4e557292f --- /dev/null +++ b/packages/toasty/src/engine/simplify/tests/expr_is_null.rs @@ -0,0 +1,35 @@ +use super::test_schema_with; +use crate::engine::simplify::Simplify; +use toasty_core::stmt::{Expr, ExprIsNull, ExprReference, Value}; + +#[test] +fn is_null_non_nullable_field() { + use crate as toasty; + use crate::schema::Model; + + #[allow(dead_code)] + #[derive(toasty::Model)] + struct User { + #[key] + id: String, + emailadres: String, + } + + let schema = test_schema_with(&[User::schema()]); + let model = schema.app.model(User::id()); + let simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + let simplify = simplify.scope(model.as_root_unwrap()); + + // `is_null(field)` → `false` (non-nullable field) + let mut field = ExprIsNull { + expr: Box::new(Expr::Reference(ExprReference::Field { + nesting: 0, + index: 1, + })), + negated: false, + }; + + let result = simplify.simplify_expr_is_null(&mut field); + + assert!(matches!(result, Some(Expr::Value(Value::Bool(false))))); +} diff --git a/packages/toasty/src/engine/simplify/tests/expr_is_superset.rs b/packages/toasty/src/engine/simplify/tests/expr_is_superset.rs new file mode 100644 index 00000000000..a6e60c5f777 --- /dev/null +++ b/packages/toasty/src/engine/simplify/tests/expr_is_superset.rs @@ -0,0 +1,55 @@ +use super::test_schema; +use crate::engine::simplify::Simplify; +use toasty_core::driver::Capability; +use toasty_core::stmt::{BinaryOp, Expr, ExprIsSuperset, Value}; + +fn is_superset(rhs: Expr) -> ExprIsSuperset { + ExprIsSuperset { + lhs: Box::new(Expr::arg(0)), + rhs: Box::new(rhs), + } +} + +#[test] +fn rewrites_value_list_into_and_of_any_op_when_not_native() { + let schema = test_schema(); + let simplify = Simplify::new(&schema, &Capability::DYNAMODB); + + let mut expr = is_superset(Expr::Value(Value::List(vec![Value::I64(1), Value::I64(2)]))); + let result = simplify + .simplify_expr_is_superset(&mut expr) + .expect("expected rewrite"); + + let Expr::And(and) = result else { + panic!("expected And, got {result:#?}"); + }; + assert_eq!(and.operands.len(), 2); + for operand in &and.operands { + let Expr::AnyOp(any) = operand else { + panic!("expected AnyOp, got {operand:#?}"); + }; + assert!(matches!(any.op, BinaryOp::Eq)); + } +} + +#[test] +fn leaves_node_alone_when_driver_is_native() { + let schema = test_schema(); + let simplify = Simplify::new(&schema, &Capability::SQLITE); + + let mut expr = is_superset(Expr::Value(Value::List(vec![Value::I64(1)]))); + let result = simplify.simplify_expr_is_superset(&mut expr); + assert!(result.is_none()); +} + +#[test] +fn leaves_node_alone_when_rhs_is_not_a_value_list() { + let schema = test_schema(); + let simplify = Simplify::new(&schema, &Capability::DYNAMODB); + + // A non-list rhs cannot be expanded; the capability check in `verify` + // rejects this shape before exec, but the simplify pass must not panic. + let mut expr = is_superset(Expr::arg(1)); + let result = simplify.simplify_expr_is_superset(&mut expr); + assert!(result.is_none()); +} diff --git a/packages/toasty/src/engine/simplify/tests/expr_map.rs b/packages/toasty/src/engine/simplify/tests/expr_map.rs new file mode 100644 index 00000000000..a276150b451 --- /dev/null +++ b/packages/toasty/src/engine/simplify/tests/expr_map.rs @@ -0,0 +1,41 @@ +use super::test_schema; +use crate::engine::simplify::Simplify; +use toasty_core::stmt::{Expr, Value}; + +#[test] +fn const_base_with_identity_map() { + let schema = test_schema(); + let simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `map([1, 2, 3], x => x) → [1, 2, 3]` + let mut expr = Expr::map( + Expr::Value(Value::List(vec![ + Value::from(1i64), + Value::from(2i64), + Value::from(3i64), + ])), + Expr::arg(0), + ); + let result = simplify.simplify_expr_map(&mut expr); + + assert!(result.is_some()); + let Expr::Value(Value::List(list)) = result.unwrap() else { + panic!("expected result to be a `Value::List`"); + }; + assert_eq!(list.len(), 3); + assert_eq!(list[0], Value::from(1i64)); + assert_eq!(list[1], Value::from(2i64)); + assert_eq!(list[2], Value::from(3i64)); +} + +#[test] +fn non_const_base_not_simplified() { + let schema = test_schema(); + let simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `map(arg(0), x => x)`, non-constant base, not simplified + let mut expr = Expr::map(Expr::arg(0), Expr::arg(0)); + let result = simplify.simplify_expr_map(&mut expr); + + assert!(result.is_none()); +} diff --git a/packages/toasty/src/engine/simplify/tests/expr_match.rs b/packages/toasty/src/engine/simplify/tests/expr_match.rs new file mode 100644 index 00000000000..62334e74b7c --- /dev/null +++ b/packages/toasty/src/engine/simplify/tests/expr_match.rs @@ -0,0 +1,218 @@ +use super::test_schema; +use crate::engine::simplify::Simplify; +use toasty_core::stmt::{self, Expr, ExprMatch, MatchArm, Projection, Value, VisitMut}; + +// End-to-end `visit_expr_mut` tests. The cheap fold rules themselves are +// covered in `engine::fold::tests::expr_match`; these tests focus on the +// `visit_expr_match_mut` override that prevents the simplifier from +// recursing into dead-code arms. + +/// `visit_expr_mut` on a `Match` with a constant subject folds the whole +/// expression to the matching arm's value. +#[test] +fn constant_subject_folds_to_arm_value() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `match I64(2) { 1 => "a", 2 => "b", 3 => "c" }` → `"b"` + let mut expr = Expr::Match(ExprMatch { + subject: Box::new(Expr::from(2i64)), + arms: vec![ + MatchArm { + pattern: Value::from(1i64), + expr: Expr::from("a"), + }, + MatchArm { + pattern: Value::from(2i64), + expr: Expr::from("b"), + }, + MatchArm { + pattern: Value::from(3i64), + expr: Expr::from("c"), + }, + ], + else_expr: Box::new(Expr::null()), + }); + + simplify.visit_expr_mut(&mut expr); + + assert!(matches!(&expr, Expr::Value(Value::String(s)) if s == "b")); +} + +/// The subject expression is simplified before the fold decision is made. +/// A `project([0], record([I64(1)]))` subject should first simplify to +/// `I64(1)` and then the matching arm should be selected. +#[test] +fn subject_simplified_before_folding() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // subject = `project([0], record([I64(1)]))` which simplifies to `I64(1)` + let subject = stmt::ExprProject { + base: Box::new(Expr::record([Expr::from(1i64)])), + projection: Projection::from(0), + }; + + let mut expr = Expr::Match(ExprMatch { + subject: Box::new(Expr::Project(subject)), + arms: vec![ + MatchArm { + pattern: Value::from(1i64), + expr: Expr::from("matched"), + }, + MatchArm { + pattern: Value::from(2i64), + expr: Expr::from("other"), + }, + ], + else_expr: Box::new(Expr::null()), + }); + + simplify.visit_expr_mut(&mut expr); + + assert!(matches!(&expr, Expr::Value(Value::String(s)) if s == "matched")); +} + +/// Dead-code arms are NOT visited when the subject is constant. This is the +/// critical bug prevention: a dead arm containing an invalid projection such +/// as `project([1], record([I64(1)]))` (index out of bounds) must not be +/// simplified, otherwise the simplifier would panic. +#[test] +fn dead_arms_not_visited_with_constant_subject() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // Arm 2 (`project([1], record([I64(1)]))`) would panic if simplified + // because the record only has 1 element. Since subject is I64(1) → arm 1 + // is selected, arm 2 must be skipped entirely. + let dead_arm_expr = stmt::ExprProject { + base: Box::new(Expr::record([Expr::from(1i64)])), + projection: Projection::from(1), // index 1 into a 1-element record → OOB + }; + + let mut expr = Expr::Match(ExprMatch { + subject: Box::new(Expr::from(1i64)), + arms: vec![ + MatchArm { + pattern: Value::from(1i64), + expr: Expr::from("ok"), + }, + MatchArm { + pattern: Value::from(2i64), + expr: Expr::Project(dead_arm_expr), + }, + ], + else_expr: Box::new(Expr::null()), + }); + + // Must not panic. + simplify.visit_expr_mut(&mut expr); + + assert!(matches!(&expr, Expr::Value(Value::String(s)) if s == "ok")); +} + +/// With a non-constant subject, all arms are simplified normally. +#[test] +fn non_constant_subject_simplifies_all_arms() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // Arms contain simplifiable sub-expressions (`record([x]) → {x}` when x + // is constant). With a non-constant subject, both arms should be visited. + let mut expr = Expr::Match(ExprMatch { + subject: Box::new(Expr::arg(0)), + arms: vec![ + MatchArm { + pattern: Value::from(1i64), + // `record(["hello"])` → constant record value after simplification + expr: Expr::record([Expr::from("hello")]), + }, + MatchArm { + pattern: Value::from(2i64), + expr: Expr::record([Expr::from("world")]), + }, + ], + else_expr: Box::new(Expr::null()), + }); + + simplify.visit_expr_mut(&mut expr); + + // Match is not folded (non-constant subject), but the arm exprs are simplified. + let Expr::Match(m) = &expr else { + panic!("expected Expr::Match") + }; + assert!(matches!(&m.arms[0].expr, Expr::Value(Value::Record(_)))); + assert!(matches!(&m.arms[1].expr, Expr::Value(Value::Record(_)))); +} + +/// When the subject is constant and no arm matches, the match folds to the +/// else expression — even if that else expression is `Expr::Error`. +#[test] +fn constant_subject_no_match_folds_to_error_else() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `match 99 { 1 => "a" } else error("unexpected")` → `error("unexpected")` + let mut expr = Expr::Match(ExprMatch { + subject: Box::new(Expr::from(99i64)), + arms: vec![MatchArm { + pattern: Value::from(1i64), + expr: Expr::from("a"), + }], + else_expr: Box::new(Expr::error("unexpected")), + }); + + simplify.visit_expr_mut(&mut expr); + + assert!(matches!(&expr, Expr::Error(e) if e.message == "unexpected")); +} + +/// When the subject is constant and the matching arm body is `Expr::Error`, +/// the match folds to that error. +#[test] +fn constant_subject_matching_arm_is_error() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `match 1 { 1 => error("bad"), 2 => "ok" } else "default"` → `error("bad")` + let mut expr = Expr::Match(ExprMatch { + subject: Box::new(Expr::from(1i64)), + arms: vec![ + MatchArm { + pattern: Value::from(1i64), + expr: Expr::error("bad"), + }, + MatchArm { + pattern: Value::from(2i64), + expr: Expr::from("ok"), + }, + ], + else_expr: Box::new(Expr::from("default")), + }); + + simplify.visit_expr_mut(&mut expr); + + assert!(matches!(&expr, Expr::Error(e) if e.message == "bad")); +} + +/// When the subject is constant and a normal arm matches, the error else +/// branch is not reached. +#[test] +fn constant_subject_match_found_error_else_not_reached() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `match 1 { 1 => "ok" } else error("unexpected")` → `"ok"` + let mut expr = Expr::Match(ExprMatch { + subject: Box::new(Expr::from(1i64)), + arms: vec![MatchArm { + pattern: Value::from(1i64), + expr: Expr::from("ok"), + }], + else_expr: Box::new(Expr::error("unexpected")), + }); + + simplify.visit_expr_mut(&mut expr); + + assert!(matches!(&expr, Expr::Value(Value::String(s)) if s == "ok")); +} diff --git a/packages/toasty/src/engine/simplify/tests/expr_or.rs b/packages/toasty/src/engine/simplify/tests/expr_or.rs new file mode 100644 index 00000000000..73f06707b29 --- /dev/null +++ b/packages/toasty/src/engine/simplify/tests/expr_or.rs @@ -0,0 +1,623 @@ +use super::test_schema; +use crate::engine::simplify::Simplify; +use toasty_core::stmt::{Expr, ExprOr}; + +#[test] +fn idempotent_two_identical() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `or(a, a) → a` + let mut expr = ExprOr { + operands: vec![Expr::arg(0), Expr::arg(0)], + }; + let result = simplify.simplify_expr_or(&mut expr); + + assert!(result.is_some()); + assert_eq!(result.unwrap(), Expr::arg(0)); +} + +#[test] +fn idempotent_three_identical() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `or(a, a, a) → a` + let mut expr = ExprOr { + operands: vec![Expr::arg(0), Expr::arg(0), Expr::arg(0)], + }; + let result = simplify.simplify_expr_or(&mut expr); + + assert!(result.is_some()); + assert_eq!(result.unwrap(), Expr::arg(0)); +} + +#[test] +fn idempotent_with_different() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `or(a, b, a) → or(a, b)` + let mut expr = ExprOr { + operands: vec![Expr::arg(0), Expr::arg(1), Expr::arg(0)], + }; + let result = simplify.simplify_expr_or(&mut expr); + + assert!(result.is_none()); + assert_eq!(expr.operands.len(), 2); + assert_eq!(expr.operands[0], Expr::arg(0)); + assert_eq!(expr.operands[1], Expr::arg(1)); +} + +#[test] +fn absorption_or_and() { + use toasty_core::stmt::ExprAnd; + + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `or(a, and(a, b))` → `a` + let mut expr = ExprOr { + operands: vec![ + Expr::arg(0), + Expr::And(ExprAnd { + operands: vec![Expr::arg(0), Expr::arg(1)], + }), + ], + }; + let result = simplify.simplify_expr_or(&mut expr); + + assert!(result.is_some()); + assert_eq!(result.unwrap(), Expr::arg(0)); +} + +#[test] +fn absorption_with_multiple_operands() { + use toasty_core::stmt::ExprAnd; + + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `or(a, b, and(a, c))` → `or(a, b)` + let mut expr = ExprOr { + operands: vec![ + Expr::arg(0), + Expr::arg(1), + Expr::And(ExprAnd { + operands: vec![Expr::arg(0), Expr::arg(2)], + }), + ], + }; + let result = simplify.simplify_expr_or(&mut expr); + + assert!(result.is_none()); + assert_eq!(expr.operands.len(), 2); + assert_eq!(expr.operands[0], Expr::arg(0)); + assert_eq!(expr.operands[1], Expr::arg(1)); +} + +#[test] +fn absorption_two_or_three_and() { + use toasty_core::stmt::ExprAnd; + + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `or(a, b, and(a, c, d))` → `or(a, b)` + let mut expr = ExprOr { + operands: vec![ + Expr::arg(0), + Expr::arg(1), + Expr::And(ExprAnd { + operands: vec![Expr::arg(0), Expr::arg(2), Expr::arg(3)], + }), + ], + }; + let result = simplify.simplify_expr_or(&mut expr); + + assert!(result.is_none()); + assert_eq!(expr.operands.len(), 2); + assert_eq!(expr.operands[0], Expr::arg(0)); + assert_eq!(expr.operands[1], Expr::arg(1)); +} + +#[test] +fn factoring_basic() { + use toasty_core::stmt::ExprAnd; + + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `(a and b) or (a and c)` → `a and (b or c)` + let mut expr = ExprOr { + operands: vec![ + Expr::And(ExprAnd { + operands: vec![Expr::arg(0), Expr::arg(1)], + }), + Expr::And(ExprAnd { + operands: vec![Expr::arg(0), Expr::arg(2)], + }), + ], + }; + let result = simplify.simplify_expr_or(&mut expr); + + // Result should be `a and (b or c)` + let Some(Expr::And(and_expr)) = result else { + panic!("expected And"); + }; + assert_eq!(and_expr.operands.len(), 2); + assert_eq!(and_expr.operands[0], Expr::arg(0)); + + let Expr::Or(or_expr) = &and_expr.operands[1] else { + panic!("expected Or"); + }; + assert_eq!(or_expr.operands.len(), 2); + assert_eq!(or_expr.operands[0], Expr::arg(1)); + assert_eq!(or_expr.operands[1], Expr::arg(2)); +} + +#[test] +fn factoring_multiple_common() { + use toasty_core::stmt::ExprAnd; + + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `(a and b and c) or (a and b and d)` → `a and b and (c or d)` + let mut expr = ExprOr { + operands: vec![ + Expr::And(ExprAnd { + operands: vec![Expr::arg(0), Expr::arg(1), Expr::arg(2)], + }), + Expr::And(ExprAnd { + operands: vec![Expr::arg(0), Expr::arg(1), Expr::arg(3)], + }), + ], + }; + let result = simplify.simplify_expr_or(&mut expr); + + // Result should be `a and b and (c or d)` + let Some(Expr::And(and_expr)) = result else { + panic!("expected And"); + }; + assert_eq!(and_expr.operands.len(), 3); + assert_eq!(and_expr.operands[0], Expr::arg(0)); + assert_eq!(and_expr.operands[1], Expr::arg(1)); + + let Expr::Or(or_expr) = &and_expr.operands[2] else { + panic!("expected Or"); + }; + assert_eq!(or_expr.operands.len(), 2); + assert_eq!(or_expr.operands[0], Expr::arg(2)); + assert_eq!(or_expr.operands[1], Expr::arg(3)); +} + +#[test] +fn factoring_three_ands() { + use toasty_core::stmt::ExprAnd; + + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `(a and b) or (a and c) or (a and d)` → `a and (b or c or d)` + let mut expr = ExprOr { + operands: vec![ + Expr::And(ExprAnd { + operands: vec![Expr::arg(0), Expr::arg(1)], + }), + Expr::And(ExprAnd { + operands: vec![Expr::arg(0), Expr::arg(2)], + }), + Expr::And(ExprAnd { + operands: vec![Expr::arg(0), Expr::arg(3)], + }), + ], + }; + let result = simplify.simplify_expr_or(&mut expr); + + // Result should be `a and (b or c or d)` + let Some(Expr::And(and_expr)) = result else { + panic!("expected And"); + }; + assert_eq!(and_expr.operands.len(), 2); + assert_eq!(and_expr.operands[0], Expr::arg(0)); + + let Expr::Or(or_expr) = &and_expr.operands[1] else { + panic!("expected Or"); + }; + assert_eq!(or_expr.operands.len(), 3); + assert_eq!(or_expr.operands[0], Expr::arg(1)); + assert_eq!(or_expr.operands[1], Expr::arg(2)); + assert_eq!(or_expr.operands[2], Expr::arg(3)); +} + +#[test] +fn factoring_no_common() { + use toasty_core::stmt::ExprAnd; + + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `(a and b) or (c and d)` → no change (no common factor) + let mut expr = ExprOr { + operands: vec![ + Expr::And(ExprAnd { + operands: vec![Expr::arg(0), Expr::arg(1)], + }), + Expr::And(ExprAnd { + operands: vec![Expr::arg(2), Expr::arg(3)], + }), + ], + }; + let result = simplify.simplify_expr_or(&mut expr); + + assert!(result.is_none()); +} + +#[test] +fn complement_basic() { + use toasty_core::stmt::ExprNot; + + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `a or not(a)` → `true` (where a is a non-nullable comparison) + let a = Expr::eq(Expr::arg(0), Expr::arg(1)); + let mut expr = ExprOr { + operands: vec![a.clone(), Expr::Not(ExprNot { expr: Box::new(a) })], + }; + let result = simplify.simplify_expr_or(&mut expr); + + assert!(result.is_some()); + assert!(result.unwrap().is_true()); +} + +#[test] +fn complement_with_other_operands() { + use toasty_core::stmt::ExprNot; + + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `a or b or not(a)` → `true` + let a = Expr::eq(Expr::arg(0), Expr::arg(1)); + let mut expr = ExprOr { + operands: vec![ + a.clone(), + Expr::arg(2), + Expr::Not(ExprNot { expr: Box::new(a) }), + ], + }; + let result = simplify.simplify_expr_or(&mut expr); + + assert!(result.is_some()); + assert!(result.unwrap().is_true()); +} + +#[test] +fn complement_nullable_not_simplified() { + use toasty_core::stmt::ExprNot; + + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `a or not(a)` where `a` is an arg (nullable) → no change + let a = Expr::arg(0); + let mut expr = ExprOr { + operands: vec![a.clone(), Expr::Not(ExprNot { expr: Box::new(a) })], + }; + let result = simplify.simplify_expr_or(&mut expr); + + assert!(result.is_none()); +} + +#[test] +fn complement_multiple_repetitions() { + use toasty_core::stmt::ExprNot; + + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `a or a or not(a) or not(a)` → `true` + let a = Expr::eq(Expr::arg(0), Expr::arg(1)); + let mut expr = ExprOr { + operands: vec![ + a.clone(), + a.clone(), + Expr::Not(ExprNot { + expr: Box::new(a.clone()), + }), + Expr::Not(ExprNot { expr: Box::new(a) }), + ], + }; + let result = simplify.simplify_expr_or(&mut expr); + + assert!(result.is_some()); + assert!(result.unwrap().is_true()); +} + +// OR-to-IN conversion tests + +#[test] +fn or_to_in_basic() { + use toasty_core::stmt::Value; + + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `a = 1 or a = 2 or a = 3` → `a in (1, 2, 3)` + let mut expr = ExprOr { + operands: vec![ + Expr::eq(Expr::arg(0), Expr::Value(Value::from(1i64))), + Expr::eq(Expr::arg(0), Expr::Value(Value::from(2i64))), + Expr::eq(Expr::arg(0), Expr::Value(Value::from(3i64))), + ], + }; + let result = simplify.simplify_expr_or(&mut expr); + + assert!(result.is_some()); + let Some(Expr::InList(in_list)) = result else { + panic!("expected InList"); + }; + assert_eq!(*in_list.expr, Expr::arg(0)); +} + +#[test] +fn or_to_in_two_values() { + use toasty_core::stmt::Value; + + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `a = 1 or a = 2` → `a in (1, 2)` + let mut expr = ExprOr { + operands: vec![ + Expr::eq(Expr::arg(0), Expr::Value(Value::from(1i64))), + Expr::eq(Expr::arg(0), Expr::Value(Value::from(2i64))), + ], + }; + let result = simplify.simplify_expr_or(&mut expr); + + assert!(result.is_some()); + assert!(matches!(result, Some(Expr::InList(_)))); +} + +#[test] +fn or_to_in_single_value_not_converted() { + use toasty_core::stmt::Value; + + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `a = 1` (single equality, not converted) + let mut expr = ExprOr { + operands: vec![Expr::eq(Expr::arg(0), Expr::Value(Value::from(1i64)))], + }; + let result = simplify.simplify_expr_or(&mut expr); + + // Single operand gets unwrapped, but not to IN + assert!(result.is_some()); + assert!(matches!(result, Some(Expr::BinaryOp(_)))); +} + +#[test] +fn or_to_in_different_lhs_not_converted() { + use toasty_core::stmt::Value; + + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `a = 1 or b = 2` (different LHS, not converted to single IN) + let mut expr = ExprOr { + operands: vec![ + Expr::eq(Expr::arg(0), Expr::Value(Value::from(1i64))), + Expr::eq(Expr::arg(1), Expr::Value(Value::from(2i64))), + ], + }; + let result = simplify.simplify_expr_or(&mut expr); + + // Not converted, stays as OR + assert!(result.is_none()); + assert_eq!(expr.operands.len(), 2); +} + +#[test] +fn or_to_in_mixed_keeps_other_operands() { + use toasty_core::stmt::Value; + + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `a = 1 or a = 2 or b = 3` → `a in (1, 2) or b = 3` + let mut expr = ExprOr { + operands: vec![ + Expr::eq(Expr::arg(0), Expr::Value(Value::from(1i64))), + Expr::eq(Expr::arg(0), Expr::Value(Value::from(2i64))), + Expr::eq(Expr::arg(1), Expr::Value(Value::from(3i64))), + ], + }; + let result = simplify.simplify_expr_or(&mut expr); + + // Stays as OR but with transformed operands + assert!(result.is_none()); + assert_eq!(expr.operands.len(), 2); + + // Should have one InList and one BinaryOp + let has_in_list = expr.operands.iter().any(|e| matches!(e, Expr::InList(_))); + let has_binary_op = expr.operands.iter().any(|e| matches!(e, Expr::BinaryOp(_))); + assert!(has_in_list); + assert!(has_binary_op); +} + +#[test] +fn or_to_in_multiple_groups() { + use toasty_core::stmt::Value; + + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `a = 1 or a = 2 or b = 3 or b = 4` → `a in (1, 2) or b in (3, 4)` + let mut expr = ExprOr { + operands: vec![ + Expr::eq(Expr::arg(0), Expr::Value(Value::from(1i64))), + Expr::eq(Expr::arg(0), Expr::Value(Value::from(2i64))), + Expr::eq(Expr::arg(1), Expr::Value(Value::from(3i64))), + Expr::eq(Expr::arg(1), Expr::Value(Value::from(4i64))), + ], + }; + let result = simplify.simplify_expr_or(&mut expr); + + // Stays as OR with two InList operands + assert!(result.is_none()); + assert_eq!(expr.operands.len(), 2); + + // Both should be InList + assert!(expr.operands.iter().all(|e| matches!(e, Expr::InList(_)))); +} + +#[test] +fn or_to_in_with_non_equality_preserved() { + use toasty_core::stmt::Value; + + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `a = 1 or a = 2 or c` → `a in (1, 2) or c` + let mut expr = ExprOr { + operands: vec![ + Expr::eq(Expr::arg(0), Expr::Value(Value::from(1i64))), + Expr::eq(Expr::arg(0), Expr::Value(Value::from(2i64))), + Expr::arg(2), // non-equality operand + ], + }; + let result = simplify.simplify_expr_or(&mut expr); + + // Stays as OR + assert!(result.is_none()); + assert_eq!(expr.operands.len(), 2); + + // One InList, one arg + let has_in_list = expr.operands.iter().any(|e| matches!(e, Expr::InList(_))); + let has_arg = expr.operands.iter().any(|e| matches!(e, Expr::Arg(_))); + assert!(has_in_list); + assert!(has_arg); +} + +#[test] +fn or_to_in_non_const_rhs_not_converted() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `a = b or a = c` (non-constant RHS, not converted) + let mut expr = ExprOr { + operands: vec![ + Expr::eq(Expr::arg(0), Expr::arg(1)), + Expr::eq(Expr::arg(0), Expr::arg(2)), + ], + }; + let result = simplify.simplify_expr_or(&mut expr); + + // Not converted, stays as OR + assert!(result.is_none()); + assert_eq!(expr.operands.len(), 2); + assert!(expr.operands.iter().all(|e| matches!(e, Expr::BinaryOp(_)))); +} + +// --------------------------------------------------------------------------- +// Determinism-aware simplification (issue #236). +// +// See the matching block in `expr_and.rs`. `is_equivalent_to` gates the +// idempotent, absorption, complement, factoring, and OR-to-IN rewrites on +// `is_stable`, so non-deterministic sub-expressions survive intact. +// --------------------------------------------------------------------------- + +#[test] +fn idempotent_not_simplified_for_non_deterministic() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `LAST_INSERT_ID() OR LAST_INSERT_ID()` must retain both operands. + let mut expr = ExprOr { + operands: vec![Expr::last_insert_id(), Expr::last_insert_id()], + }; + let result = simplify.simplify_expr_or(&mut expr); + + assert!(result.is_none()); + assert_eq!(expr.operands.len(), 2); +} + +#[test] +fn complement_not_simplified_for_non_deterministic() { + use toasty_core::stmt::ExprNot; + + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `f() = 1 OR NOT (f() = 1)` — both draws are independent, so the + // complement law must NOT fire. + let a = Expr::eq(Expr::last_insert_id(), 1i64); + let mut expr = ExprOr { + operands: vec![a.clone(), Expr::Not(ExprNot { expr: Box::new(a) })], + }; + let result = simplify.simplify_expr_or(&mut expr); + + assert!(result.is_none()); + assert_eq!(expr.operands.len(), 2); +} + +#[test] +fn factoring_not_simplified_for_non_deterministic() { + use toasty_core::stmt::ExprAnd; + + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `(f() AND b) OR (f() AND c)` would factor to `f() AND (b OR c)` under + // PartialEq, but the original evaluates `f()` twice while the factored + // form evaluates it once — unsound for non-deterministic `f()`. + let mut expr = ExprOr { + operands: vec![ + Expr::And(ExprAnd { + operands: vec![Expr::last_insert_id(), Expr::arg(0)], + }), + Expr::And(ExprAnd { + operands: vec![Expr::last_insert_id(), Expr::arg(1)], + }), + ], + }; + let result = simplify.simplify_expr_or(&mut expr); + + // No factoring should have taken place — we still have two AND branches, + // each with the original two operands. + assert!(result.is_none()); + assert_eq!(expr.operands.len(), 2); + for operand in &expr.operands { + let Expr::And(and) = operand else { + panic!("expected AND branch, got {operand:?}"); + }; + assert_eq!(and.operands.len(), 2); + } +} + +#[test] +fn or_to_in_list_not_simplified_for_non_deterministic() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `f() = 1 OR f() = 2` must NOT be rewritten to `f() IN (1, 2)`: the + // former evaluates `f()` twice (two draws against 1 and 2), the latter + // evaluates it once (one draw tested against a set). + let mut expr = ExprOr { + operands: vec![ + Expr::eq(Expr::last_insert_id(), 1i64), + Expr::eq(Expr::last_insert_id(), 2i64), + ], + }; + let result = simplify.simplify_expr_or(&mut expr); + + assert!(result.is_none()); + assert_eq!(expr.operands.len(), 2); + for operand in &expr.operands { + assert!(matches!(operand, Expr::BinaryOp(_))); + } +} diff --git a/packages/toasty/src/engine/simplify/tests/expr_project.rs b/packages/toasty/src/engine/simplify/tests/expr_project.rs new file mode 100644 index 00000000000..e71ea81eb97 --- /dev/null +++ b/packages/toasty/src/engine/simplify/tests/expr_project.rs @@ -0,0 +1,80 @@ +use super::test_schema; +use crate::engine::simplify::Simplify; +use toasty_core::stmt::{self, Expr, ExprMatch, MatchArm, Projection, Value}; + +#[test] +fn project_non_constant_not_simplified() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `project(arg(0), [0])` is not simplified (non-constant base) + let mut expr = stmt::ExprProject { + base: Box::new(Expr::arg(0)), + projection: Projection::from(0), + }; + + let result = simplify.simplify_expr_project(&mut expr); + + assert!(result.is_none()); +} + +#[test] +fn project_identity_path() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `project(42, [])` → `42` (identity projection) + let mut expr = stmt::ExprProject { + base: Box::new(Expr::from(42i64)), + projection: Projection::identity(), + }; + + let result = simplify.simplify_expr_project(&mut expr); + + assert!(matches!(result, Some(Expr::Value(Value::I64(42))))); +} + +#[test] +fn project_into_match_distributes() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // project(Match(arg(0), [1 => Record([arg(0), arg(1)]), 2 => Record([arg(0), arg(2)])], + // else: Record([arg(0), Error])), [0]) + // → Match(arg(0), [1 => project(Record([arg(0), arg(1)]), [0]), + // 2 => project(Record([arg(0), arg(2)]), [0])], + // else: project(Record([arg(0), Error]), [0])) + let mut expr = stmt::ExprProject { + base: Box::new(Expr::Match(ExprMatch { + subject: Box::new(Expr::arg(0)), + arms: vec![ + MatchArm { + pattern: Value::from(1i64), + expr: Expr::record([Expr::arg(0), Expr::arg(1)]), + }, + MatchArm { + pattern: Value::from(2i64), + expr: Expr::record([Expr::arg(0), Expr::arg(2)]), + }, + ], + else_expr: Box::new(Expr::record([Expr::arg(0), Expr::error("unreachable")])), + })), + projection: Projection::from(0), + }; + + let result = simplify.simplify_expr_project(&mut expr); + + // The result should be a Match with projected arms and else + assert!(result.is_some()); + let result = result.unwrap(); + if let Expr::Match(m) = result { + assert_eq!(m.arms.len(), 2); + // Each arm should now be project(Record(...), [0]) + assert!(m.arms[0].expr.is_project()); + assert!(m.arms[1].expr.is_project()); + // Else should also be projected + assert!(m.else_expr.is_project()); + } else { + panic!("expected Match, got {:?}", result); + } +} diff --git a/packages/toasty/src/engine/simplify/tests/merge_in_subqueries.rs b/packages/toasty/src/engine/simplify/tests/merge_in_subqueries.rs new file mode 100644 index 00000000000..0b3fda9989e --- /dev/null +++ b/packages/toasty/src/engine/simplify/tests/merge_in_subqueries.rs @@ -0,0 +1,447 @@ +use super::test_schema_with; +use crate as toasty; +use crate::{engine::simplify::Simplify, schema::Model}; +use toasty_core::{ + driver::Capability, + schema::app::ModelId, + stmt::{self, Expr, ExprContext, Query, Returning, VisitMut}, +}; + +#[allow(dead_code)] +#[derive(toasty::Model)] +struct User { + #[key] + id: i64, + name: String, + active: bool, + #[unique] + email: String, + #[unique] + nullable_email: Option, + #[has_many] + posts: toasty::Deferred>, +} + +#[allow(dead_code)] +#[derive(toasty::Model)] +struct Post { + #[key] + id: i64, + #[index] + user_id: i64, + #[belongs_to(key = user_id)] + user: toasty::Deferred, + optional_key: Option, + other_key: i64, +} + +#[allow(dead_code)] +#[derive(toasty::Model)] +#[key(partition = tenant, local = id)] +struct Item { + tenant: String, + id: i64, + value: String, +} + +fn schema() -> toasty_core::Schema { + test_schema_with(&[User::schema(), Post::schema(), Item::schema()]) +} + +fn field(index: usize) -> Expr { + Expr::ref_self_field(M::id().field(index)) +} + +fn query(model: ModelId, projection: Expr, filter: Expr) -> Query { + let mut query = Query::new_select(model, filter); + query.body.as_select_mut_unwrap().returning = Returning::Project(projection); + query +} + +fn users(filter: Expr) -> Query { + query(User::id(), field::(0), filter) +} + +fn membership(filter: Expr) -> Expr { + Expr::in_subquery(field::(1), users(filter)) +} + +fn name() -> Expr { + Expr::eq(field::(1), "alice") +} + +fn active() -> Expr { + field::(2) +} + +fn simplify_filter(expr: Expr) -> Expr { + let schema = schema(); + let mut query = Query::new_select(Post::id(), expr); + Simplify::new(&schema, &Capability::SQLITE).visit_stmt_query_mut(&mut query); + query.body.as_select_mut_unwrap().filter.take().into_expr() +} + +#[test] +fn combines_explicit_subqueries() { + assert_eq!( + simplify_filter(Expr::and(membership(name()), membership(active()))), + membership(Expr::and(name(), active())) + ); +} + +#[test] +fn combines_nonadjacent_operands_and_multiple_groups() { + let other = |filter| Expr::in_subquery(field::(4), users(filter)); + let unrelated = Expr::eq(field::(0), 42); + let input = stmt::ExprAnd { + operands: vec![ + membership(name()), + other(name()), + unrelated.clone(), + membership(active()), + other(active()), + membership(Expr::eq(field::(3), "a@b")), + ], + } + .into(); + assert_eq!( + simplify_filter(input), + Expr::from(stmt::ExprAnd { + operands: vec![ + membership(Expr::and( + Expr::and(name(), active()), + Expr::eq(field::(3), "a@b") + )), + other(Expr::and(name(), active())), + unrelated + ], + }) + ); +} + +#[test] +fn combines_compound_filters() { + let compound = Expr::or(name(), active()); + let email = Expr::eq(field::(3), "a@b"); + assert_eq!( + simplify_filter(Expr::and(membership(compound), membership(email.clone()))), + membership(Expr::and(email, Expr::or(active(), name()))) + ); +} + +#[test] +fn combines_filters_containing_nested_subqueries() { + let nested = Expr::in_subquery( + field::(0), + query(Post::id(), field::(1), Expr::eq(field::(0), 42)), + ); + assert_eq!( + simplify_filter(Expr::and(membership(name()), membership(nested.clone()))), + membership(Expr::and(name(), nested)) + ); +} + +#[test] +fn simplifies_newly_combined_filters_recursively() { + let nested = |filter| Expr::in_subquery(field::(0), users(filter)); + assert_eq!( + simplify_filter(Expr::and( + membership(nested(name())), + membership(nested(active())) + )), + membership(nested(Expr::and(name(), active()))) + ); +} + +#[test] +fn combines_unique_secondary_key() { + let by_email = |filter| Expr::in_subquery("a@b", query(User::id(), field::(3), filter)); + assert_eq!( + simplify_filter(Expr::and(by_email(name()), by_email(active()))), + by_email(Expr::and(name(), active())) + ); +} + +#[test] +fn combines_complete_composite_key_but_not_partial_key() { + let a = Expr::eq(field::(2), "a"); + let b = Expr::ne(field::(2), "b"); + let key = Expr::record([field::(0), field::(1)]); + let by_key = |filter| { + Expr::in_subquery( + Expr::Value(stmt::Value::Record(stmt::ValueRecord::from_vec(vec![ + "tenant".into(), + 1.into(), + ]))), + query(Item::id(), key.clone(), filter), + ) + }; + assert_eq!( + simplify_filter(Expr::and(by_key(a.clone()), by_key(b.clone()))), + by_key(Expr::and(a.clone(), b.clone())) + ); + + let partial = |filter| Expr::in_subquery(1, query(Item::id(), field::(1), filter)); + let input = Expr::and(partial(a), partial(b)); + assert_eq!(simplify_filter(input.clone()), input); +} + +#[test] +fn preserves_nonunique_and_nullable_projections() { + for key in [field::(1), field::(4)] { + let by_key = |filter| Expr::in_subquery("alice", query(User::id(), key.clone(), filter)); + let input = Expr::and(by_key(name()), by_key(active())); + assert_eq!(simplify_filter(input.clone()), input); + } +} + +#[test] +fn preserves_has_many_memberships() { + let child = |id| { + Expr::in_subquery( + field::(0), + query(Post::id(), field::(1), Expr::eq(field::(0), id)), + ) + }; + let input = Expr::and(child(1), child(2)); + let mut outer = Query::new_select(User::id(), input.clone()); + Simplify::new(&schema(), &Capability::SQLITE).visit_stmt_query_mut(&mut outer); + assert_eq!(outer.body.as_select_unwrap().filter.as_expr(), &input); +} + +#[test] +fn preserves_different_sources_projections_and_operands() { + let others = [ + Expr::in_subquery(field::(4), users(active())), + Expr::in_subquery( + field::(1), + query(Post::id(), field::(0), Expr::eq(field::(4), 1)), + ), + Expr::in_subquery( + field::(1), + query(User::id(), field::(3), active()), + ), + ]; + for other in others { + let input = Expr::and(membership(name()), other); + assert_eq!(simplify_filter(input.clone()), input); + } +} + +#[test] +fn preserves_query_modifiers_and_negated_membership() { + let base = users(active()); + let mut limited = base.clone(); + limited.limit = Some(stmt::Limit::Offset(stmt::LimitOffset { + limit: 1.into(), + offset: None, + })); + let mut locked = base.clone(); + locked.locks.push(stmt::Lock::Update); + let mut single = base.clone(); + single.single = true; + let others = [ + Expr::in_subquery(field::(1), limited), + Expr::in_subquery(field::(1), locked), + Expr::in_subquery(field::(1), single), + Expr::not_in_subquery(field::(1), base), + ]; + for other in others { + let input = Expr::and(membership(name()), other); + assert_eq!(simplify_filter(input.clone()), input); + } +} + +#[test] +fn preserves_unstable_and_correlated_filters() { + for filter in [ + Expr::eq(field::(0), Expr::last_insert_id()), + Expr::eq(field::(0), Expr::ref_field(1, Post::id().field(0))), + ] { + let input = Expr::and(membership(name()), membership(filter)); + assert_eq!(simplify_filter(input.clone()), input); + } + let unstable = |filter| Expr::in_subquery(Expr::last_insert_id(), users(filter)); + let input = Expr::and(unstable(name()), unstable(active())); + assert_eq!(simplify_filter(input.clone()), input); +} + +#[test] +fn nullable_operand_merges_only_in_positive_filters() { + let nullable = |filter| Expr::in_subquery(field::(3), users(filter)); + let input = Expr::and(nullable(name()), nullable(active())); + assert_eq!( + simplify_filter(input.clone()), + nullable(Expr::and(name(), active())) + ); + + let schema = schema(); + let source = stmt::Source::from(Post::id()); + let mut value = input.clone(); + Simplify::with_context( + ExprContext::new(&schema).scope(&source), + &Capability::SQLITE, + ) + .visit_expr_mut(&mut value); + assert_eq!(value, input); + assert_eq!( + simplify_filter(Expr::is_null(input.clone())), + Expr::is_null(input.clone()) + ); + // Folding distributes NOT; the memberships must still be separate. + assert_eq!( + simplify_filter(Expr::not(input)), + Expr::or(Expr::not(nullable(name())), Expr::not(nullable(active()))) + ); +} + +#[test] +fn nonnullable_operand_merges_as_a_value() { + let schema = schema(); + let source = stmt::Source::from(Post::id()); + let mut input = Expr::and(membership(name()), membership(active())); + Simplify::with_context( + ExprContext::new(&schema).scope(&source), + &Capability::SQLITE, + ) + .visit_expr_mut(&mut input); + assert_eq!(input, membership(Expr::and(name(), active()))); +} + +#[test] +fn nosql_nullable_membership_preserves_boolean_value_semantics() { + let schema = schema(); + let source = stmt::Source::from(Post::id()); + let nullable = |filter| Expr::in_subquery(field::(3), users(filter)); + let mut input = Expr::and(nullable(name()), nullable(active())); + Simplify::with_context( + ExprContext::new(&schema).scope(&source), + &Capability::DYNAMODB, + ) + .visit_expr_mut(&mut input); + assert_eq!(input, nullable(Expr::and(name(), active()))); +} + +#[test] +fn merges_mixed_relation_path_and_explicit_subquery_before_extraction() { + let path = Expr::project(field::(2), [1]); + let input = Expr::and(Expr::eq(path, "alice"), membership(active())); + let expected = membership(Expr::and(name(), active())); + let engine = crate::engine::Engine::new(schema().into(), &Capability::SQLITE); + let actual = engine + .lower_stmt(Query::new_select(Post::id(), input).into()) + .unwrap(); + let expected = engine + .lower_stmt(Query::new_select(Post::id(), expected).into()) + .unwrap(); + assert_eq!(actual.root().stmt(), expected.root().stmt()); +} + +#[test] +fn combines_before_nosql_dependency_extraction() { + let capability = &Capability::DYNAMODB; + let app = toasty_core::schema::app::Schema::from_macro([ + User::schema(), + Post::schema(), + Item::schema(), + ]) + .unwrap(); + let schema = toasty_core::schema::Builder::new() + .build(app, capability) + .unwrap(); + let engine = crate::engine::Engine::new(schema.into(), capability); + let path = Expr::project(field::(2), [1]); + let input = Expr::and(Expr::eq(path, "alice"), membership(active())); + let hir = engine + .lower_stmt(Query::new_select(Post::id(), input).into()) + .unwrap(); + assert_eq!(hir.statements().count(), 2); + assert_eq!(hir.root().args.len(), 1); + engine.plan_hir_statement(hir).unwrap(); +} + +#[test] +fn combines_lowered_table_subqueries() { + let schema = schema(); + let user_table = schema.mapping.model(User::id()).table; + let post_table = schema.mapping.model(Post::id()).table; + let column = |index| { + Expr::Reference(stmt::ExprReference::Column(stmt::ExprColumn { + nesting: 0, + table: 0, + column: index, + })) + }; + let name = Expr::eq(column(1), "alice"); + let active = column(2); + let membership = |filter| { + let mut query = Query::new_select(user_table, filter); + query.body.as_select_mut_unwrap().returning = Returning::Project(column(0)); + Expr::in_subquery(column(1), query) + }; + let mut outer = Query::new_select( + post_table, + Expr::and(membership(name.clone()), membership(active.clone())), + ); + Simplify::new(&schema, &Capability::SQLITE).visit_stmt_query_mut(&mut outer); + assert_eq!( + outer.body.as_select_unwrap().filter.as_expr(), + &membership(Expr::and(name, active)) + ); +} + +#[test] +fn nullable_filter_context_propagates_through_or_but_not_returning() { + let nullable = |filter| Expr::in_subquery(field::(3), users(filter)); + let input = Expr::and(nullable(name()), nullable(active())); + let unrelated = Expr::eq(field::(0), 42); + assert_eq!( + simplify_filter(Expr::or(input.clone(), unrelated.clone())), + Expr::or(nullable(Expr::and(name(), active())), unrelated) + ); + + let mut outer = query(Post::id(), input.clone(), input.clone()); + Simplify::new(&schema(), &Capability::SQLITE).visit_stmt_query_mut(&mut outer); + assert_eq!( + outer.body.as_select_unwrap().returning, + Returning::Project(input) + ); +} + +#[test] +fn preserves_unstable_and_locked_nested_queries() { + let mut locked = users(name()); + locked.locks.push(stmt::Lock::Update); + let unstable = users(Expr::eq(field::(0), Expr::last_insert_id())); + for nested in [locked, unstable] { + let input = Expr::and( + membership(name()), + membership(Expr::in_subquery(field::(0), nested)), + ); + assert_eq!(simplify_filter(input.clone()), input); + } +} + +#[test] +fn preserves_ordering_and_ctes() { + let mut ordered = users(active()); + ordered.order_by = Some(stmt::OrderBy { + exprs: vec![stmt::OrderByExpr { + expr: field::(0), + order: Some(stmt::Direction::Asc), + }], + }); + let mut with = users(active()); + with.with = Some(stmt::With { + ctes: vec![stmt::Cte { + query: users(name()), + }], + }); + for query in [ordered, with] { + let input = Expr::and( + membership(name()), + Expr::in_subquery(field::(1), query), + ); + assert_eq!(simplify_filter(input.clone()), input); + } +} diff --git a/packages/toasty/src/engine/simplify/tests/prop_const.rs b/packages/toasty/src/engine/simplify/tests/prop_const.rs new file mode 100644 index 00000000000..7a5886deb5a --- /dev/null +++ b/packages/toasty/src/engine/simplify/tests/prop_const.rs @@ -0,0 +1,103 @@ +use super::test_schema; +use crate::engine::simplify::Simplify; +use proptest::prelude::*; +use toasty_core::stmt::{ + BinaryOp, Expr, ExprAny, ExprBinaryOp, ExprInList, ExprList, Value, VisitMut, +}; + +/// Generates a leaf integer expression in the range -100..=100. +fn arb_i64_value() -> impl Strategy { + (-100i64..=100).prop_map(|n| Expr::Value(Value::I64(n))) +} + +/// Generates a raw i64 in the range -100..=100. +fn arb_i64_raw() -> impl Strategy { + -100i64..=100 +} + +/// Generates one of the six comparison operators. +fn arb_binary_op() -> impl Strategy { + prop_oneof![ + Just(BinaryOp::Eq), + Just(BinaryOp::Ne), + Just(BinaryOp::Lt), + Just(BinaryOp::Le), + Just(BinaryOp::Gt), + Just(BinaryOp::Ge), + ] +} + +/// Generates a pure constant boolean expression that can be evaluated +/// without a database or schema via `eval_const()`. +/// +/// Leaf cases are always evaluable and never produce null in a boolean +/// position. Recursive cases combine boolean sub-expressions. +fn arb_bool_expr() -> impl Strategy { + let leaf = prop_oneof![ + // Literal true/false + Just(Expr::from(true)), + Just(Expr::from(false)), + // BinaryOp(i64, op, i64) → bool + (arb_i64_value(), arb_binary_op(), arb_i64_value()).prop_map(|(lhs, op, rhs)| { + Expr::BinaryOp(ExprBinaryOp { + lhs: Box::new(lhs), + op, + rhs: Box::new(rhs), + }) + }), + // IsNull(i64) → always false, IsNull(null) → always true + arb_i64_value().prop_map(Expr::is_null), + Just(Expr::is_null(Expr::null())), + // InList(i64, [i64; 0..=3]) → bool, exercises empty/single/multi list paths + (arb_i64_value(), prop::collection::vec(arb_i64_raw(), 0..=3)).prop_map( + |(expr, values)| { + let list = Expr::Value(Value::List(values.into_iter().map(Value::I64).collect())); + Expr::InList(ExprInList { + expr: Box::new(expr), + list: Box::new(list), + }) + } + ), + ]; + + leaf.prop_recursive( + 4, // max recursion depth + 64, // max total nodes + 4, // max items per collection + |inner| { + prop_oneof![ + // Not(bool_expr) — double negation, De Morgan's, constant folding + inner.clone().prop_map(Expr::not), + // And([bool_expr; 2..=4]) — flattening, identity, contradiction + prop::collection::vec(inner.clone(), 2..=4).prop_map(Expr::and_from_vec), + // Or([bool_expr; 2..=4]) — flattening, OR-to-IN conversion + prop::collection::vec(inner.clone(), 2..=4).prop_map(Expr::or_from_vec), + // Any(List([bool_expr; 1..=4])) — Any+List simplifiers + prop::collection::vec(inner.clone(), 1..=4).prop_map(|items| { + Expr::Any(ExprAny { + expr: Box::new(Expr::List(ExprList { items })), + }) + }), + // BinaryOp(bool_expr, Eq/Ne, bool_literal) → bool_expr or not(bool_expr) + (inner.clone(), any::(), any::()).prop_map(|(expr, b, use_eq)| { + let op = if use_eq { BinaryOp::Eq } else { BinaryOp::Ne }; + Expr::binary_op(expr, op, Expr::from(b)) + }), + ] + }, + ) +} + +proptest! { + #[test] + fn simplify_preserves_eval(expr in arb_bool_expr()) { + let schema = test_schema(); + let oracle = expr.eval_const().expect("generated expr must be evaluable"); + + let mut expr = expr; + Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE).visit_expr_mut(&mut expr); + + let result = expr.eval_const().expect("simplified expr must be evaluable"); + prop_assert_eq!(oracle, result); + } +} diff --git a/packages/toasty/src/engine/simplify/tests/stmt_query.rs b/packages/toasty/src/engine/simplify/tests/stmt_query.rs new file mode 100644 index 00000000000..cc05beface1 --- /dev/null +++ b/packages/toasty/src/engine/simplify/tests/stmt_query.rs @@ -0,0 +1,89 @@ +use super::test_schema; +use crate::engine::simplify::Simplify; +use toasty_core::stmt::{self, Direction, Limit, LimitOffset, OrderBy, OrderByExpr, Query, Values}; + +#[test] +fn null_filter_query_is_empty() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + let mut query = Query::new_select( + stmt::Source::table(toasty_core::schema::db::TableId(0)), + stmt::Expr::null(), + ); + simplify.simplify_stmt_query_when_empty(&mut query); + assert!(matches!(query.body, stmt::ExprSet::Values(values) if values.is_empty())); +} + +#[test] +fn empty_values_query_is_empty() { + let schema = test_schema(); + let simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `stmt_query_is_empty(values([])) → true` + let query = Query::values(Values::default()); + assert!(simplify.stmt_query_is_empty(&query)); +} + +#[test] +fn non_empty_values_query_not_empty() { + let schema = test_schema(); + let simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `stmt_query_is_empty(values([1])) → false` + let mut values = Values::default(); + values.rows.push(stmt::Expr::Value(stmt::Value::from(1i64))); + let query = Query::values(values); + assert!(!simplify.stmt_query_is_empty(&query)); +} + +#[test] +fn simplify_clears_order_by_and_limit() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `simplify_stmt_query_when_empty(empty_query)` clears `order_by` and + // `limit` + let mut query = Query::values(Values::default()); + query.order_by = Some(OrderBy { + exprs: vec![OrderByExpr { + expr: stmt::Expr::Value(stmt::Value::from(1i64)), + order: Some(Direction::Asc), + }], + }); + query.limit = Some(Limit::Offset(LimitOffset { + limit: stmt::Expr::Value(stmt::Value::from(10i64)), + offset: None, + })); + + simplify.simplify_stmt_query_when_empty(&mut query); + + assert!(query.order_by.is_none()); + assert!(query.limit.is_none()); +} + +#[test] +fn non_empty_query_keeps_order_by_and_limit() { + let schema = test_schema(); + let mut simplify = Simplify::new(&schema, &toasty_core::driver::Capability::SQLITE); + + // `simplify_stmt_query_when_empty(non_empty_query)` keeps `order_by` + // and `limit` + let mut values = Values::default(); + values.rows.push(stmt::Expr::Value(stmt::Value::from(1i64))); + let mut query = Query::values(values); + query.order_by = Some(OrderBy { + exprs: vec![OrderByExpr { + expr: stmt::Expr::Value(stmt::Value::from(1i64)), + order: Some(Direction::Desc), + }], + }); + query.limit = Some(Limit::Offset(LimitOffset { + limit: stmt::Expr::Value(stmt::Value::from(10i64)), + offset: None, + })); + + simplify.simplify_stmt_query_when_empty(&mut query); + + assert!(query.order_by.is_some()); + assert!(query.limit.is_some()); +} diff --git a/packages/toasty/src/engine/test_util.rs b/packages/toasty/src/engine/test_util.rs new file mode 100644 index 00000000000..f25da0d204f --- /dev/null +++ b/packages/toasty/src/engine/test_util.rs @@ -0,0 +1,36 @@ +//! Shared test utilities for engine unit tests. + +use toasty_core::{ + driver::Capability, + schema::{ + Builder, + app::{self, Model}, + }, +}; + +/// Build a minimal schema with no models (useful for expression-level tests). +pub fn test_schema() -> toasty_core::Schema { + Builder::new() + .build(app::Schema::default(), &Capability::SQLITE) + .expect("empty schema should build") +} + +/// Build a schema from the given macro-generated models. +pub fn test_schema_with(models: &[Model]) -> toasty_core::Schema { + let app_schema = + app::Schema::from_macro(models.iter().cloned()).expect("schema should build from macro"); + + Builder::new() + .build(app_schema, &Capability::SQLITE) + .expect("schema should build") +} + +/// Build a schema using PostgreSQL capability (for enum type tests). +pub fn test_schema_postgresql(models: &[Model]) -> toasty_core::Schema { + let app_schema = + app::Schema::from_macro(models.iter().cloned()).expect("schema should build from macro"); + + Builder::new() + .build(app_schema, &Capability::POSTGRESQL) + .expect("schema should build") +} diff --git a/packages/toasty/src/engine/ty.rs b/packages/toasty/src/engine/ty.rs new file mode 100644 index 00000000000..7a7b2ad606f --- /dev/null +++ b/packages/toasty/src/engine/ty.rs @@ -0,0 +1,17 @@ +use super::Engine; +use toasty_core::{schema::db::Index, stmt}; + +impl Engine { + pub(crate) fn infer_ty(&self, stmt: &stmt::Statement, args: &[stmt::Type]) -> stmt::Type { + stmt::ExprContext::new(&*self.schema).infer_stmt_ty(stmt, args) + } + + pub(crate) fn index_key_record_ty(&self, index: &Index) -> stmt::Type { + let field_tys = index + .columns + .iter() + .map(|id| self.schema.db.column(id.column).ty.clone()) + .collect(); + stmt::Type::Record(field_tys) + } +} diff --git a/packages/toasty/src/engine/upsert.rs b/packages/toasty/src/engine/upsert.rs new file mode 100644 index 00000000000..6d824828b4d --- /dev/null +++ b/packages/toasty/src/engine/upsert.rs @@ -0,0 +1,221 @@ +use toasty_core::{Result, stmt}; + +/// Converts declarative upsert branches into an insert row and one conflict +/// assignment map before model fields are lowered to database columns. +pub(super) fn normalize(insert: &mut stmt::Insert, preserve_defaults: bool) -> Result<()> { + let Some(upsert) = &mut insert.upsert else { + return Ok(()); + }; + + let stmt::ExprSet::Values(values) = &mut insert.source.body else { + return Err(crate::Error::invalid_statement( + "upsert branches require a VALUES source", + )); + }; + let [row] = values.rows.as_mut_slice() else { + return Err(crate::Error::invalid_statement( + "upsert branches require exactly one source row", + )); + }; + if !row.is_record() { + let Some(fields) = row.take().into_record_items() else { + return Err(crate::Error::invalid_statement( + "upsert branches require a record source row", + )); + }; + *row = stmt::Expr::record(fields); + } + let Some(record) = row.as_record_mut() else { + return Err(crate::Error::invalid_statement( + "upsert branches require a record source row", + )); + }; + + for (projection, assignment) in &upsert.defaults { + let stmt::Assignment::Set(expr) = assignment else { + return Err(crate::Error::invalid_statement( + "upsert field defaults only support value assignments", + )); + }; + set_create_value(record, projection, expr.clone())?; + } + + for (projection, assignment) in &upsert.shared { + if upsert.create.contains(projection) { + continue; + } + let default = upsert.defaults.get(projection); + let expr = create_expr_for_assignment(assignment, default)?; + set_create_value(record, projection, expr)?; + } + + for (projection, assignment) in std::mem::take(&mut upsert.create) { + let stmt::Assignment::Set(expr) = assignment else { + return Err(crate::Error::invalid_statement( + "upsert create branch only supports value assignments", + )); + }; + set_create_value(record, &projection, expr)?; + } + + if upsert.action == stmt::UpsertAction::Update { + upsert.shared.overlay(std::mem::take(&mut upsert.update)); + } else { + upsert.shared = stmt::Assignments::new(); + upsert.update = stmt::Assignments::new(); + } + + if !preserve_defaults { + upsert.defaults = stmt::Assignments::new(); + } + + Ok(()) +} + +fn create_expr_for_assignment( + assignment: &stmt::Assignment, + default: Option<&stmt::Assignment>, +) -> Result { + match assignment { + stmt::Assignment::Set(expr) => Ok(expr.clone()), + assignment if !requires_current_value(assignment) => { + Ok(stmt::Expr::Value(apply_assignment(None, assignment)?)) + } + assignment => { + let default = default.ok_or_else(|| { + crate::Error::invalid_statement( + "shared upsert mutations require a field with #[default]", + ) + })?; + let stmt::Assignment::Set(default) = default else { + return Err(crate::Error::invalid_statement( + "upsert field defaults only support value assignments", + )); + }; + let value = apply_assignment(Some(default.eval_const()?), assignment)?; + Ok(stmt::Expr::Value(value)) + } + } +} + +pub(super) fn requires_current_value(assignment: &stmt::Assignment) -> bool { + match assignment { + stmt::Assignment::Set(_) => false, + stmt::Assignment::Batch(entries) => { + let mut initialized = false; + for entry in entries { + let requires_current = requires_current_value(entry); + if !initialized && requires_current { + return true; + } + if !requires_current { + initialized = true; + } + } + false + } + _ => true, + } +} + +fn apply_assignment( + current: Option, + assignment: &stmt::Assignment, +) -> Result { + use stmt::Assignment::*; + + match assignment { + Set(expr) => expr.eval_const(), + Insert(expr) => { + let mut items = current_list(current)?; + items.push(expr.eval_const()?); + Ok(stmt::Value::List(items)) + } + Remove(expr) => { + let mut items = current_list(current)?; + let removed = expr.eval_const()?; + items.retain(|item| item != &removed); + Ok(stmt::Value::List(items)) + } + Append(expr) => { + let mut items = current_list(current)?; + let stmt::Value::List(appended) = expr.eval_const()? else { + return Err(crate::Error::invalid_statement( + "upsert append assignment requires a list value", + )); + }; + items.extend(appended); + Ok(stmt::Value::List(items)) + } + Pop => { + let mut items = current_list(current)?; + items.pop(); + Ok(stmt::Value::List(items)) + } + RemoveAt(expr) => { + let mut items = current_list(current)?; + let index = usize::try_from(expr.eval_const()?)?; + if index < items.len() { + items.remove(index); + } + Ok(stmt::Value::List(items)) + } + Add(expr) => { + let current = current_value(current)?; + current.checked_add(&expr.eval_const()?).ok_or_else(|| { + crate::Error::invalid_statement( + "upsert add assignment overflowed or used incompatible numeric types", + ) + }) + } + Subtract(expr) => { + let current = current_value(current)?; + current.checked_sub(&expr.eval_const()?).ok_or_else(|| { + crate::Error::invalid_statement( + "upsert subtract assignment overflowed or used incompatible numeric types", + ) + }) + } + Batch(assignments) => { + let mut current = current; + for assignment in assignments { + current = Some(apply_assignment(current, assignment)?); + } + current_value(current) + } + } +} + +fn current_value(current: Option) -> Result { + current.ok_or_else(|| { + crate::Error::invalid_statement("shared upsert mutations require a field with #[default]") + }) +} + +fn current_list(current: Option) -> Result> { + let stmt::Value::List(items) = current_value(current)? else { + return Err(crate::Error::invalid_statement( + "upsert collection assignment requires a list default", + )); + }; + Ok(items) +} + +fn set_create_value( + record: &mut stmt::ExprRecord, + projection: &stmt::Projection, + expr: stmt::Expr, +) -> Result<()> { + let [field] = projection.as_slice() else { + return Err(crate::Error::invalid_statement( + "upsert create assignments must target one model field", + )); + }; + let Some(slot) = record.fields.get_mut(*field) else { + return Err(crate::Error::invalid_statement( + "upsert create assignment targets an unknown model field", + )); + }; + *slot = expr; + Ok(()) +} diff --git a/packages/toasty/src/engine/verify.rs b/packages/toasty/src/engine/verify.rs new file mode 100644 index 00000000000..91119052145 --- /dev/null +++ b/packages/toasty/src/engine/verify.rs @@ -0,0 +1,895 @@ +use crate::Result; +use crate::engine::lower::relation_expr::EmbedTarget; +use crate::engine::{Engine, upsert}; +use toasty_core::Error; +use toasty_core::driver::Capability; +use toasty_core::{ + schema::{ + Schema, + app::{self, ModelId}, + }, + stmt::{self, Statement, Visit}, +}; + +struct Verify<'a, 'v> { + schema: &'a Schema, + capability: &'a Capability, + error: &'v mut Option, +} + +struct VerifyExpr<'a, 'v> { + schema: &'a Schema, + capability: &'a Capability, + model: ModelId, + error: &'v mut Option, +} + +/// What an expression path denotes in the application schema. See +/// `VerifyExpr::resolve_expr_path`. +enum PathTarget<'a> { + /// A field of a model, embed, or relation target. + Field(&'a app::Field), + + /// The payload of an embedded enum's selected variant. + Variant(EmbedTarget<'a>), + + /// A position inside a `#[document]` value, which the schema does not + /// describe field by field. + Document, +} + +impl Engine { + pub(crate) fn verify(&self, stmt: &Statement) -> Result<()> { + let mut error = None; + Verify { + schema: &self.schema, + capability: self.capability, + error: &mut error, + } + .visit(stmt); + match error { + Some(err) => Err(err), + None => Ok(()), + } + } +} + +impl stmt::Visit for Verify<'_, '_> { + fn visit_stmt_insert(&mut self, i: &stmt::Insert) { + stmt::visit::visit_stmt_insert(self, i); + + let Some(upsert) = &i.upsert else { + return; + }; + let model = self + .schema + .app + .model(i.target.model_id_unwrap()) + .as_root_unwrap(); + let stmt::UpsertTarget::Fields(target) = &upsert.target else { + self.record(Error::invalid_statement( + "upsert conflict target must contain model fields before lowering", + )); + return; + }; + let target = target + .iter() + .filter_map(|projection| projection.as_slice().first().copied()) + .collect::>(); + let Some(index) = model.indices.iter().find(|index| { + index.unique + && index.fields.len() == target.len() + && index + .fields + .iter() + .zip(&target) + .all(|(field, target)| field.field.index == *target) + }) else { + self.record(Error::invalid_statement( + "upsert conflict target must exactly match a unique constraint", + )); + return; + }; + + if index.primary_key && !self.capability.upsert_primary_key { + self.record(Error::unsupported_feature(format!( + "{} does not support primary-key upsert", + self.capability.driver_name + ))); + } else if !index.primary_key && !self.capability.upsert_unique { + self.record(Error::unsupported_feature(format!( + "{} does not support upsert by a secondary unique constraint", + self.capability.driver_name + ))); + } + + if upsert.action == stmt::UpsertAction::Ignore && !self.capability.upsert_targeted_ignore { + self.record(Error::unsupported_feature(format!( + "{} does not support targeted upsert ignore", + self.capability.driver_name + ))); + } + + if upsert.action == stmt::UpsertAction::Update + && !upsert.update.is_empty() + && !self.capability.upsert_branch_assignments + { + self.record(Error::unsupported_feature(format!( + "{} does not support upsert on_update assignments", + self.capability.driver_name + ))); + } + + if upsert.action == stmt::UpsertAction::Update + && upsert.shared.is_empty() + && upsert.update.is_empty() + { + self.record(Error::invalid_statement( + "upsert requires at least one update assignment; use or_ignore() instead", + )); + } + + for (projection, assignment) in &upsert.shared { + let has_default = upsert.defaults.contains(projection); + if upsert::requires_current_value(assignment) && !has_default { + self.record(Error::invalid_statement( + "shared upsert mutations require a field with #[default]; use on_create and on_update instead", + )); + } + } + + if !self.capability.upsert_branch_assignments && upsert.action == stmt::UpsertAction::Update + { + for (projection, _) in &upsert.defaults { + let used = upsert + .shared + .get(projection) + .is_some_and(upsert::requires_current_value) + || (!upsert.shared.contains(projection) && !upsert.create.contains(projection)); + if !used { + continue; + } + let Some(&field) = projection.as_slice().first() else { + continue; + }; + if model.fields[field].nullable { + self.record(Error::unsupported_feature(format!( + "{} does not support nullable upsert field defaults", + self.capability.driver_name + ))); + } + } + + for (projection, _) in &upsert.create { + let Some(&field) = projection.as_slice().first() else { + continue; + }; + if model.fields[field].nullable { + self.record(Error::unsupported_feature(format!( + "{} does not support nullable upsert create assignments", + self.capability.driver_name + ))); + } + if upsert.shared.contains(projection) { + self.record(Error::unsupported_feature(format!( + "{} does not support different create and update assignments for one field", + self.capability.driver_name + ))); + } + } + } + + if !self.capability.sql() && upsert.action == stmt::UpsertAction::Update { + for secondary in model + .indices + .iter() + .filter(|index| index.unique && !index.primary_key) + { + if secondary.fields.iter().any(|field| { + upsert + .shared + .keys() + .any(|projection| projection.as_slice().first() == Some(&field.field.index)) + || upsert.create.keys().any(|projection| { + projection.as_slice().first() == Some(&field.field.index) + }) + || upsert.defaults.keys().any(|projection| { + projection.as_slice().first() == Some(&field.field.index) + }) + || upsert.update.keys().any(|projection| { + projection.as_slice().first() == Some(&field.field.index) + }) + || model.fields[field.field.index].auto.is_some() + }) { + self.record(Error::unsupported_feature(format!( + "{} upsert does not support updating a unique secondary-index field", + self.capability.driver_name + ))); + } + } + } + } + + fn visit_stmt_delete(&mut self, i: &stmt::Delete) { + stmt::visit::visit_stmt_delete(self, i); + + VerifyExpr { + schema: self.schema, + model: i.from.model_id_unwrap(), + capability: self.capability, + error: &mut *self.error, + } + .verify_filter(&i.filter); + } + + fn visit_stmt_query(&mut self, i: &stmt::Query) { + stmt::visit::visit_stmt_query(self, i); + + self.verify_single_query(i); + self.verify_offset_key_matches_order_by(i); + self.verify_limit_is_integer_literal(i); + } + + fn visit_stmt_select(&mut self, i: &stmt::Select) { + stmt::visit::visit_stmt_select(self, i); + + self.verify_include_modifiers(i); + + VerifyExpr { + schema: self.schema, + model: i.source.model_id_unwrap(), + capability: self.capability, + error: &mut *self.error, + } + .verify_filter(&i.filter); + } + + fn visit_expr_stmt(&mut self, i: &stmt::ExprStmt) { + // Mutation sub-statements (delete, update, insert) embedded in + // expressions must have a returning clause so their result can be + // used as a value. Query sub-statements produce results implicitly. + if !i.stmt.is_query() { + assert!( + i.stmt.returning().is_some(), + "mutation sub-statement in expression must have a returning clause; stmt={:#?}", + i.stmt + ); + } + + stmt::visit::visit_expr_stmt(self, i); + } + + fn visit_assignments(&mut self, i: &stmt::Assignments) { + // Builder combinators that cannot express their operation (a + // `stmt::patch` into an enum variant) record a reason instead of an + // entry. The statement fails as a whole, before planning. + for reason in i.unsupported() { + self.record(Error::unsupported_feature(reason.clone())); + } + + stmt::visit::visit_assignments(self, i); + } + + fn visit_stmt_update(&mut self, i: &stmt::Update) { + stmt::visit::visit_stmt_update(self, i); + + // A rejected assignment may be the only one requested, so there is + // nothing further to check. + if !i.assignments.unsupported().is_empty() { + return; + } + + // Is not an empty update + assert!(!i.assignments.is_empty(), "stmt = {i:#?}"); + + let mut verify_expr = VerifyExpr { + schema: self.schema, + model: i.target.model_id_unwrap(), + capability: self.capability, + error: &mut *self.error, + }; + + // The target query was verified above in its own scope. Its include + // paths may be rooted at enum variants, not at the updated model. + verify_expr.visit_assignments(&i.assignments); + verify_expr.visit_filter(&i.filter); + verify_expr.visit_condition(&i.condition); + } +} + +impl Verify<'_, '_> { + fn record(&mut self, err: Error) { + if self.error.is_none() { + *self.error = Some(err); + } + } + + fn verify_offset_key_matches_order_by(&mut self, i: &stmt::Query) { + let Some(stmt::Limit::Cursor(cursor)) = i.limit.as_ref() else { + return; + }; + + let Some(after) = cursor.after.as_ref() else { + return; + }; + + // SQL requires ORDER BY for cursor-based pagination. + // NoSQL drivers (DynamoDB) use a driver-level cursor (ExclusiveStartKey) + // and do not require ORDER BY. + if !self.capability.sql() { + return; + } + + let Some(order_by) = i.order_by.as_ref() else { + self.record(Error::invalid_statement( + "cursor-based pagination requires an ORDER BY clause", + )); + return; + }; + + match after { + stmt::Expr::Value(stmt::Value::Record(record)) => { + if record.fields.is_empty() { + self.record(Error::invalid_statement( + "cursor must contain at least one ORDER BY value", + )); + } else if record.fields.len() > order_by.exprs.len() { + self.record(Error::invalid_statement(format!( + "cursor contains {} values but the query has {} ORDER BY fields", + record.fields.len(), + order_by.exprs.len(), + ))); + } + } + // A scalar cursor specifies the first ORDER BY value. This remains + // valid when normalization appends hidden tie-breaker fields. + stmt::Expr::Value(_) => {} + _ => self.record(Error::invalid_statement( + "cursor must be a literal value or record", + )), + } + } + + /// Reject include ordering on singular relations and preserve the existing + /// rule that filters are rejected only on required singular relations. + fn verify_include_modifiers(&mut self, i: &stmt::Select) { + for include in i.returning.model_includes() { + let Some(query) = &include.query else { + continue; + }; + let has_filter = match &query.body { + stmt::ExprSet::Select(select) => select.filter.expr.is_some(), + _ => false, + }; + let has_order_by = query.order_by.is_some(); + if !has_filter && !has_order_by { + continue; + } + let mut error = None; + let verifier = VerifyExpr { + schema: self.schema, + capability: self.capability, + model: i.source.model_id_unwrap(), + error: &mut error, + }; + let Some(PathTarget::Field(field)) = + verifier.resolve_expr_path(&include.path.clone().into_stmt()) + else { + continue; + }; + let singular = match &field.ty { + app::FieldTy::Has(rel) => rel.is_one(), + app::FieldTy::BelongsTo(_) => true, + app::FieldTy::Via(via) => via.is_one(), + _ => continue, + }; + if has_order_by && singular { + self.record(Error::invalid_statement(format!( + "cannot order the include of singular relation `{}`; \ + include ordering requires a many-valued relation", + field.name, + ))); + continue; + } + let required_one = singular && !field.nullable; + if has_filter && required_one { + self.record(Error::invalid_statement(format!( + "cannot filter the include of required relation `{}`; \ + filter the parent query instead", + field.name, + ))); + continue; + } + } + } + + fn verify_single_query(&self, i: &stmt::Query) { + if !i.single { + return; + } + + if let stmt::ExprSet::Values(values) = &i.body { + assert_eq!(1, values.rows.len(), "stmt={i:#?}"); + } + } + + /// Assert that every field inside a `LIMIT` clause is an `I64` literal. + /// + /// Runtime pagination fields use `Expr::Value`; the fixed limit from + /// `.first()` uses `Expr::Static`. Downstream consumers rely on this + /// invariant. Any other form means either a builder regressed or the AST was + /// hand-constructed with a non-canonical shape. + fn verify_limit_is_integer_literal(&self, i: &stmt::Query) { + let Some(limit) = i.limit.as_ref() else { + return; + }; + match limit { + stmt::Limit::Cursor(c) => { + assert_i64_value(&c.page_size, "Cursor page_size"); + } + stmt::Limit::Offset(o) => { + assert_i64_literal(&o.limit, "Offset limit"); + if let Some(off) = o.offset.as_ref() { + assert_i64_value(off, "Offset offset"); + } + } + } + } +} + +#[track_caller] +fn assert_i64_literal(expr: &stmt::Expr, what: &str) { + assert!( + matches!( + expr, + stmt::Expr::Value(stmt::Value::I64(_)) | stmt::Expr::Static(stmt::Value::I64(_)) + ), + "{what} must be an I64 literal; got {expr:#?}" + ); +} + +#[track_caller] +fn assert_i64_value(expr: &stmt::Expr, what: &str) { + assert!( + matches!(expr, stmt::Expr::Value(stmt::Value::I64(_))), + "{what} must be a Value::I64 literal; got {expr:#?}" + ); +} + +impl<'a> VerifyExpr<'a, '_> { + fn verify_filter(&mut self, filter: &stmt::Filter) { + self.assert_bool_expr(filter.as_expr()); + self.visit_expr(filter.as_expr()); + } + + fn record(&mut self, err: Error) { + if self.error.is_none() { + *self.error = Some(err); + } + } + + /// Whether `expr` references a whole document-stored field of the current + /// model: a `#[document]` embed (`Type::Model`) or an embed collection + /// (`List(Model)`). + fn is_document_field(&self, expr: &stmt::Expr) -> bool { + let stmt::Expr::Reference(stmt::ExprReference::Field { nesting: 0, index }) = expr else { + return false; + }; + let Some(root) = self.schema.app.model(self.model).as_root() else { + return false; + }; + let Some(field) = root.fields.get(*index) else { + return false; + }; + let app::FieldTy::Primitive(primitive) = &field.ty else { + return false; + }; + let embed_id = match &primitive.ty { + stmt::Type::Model(id) => *id, + stmt::Type::List(elem) => match &**elem { + stmt::Type::Model(id) => *id, + _ => return false, + }, + _ => return false, + }; + matches!( + self.schema.app.model(embed_id), + app::Model::EmbeddedStruct(_) + ) + } + + /// Resolve an expression path — a field reference in the current scope + /// under `Project` and `Variant` layers — to what it denotes in the + /// application schema. `None` when the path does not resolve. + /// + /// A projection step continues into a struct embed's fields, a relation + /// target's fields, or the fields of a selected enum variant, by their + /// variant-local position (see [`EmbedTarget::field_at`]). + fn resolve_expr_path(&self, expr: &stmt::Expr) -> Option> { + match expr { + stmt::Expr::Reference(stmt::ExprReference::Field { nesting: 0, index }) => self + .schema + .app + .model(self.model) + .as_root()? + .fields + .get(*index) + .map(PathTarget::Field), + stmt::Expr::Project(project) => { + let mut target = self.resolve_expr_path(&project.base)?; + for step in project.projection.as_slice() { + target = self.resolve_expr_step(target, *step)?; + } + Some(target) + } + stmt::Expr::Variant(variant) => { + let PathTarget::Field(field) = self.resolve_expr_path(&variant.base)? else { + return None; + }; + let app::FieldTy::Embedded(embedded) = &field.ty else { + return None; + }; + EmbedTarget::embed(&self.schema.app, embedded.target)? + .select(variant.variant) + .map(PathTarget::Variant) + } + _ => None, + } + } + + /// The target one projection step reaches from `target`. + fn resolve_expr_step(&self, target: PathTarget<'a>, step: usize) -> Option> { + use app::FieldTy; + + let field = match target { + PathTarget::Variant(target) => target.field_at(step), + PathTarget::Document => return Some(PathTarget::Document), + PathTarget::Field(field) => match &field.ty { + FieldTy::Embedded(embedded) => { + EmbedTarget::embed(&self.schema.app, embedded.target)?.field_at(step) + } + FieldTy::BelongsTo(_) | FieldTy::Has(_) | FieldTy::Via(_) => { + let target = field.relation_target_id().expect("relation has a target"); + self.schema + .app + .model(target) + .as_root_unwrap() + .fields + .get(step) + } + // A `#[document]` embed stores sub-fields in the document + // type rather than as `app::Field`s; the path was + // type-checked by the generated accessors. + FieldTy::Primitive(app::FieldPrimitive { + ty: stmt::Type::Model(_), + .. + }) => return Some(PathTarget::Document), + FieldTy::Primitive(_) => None, + }, + }; + + field.map(PathTarget::Field) + } + + /// Whether `expr` is an expression path rooted at a field reference in + /// the current scope — the shape `resolve_expr_path` validates. + fn is_scoped_expr_path(&self, expr: &stmt::Expr) -> bool { + match expr { + stmt::Expr::Reference(stmt::ExprReference::Field { nesting: 0, .. }) => true, + stmt::Expr::Project(project) => self.is_scoped_expr_path(&project.base), + stmt::Expr::Variant(variant) => self.is_scoped_expr_path(&variant.base), + _ => false, + } + } + + fn assert_bool_expr(&self, expr: &stmt::Expr) { + use stmt::Expr::*; + + match expr { + And(_) + | AllOp(_) + | AnyOp(_) + | Between(_) + | BinaryOp(_) + | Like(_) + | InList(_) + | InSubquery(_) + | Intersects(_) + | IsNull(_) + | IsSuperset(_) + | IsVariant(_) + | Not(_) + | Or(_) + | StartsWith(_) + | Value(stmt::Value::Bool(_)) => {} + expr => panic!("Not a bool? {expr:#?}"), + } + } +} + +impl stmt::Visit for VerifyExpr<'_, '_> { + fn visit_expr_and(&mut self, i: &stmt::ExprAnd) { + stmt::visit::visit_expr_and(self, i); + + for expr in &i.operands { + self.assert_bool_expr(expr); + } + } + + fn visit_expr_not(&mut self, i: &stmt::ExprNot) { + stmt::visit::visit_expr_not(self, i); + self.assert_bool_expr(&i.expr); + } + + fn visit_expr_or(&mut self, i: &stmt::ExprOr) { + stmt::visit::visit_expr_or(self, i); + + for expr in &i.operands { + self.assert_bool_expr(expr); + } + } + + fn visit_projection(&mut self, i: &stmt::Projection) { + let root = self.schema.app.model(self.model); + assert!( + self.schema.app.resolve(root, i).is_some(), + "invalid projection: {i:?}" + ); + } + + fn visit_expr_project(&mut self, i: &stmt::ExprProject) { + // For a path rooted at a field reference in the current scope, + // validate the whole path against the schema: each step must name a + // field of the embed, relation target, or selected enum variant it + // steps into. + if self.is_scoped_expr_path(&i.base) { + assert!( + self.resolve_expr_path(&stmt::Expr::Project(i.clone())) + .is_some(), + "failed to resolve projection: {i:#?}" + ); + } else { + // For other base expressions, visit the base but skip projection + // validation since the projection is relative to the base + // expression's type. + self.visit_expr(&i.base); + } + } + + fn visit_expr_variant(&mut self, i: &stmt::ExprVariant) { + // A selection applies to an enum field reached by a scoped path; it + // must name one of that enum's variants. + if self.is_scoped_expr_path(&i.base) { + assert!( + self.resolve_expr_path(&stmt::Expr::Variant(i.clone())) + .is_some(), + "failed to resolve variant selection: {i:#?}" + ); + } else { + self.visit_expr(&i.base); + } + } + + fn visit_expr_binary_op(&mut self, i: &stmt::ExprBinaryOp) { + stmt::visit::visit_expr_binary_op(self, i); + + // Comparing a `#[document]` field against a whole embed value is not + // yet supported (document value equality is planned — see the design + // doc). Reject it here with a clear error instead of letting it reach + // the engine's type inference, which cannot merge a document column + // with a record value. + if self.is_document_field(&i.lhs) || self.is_document_field(&i.rhs) { + self.record(Error::unsupported_feature( + "comparing a #[document] field to a whole value is not yet supported; \ + filter on individual fields inside the document instead", + )); + } + } + + fn visit_expr_in_subquery(&mut self, i: &stmt::ExprInSubquery) { + // stmt::visit::visit_expr_in_subquery(self, i); + + // Visit **only** the subquery expression + self.visit(&*i.expr); + + // The subquery is verified independently, sharing the error slot so + // failures inside it surface to the caller. + Verify { + schema: self.schema, + capability: self.capability, + error: &mut *self.error, + } + .visit(&*i.query); + } + + fn visit_expr_like(&mut self, i: &stmt::ExprLike) { + // `.ilike()` is a pass-through to the database's own case-insensitive + // LIKE operator. Only PostgreSQL has one (`ILIKE`), so reject a + // case-insensitive match on any other backend rather than silently + // emitting plain `LIKE`, whose case behavior differs across engines. + if i.case_insensitive && !self.capability.native_ilike { + self.record(Error::unsupported_feature(format!( + "{} does not provide a native ILIKE operator; use like instead", + self.capability.driver_name + ))); + } + stmt::visit::visit_expr_like(self, i); + } + + fn visit_expr_is_superset(&mut self, i: &stmt::ExprIsSuperset) { + if !self.capability.native_array_set_predicates && !rhs_is_concrete_list(&i.rhs) { + self.record(Error::unsupported_feature(format!( + "{} requires a literal list on the right-hand side of is_superset", + self.capability.driver_name + ))); + } + stmt::visit::visit_expr_is_superset(self, i); + } + + fn visit_expr_intersects(&mut self, i: &stmt::ExprIntersects) { + if !self.capability.native_array_set_predicates && !rhs_is_concrete_list(&i.rhs) { + self.record(Error::unsupported_feature(format!( + "{} requires a literal list on the right-hand side of intersects", + self.capability.driver_name + ))); + } + stmt::visit::visit_expr_intersects(self, i); + } +} + +/// True when the expression is — or will fold to — a `Value::List` of +/// concrete values. Verify runs before the simplifier, so the user's +/// `vec![…]` still appears as an `Expr::List` of `Expr::Value` items; +/// `fold::expr_list` collapses that shape to `Value::List` during +/// lowering, which is what the driver eventually sees. +fn rhs_is_concrete_list(expr: &stmt::Expr) -> bool { + match expr { + stmt::Expr::Value(stmt::Value::List(_)) => true, + stmt::Expr::List(list) => list + .items + .iter() + .all(|item| matches!(item, stmt::Expr::Value(_))), + _ => false, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::engine::test_util::test_schema; + use toasty_core::driver::Capability; + use toasty_core::stmt::{Expr, ExprIsSuperset, ExprList, Value}; + + fn verify_with(capability: &'static Capability, stmt: Statement) -> Result<()> { + let schema = test_schema(); + let mut error = None; + Verify { + schema: &schema, + capability, + error: &mut error, + } + .visit(&stmt); + match error { + Some(err) => Err(err), + None => Ok(()), + } + } + + fn verify_expr_with(capability: &'static Capability, expr: &Expr) -> Option { + let schema = test_schema(); + let mut error = None; + // ModelId is only used by projection-checking visitor methods, which + // these expression-only tests don't trigger. + VerifyExpr { + schema: &schema, + capability, + model: toasty_core::schema::app::ModelId(0), + error: &mut error, + } + .visit_expr(expr); + error + } + + fn is_superset(rhs: Expr) -> Expr { + Expr::IsSuperset(ExprIsSuperset { + lhs: Box::new(Expr::arg(0)), + rhs: Box::new(rhs), + }) + } + + #[test] + fn update_with_unsupported_assignment_is_rejected() { + let mut assignments = stmt::Assignments::new(); + assignments.reject_unsupported("patch into variant"); + let update = stmt::Update { + target: stmt::UpdateTarget::Model(ModelId(0)), + assignments, + filter: stmt::Filter::new(stmt::Expr::from(true)), + condition: stmt::Condition::default(), + returning: None, + }; + + let err = verify_with(&Capability::SQLITE, Statement::Update(update)) + .expect_err("expected unsupported_feature error"); + assert!(err.is_unsupported_feature()); + } + + #[test] + #[should_panic(expected = "Offset offset must be a Value::I64 literal")] + fn offset_with_non_i64_limit_panics() { + let mut query = stmt::Query::unit(); + query.limit = Some(stmt::Limit::Offset(stmt::LimitOffset { + limit: stmt::Value::I64(10).into(), + offset: Some(stmt::Value::U64(5).into()), + })); + verify_with(&Capability::SQLITE, Statement::Query(query)).unwrap(); + } + + #[test] + fn is_superset_literal_rhs_accepted_on_ddb() { + let expr = is_superset(Expr::Value(Value::List(vec![Value::I64(1)]))); + assert!(verify_expr_with(&Capability::DYNAMODB, &expr).is_none()); + } + + #[test] + fn is_superset_pre_fold_expr_list_accepted_on_ddb() { + // Pre-simplifier shape produced by `is_superset(vec![…])`: an + // `Expr::List` of `Expr::Value` items. The fold pass will collapse + // this to `Value::List` during lowering. + let expr = is_superset(Expr::List(ExprList { + items: vec![Expr::Value(Value::I64(1)), Expr::Value(Value::I64(2))], + })); + assert!(verify_expr_with(&Capability::DYNAMODB, &expr).is_none()); + } + + #[test] + fn is_superset_non_literal_rhs_rejected_on_ddb() { + let expr = is_superset(Expr::arg(1)); + let err = verify_expr_with(&Capability::DYNAMODB, &expr) + .expect("expected unsupported_feature error"); + assert!(err.is_unsupported_feature()); + } + + #[test] + fn is_superset_non_literal_rhs_accepted_on_sqlite() { + let expr = is_superset(Expr::arg(1)); + assert!(verify_expr_with(&Capability::SQLITE, &expr).is_none()); + } + + #[test] + fn ilike_accepted_on_postgresql() { + let expr = Expr::ilike(Expr::arg(0), Expr::arg(1)); + assert!(verify_expr_with(&Capability::POSTGRESQL, &expr).is_none()); + } + + #[test] + fn ilike_rejected_on_sqlite() { + let expr = Expr::ilike(Expr::arg(0), Expr::arg(1)); + let err = verify_expr_with(&Capability::SQLITE, &expr) + .expect("expected unsupported_feature error"); + assert!(err.is_unsupported_feature()); + assert!(err.to_string().contains(Capability::SQLITE.driver_name)); + } + + #[test] + fn ilike_rejected_on_mysql() { + let expr = Expr::ilike(Expr::arg(0), Expr::arg(1)); + let err = verify_expr_with(&Capability::MYSQL, &expr) + .expect("expected unsupported_feature error"); + assert!(err.is_unsupported_feature()); + } + + #[test] + fn ilike_rejected_on_dynamodb() { + let expr = Expr::ilike(Expr::arg(0), Expr::arg(1)); + let err = verify_expr_with(&Capability::DYNAMODB, &expr) + .expect("expected unsupported_feature error"); + assert!(err.is_unsupported_feature()); + } + + #[test] + fn case_sensitive_like_accepted_on_sqlite() { + let expr = Expr::like(Expr::arg(0), Expr::arg(1)); + assert!(verify_expr_with(&Capability::SQLITE, &expr).is_none()); + } +} diff --git a/packages/toasty/src/instrument.rs b/packages/toasty/src/instrument.rs new file mode 100644 index 00000000000..5052fb88d38 --- /dev/null +++ b/packages/toasty/src/instrument.rs @@ -0,0 +1,78 @@ +//! Tracing spans for statement execution. +//! +//! Statements execute on a dedicated per-connection worker task, so a span +//! entered by the caller does not naturally cover the driver's work. The +//! spans created here are constructed on the caller's task — parenting them +//! to the caller's current span (e.g. a per-request span) — then carried +//! across the channel to the worker task and entered there, so every event +//! the engine and driver emit lands inside the caller's span tree. + +use toasty_core::{ + Schema, + schema::app::ModelId, + stmt::{self, ExprSet, InsertTarget, UpdateTarget}, +}; + +/// Creates the `toasty::query` span for one statement execution. +/// +/// When the span is disabled by the subscriber's filter, returns the +/// caller's current span instead so request context still reaches the +/// worker task's events. +pub(crate) fn query_span(schema: &Schema, stmt: &stmt::Statement) -> tracing::Span { + let span = tracing::debug_span!( + target: "toasty::query", + "query", + stmt.kind = stmt.name(), + model = tracing::field::Empty, + ); + if span.is_disabled() { + return tracing::Span::current(); + } + if let Some(model_id) = stmt_model_id(stmt) { + let name = schema.app.model(model_id).name().upper_camel_case(); + span.record("model", name.as_str()); + } + span +} + +/// Creates the `toasty::query` span for a user-authored SQL execution. +/// Falls back to the caller's current span like [`query_span`]. +pub(crate) fn raw_sql_span() -> tracing::Span { + let span = tracing::debug_span!(target: "toasty::query", "query", stmt.kind = "raw_sql"); + if span.is_disabled() { + return tracing::Span::current(); + } + span +} + +/// Resolves the model a statement targets, when it targets one directly. +/// Statements arriving here are model-level; lowered (table-level) shapes +/// return `None`. +fn stmt_model_id(stmt: &stmt::Statement) -> Option { + match stmt { + stmt::Statement::Query(query) => query_model_id(query), + stmt::Statement::Insert(insert) => match &insert.target { + InsertTarget::Model(model_id) => Some(*model_id), + InsertTarget::Scope(query) => query_model_id(query), + InsertTarget::Table(_) => None, + }, + stmt::Statement::Update(update) => match &update.target { + UpdateTarget::Model(model_id) => Some(*model_id), + UpdateTarget::Query(query) => query_model_id(query), + UpdateTarget::Table(_) => None, + }, + stmt::Statement::Delete(delete) => delete.from.as_model().map(|source| source.id), + } +} + +fn query_model_id(query: &stmt::Query) -> Option { + match &query.body { + ExprSet::Select(select) => select.source.as_model().map(|source| source.id), + ExprSet::Update(update) => match &update.target { + UpdateTarget::Model(model_id) => Some(*model_id), + UpdateTarget::Query(query) => query_model_id(query), + UpdateTarget::Table(_) => None, + }, + _ => None, + } +} diff --git a/packages/toasty/src/lib.rs b/packages/toasty/src/lib.rs new file mode 100644 index 00000000000..236d0282a91 --- /dev/null +++ b/packages/toasty/src/lib.rs @@ -0,0 +1,162 @@ +#![warn(missing_docs)] +//! Toasty is an async ORM for Rust supporting both SQL (SQLite, PostgreSQL, +//! MySQL) and NoSQL (DynamoDB) databases. +//! +//! This crate is the user-facing API. It contains the database handle, +//! query execution traits, and the types that generated code builds on. For +//! a tutorial-style introduction, see the [Toasty guide]. +//! +#![doc = include_str!(concat!(env!("OUT_DIR"), "/guide_link.md"))] +//! +//! # Modules +//! +//! ## [`db`] — database handle and connection pool +//! +//! The [`Db`] type is the entry point for all database interaction. It owns +//! a connection pool and provides [`Db::builder`] for configuration. The +//! module also contains [`Builder`](db::Builder) and the pool internals. +//! +//! ## [`schema`] — model, relation, and schema inspection +//! +//! The [`Model`](schema::Model) trait represents a root model that maps to a +//! database table. It is implemented by `#[derive(Model)]` — users do not +//! implement it manually. The module also contains [`Field`](schema::Field), +//! which provides schema registration and runtime helpers for field types, and +//! [`Auto`](schema::Auto), a wrapper for auto-generated values such as +//! database-assigned IDs. +//! +//! Relation fields can use [`Deferred`](schema::Deferred) for lazy loading or +//! direct relation values for eager loading. Lazy relations are populated +//! through `.include(...)` or generated relation accessors. Eager relations are +//! loaded whenever the model is loaded. +//! +//! A many-to-many relationship uses a join model with one `BelongsTo` relation +//! to each endpoint. Endpoint models can expose the opposite side with +//! `#[has_many(via = memberships.group)]`; code mutates the join model and uses +//! the derived `via` field for read-only traversal. +//! +//! The module also re-exports from `toasty-core` for inspecting the +//! app-level and db-level schema representations at runtime. +//! +//! ## [`stmt`] — typed statement and expression types +//! +//! Contains the typed wrappers around the statement AST: +//! [`Query`](stmt::Query), [`Insert`](stmt::Insert), +//! [`Update`](stmt::Update), [`Upsert`](stmt::Upsert), +//! [`Delete`](stmt::Delete), and +//! [`Statement`]. Also includes expression helpers like [`Expr`](stmt::Expr), +//! [`Path`](stmt::Path), and the [`in_list`](stmt::in_list) function. +//! Generated query builders (e.g. `find_by_*`, `filter_by_*`) produce these +//! types. +//! +//! ## [`sql`] — raw SQL execution helpers +//! +//! Contains [`sql::statement`] and [`sql::query`] for running backend SQL +//! through [`Db`], [`Connection`], or [`Transaction`] handles. +//! +//! # Key traits +//! +//! - [`Model`](schema::Model) — a root model backed by a database table. +//! Implemented by `#[derive(Model)]`. +//! - [`Embed`](schema::Embed) — an embedded type whose fields are flattened +//! into the parent model's table. Implemented by `#[derive(Embed)]`. +//! - [`Executor`] — the dyn-compatible interface for running statements. +//! [`Db`] and [`Transaction`] both implement it. The generic +//! [`exec`](Executor::exec) method on `dyn Executor` accepts any typed +//! [`Statement`]. +//! - [`Load`](schema::Load) — deserializes a model instance from the database +//! value representation. +//! +//! # Other key types +//! +//! - [`Transaction`] / [`TransactionBuilder`] — transactions with +//! auto-rollback on drop and nested savepoint support. +//! - [`Page`](stmt::Page) — a page of results from a paginated query, with cursor-based +//! navigation. +//! - [`Batch`](stmt::Batch) — groups multiple queries into a single round-trip. +//! - [`Capability`] / [`Dialect`] / [`SqlPlaceholder`] — driver metadata, +//! including the SQL dialect spoken and placeholder syntax for raw SQL. +//! - [`Error`] / [`Result`] — re-exported from `toasty-core`. +//! +//! # Derive macros +//! +//! - [`#[derive(Model)]`](derive@Model) — generates the +//! [`Model`](schema::Model) impl, query builders, create/update/upsert +//! builders, relation accessors, and schema registration for a struct. +//! - [`#[derive(Embed)]`](derive@Embed) — generates the +//! [`Embed`](schema::Embed) impl for a type whose fields are stored inline +//! in a parent model's table. +//! +//! # Feature flags +//! +//! Each database driver is behind an optional feature flag: +//! +//! | Feature | Driver crate | +//! |----------------|------------------------------| +//! | `sqlite` | `toasty-driver-sqlite` | +//! | `postgresql` | `toasty-driver-postgresql` | +//! | `mysql` | `toasty-driver-mysql` | +//! | `mariadb` | `toasty-driver-mysql` | +//! | `dynamodb` | `toasty-driver-dynamodb` | +//! | `turso` | `toasty-driver-turso` | +//! +//! Additional feature flags: `rust_decimal`, `bigdecimal`, `jiff` (date/time +//! via the `jiff` crate), `net` (IP network and MAC address types via the +//! `cidr` and `macaddr` crates), and `serde` (JSON serialization support). +//! +//! # Other crates in the workspace +//! +//! `toasty` depends on several internal crates that most users do not need +//! to use directly: +//! +//! - **`toasty-core`** — shared types used across the workspace: the schema +//! representations (app-level and db-level), the statement AST, the +//! [`Driver`](driver::Driver) trait, and [`Error`] / [`Result`]. +//! - **`toasty-macros`** — the proc-macro entry points and the code generation +//! logic they call. +//! - **`toasty-sql`** — serializes the statement AST to SQL strings. Used by +//! the SQL driver crates. +//! - **`toasty-driver-*`** — one crate per database backend. + +mod update_target; +pub use update_target::UpdateTarget; + +// `Batch`, `batch()`, and `CreateMany` live in `stmt`. +pub use stmt::{Batch, batch}; + +/// Database handle, connection pool, executor trait, and transaction support. +pub mod db; +pub use db::{ + Capability, Connection, Db, Dialect, Executor, SqlPlaceholder, Transaction, TransactionBuilder, +}; + +mod engine; + +mod instrument; + +/// Schema migration types: history files, snapshots, and generation helpers. +#[cfg(feature = "migration")] +pub mod migration; + +/// Model, relation, and schema inspection types. +pub mod schema; +pub use schema::Deferred; + +// `Page` lives in `stmt`. + +/// Typed statement, expression, and query builder types. +pub mod stmt; +#[cfg(feature = "serde")] +pub use stmt::Json; +pub use stmt::Statement; + +/// Raw SQL execution helpers. +pub mod sql; + +#[cfg(feature = "migration")] +pub use toasty_macros::embed_migrations; +pub use toasty_macros::{Embed, Model, create, query, update}; + +pub use toasty_core::{Error, Result, schema::app::ModelSet}; + +pub mod codegen_support; diff --git a/packages/toasty/src/migration.rs b/packages/toasty/src/migration.rs new file mode 100644 index 00000000000..b611a31b66b --- /dev/null +++ b/packages/toasty/src/migration.rs @@ -0,0 +1,16 @@ +//! Schema migration types: history files, snapshots, and supporting +//! configuration. +//! +//! The migration system tracks a sequence of generated SQL migrations on disk +//! alongside the schema snapshots they were derived from. [`History`] is the +//! in-memory representation of the TOML history file; each [`HistoryEntry`] +//! records one generated migration. + +mod embed; +mod generate; +mod snapshot; + +pub use embed::{MigrationFile, MigrationReport, MigrationSet}; +pub use generate::{Generated, generate}; +pub use snapshot::Snapshot; +pub use toasty_core::migration::{History, HistoryEntry}; diff --git a/packages/toasty/src/migration/embed.rs b/packages/toasty/src/migration/embed.rs new file mode 100644 index 00000000000..2d8b36c1e29 --- /dev/null +++ b/packages/toasty/src/migration/embed.rs @@ -0,0 +1,103 @@ +use std::collections::HashSet; + +use crate::{Db, Result, schema::db::Migration as DbMigration}; + +/// A migration SQL file and its tracking metadata. +#[derive(Debug, Clone, Copy)] +pub struct MigrationFile { + id: u64, + name: &'static str, + sql: &'static str, +} + +impl MigrationFile { + /// Creates a migration file. + pub const fn new(id: u64, name: &'static str, sql: &'static str) -> Self { + Self { id, name, sql } + } + + /// Returns the migration ID recorded in the database. + pub fn id(&self) -> u64 { + self.id + } + + /// Returns the migration file name. + pub fn name(&self) -> &str { + self.name + } + + /// Returns the migration SQL. + pub fn sql(&self) -> &str { + self.sql + } +} + +/// An ordered collection of migrations that can be applied to a database. +#[derive(Debug, Clone, Copy, Default)] +pub struct MigrationSet { + migrations: &'static [MigrationFile], +} + +impl MigrationSet { + /// Creates a migration set in application order. + pub const fn new(migrations: &'static [MigrationFile]) -> Self { + Self { migrations } + } + + /// Returns the migrations in application order. + pub fn migrations(&self) -> &[MigrationFile] { + self.migrations + } + + /// Applies pending migrations to `db` in order. + /// + /// Migration IDs already recorded by the driver are skipped. If a + /// migration fails, later migrations are not attempted. + pub async fn apply(&self, db: &Db) -> Result { + let conn = db.connection().await?; + let mut applied_ids = conn + .applied_migrations() + .await? + .into_iter() + .map(|migration| migration.id()) + .collect::>(); + let mut report = MigrationReport::default(); + + for migration in self.migrations { + if applied_ids.contains(&migration.id) { + report.skipped += 1; + continue; + } + + conn.apply_migration( + migration.id, + migration.name, + DbMigration::new_sql(migration.sql.to_string()), + ) + .await?; + applied_ids.insert(migration.id); + report.applied += 1; + } + + Ok(report) + } +} + +/// Counts returned after applying a [`MigrationSet`]. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct MigrationReport { + applied: usize, + skipped: usize, +} + +impl MigrationReport { + /// Returns the number of migrations applied by this call. + pub fn applied(&self) -> usize { + self.applied + } + + /// Returns the number of migrations skipped because their IDs were already applied. + pub fn skipped(&self) -> usize { + self.skipped + } +} diff --git a/packages/toasty/src/migration/generate.rs b/packages/toasty/src/migration/generate.rs new file mode 100644 index 00000000000..2b0fbba603a --- /dev/null +++ b/packages/toasty/src/migration/generate.rs @@ -0,0 +1,42 @@ +use crate::{ + db::Driver, + schema::{db, diff}, +}; + +use super::Snapshot; + +/// A generated database migration and the schema snapshot it advances to. +/// +/// This is the reusable core of migration generation. It deliberately does not +/// include filenames, history IDs, or persistence decisions; callers own those +/// policies. +#[derive(Debug)] +pub struct Generated { + /// The driver-specific migration statements. + pub migration: db::Migration, + + /// Snapshot of the schema after the migration is applied. + pub snapshot: Snapshot, +} + +/// Generate a database migration from `previous` to `next`. +/// +/// Returns `None` when the schemas are equivalent after applying +/// `rename_hints`. +pub fn generate( + driver: &dyn Driver, + previous: &db::Schema, + next: &db::Schema, + rename_hints: &diff::RenameHints, +) -> Option { + let schema_diff = diff::Schema::from(previous, next, rename_hints); + + if schema_diff.is_empty() { + return None; + } + + Some(Generated { + migration: driver.generate_migration(&schema_diff), + snapshot: Snapshot::new(next.clone()), + }) +} diff --git a/packages/toasty/src/migration/snapshot.rs b/packages/toasty/src/migration/snapshot.rs new file mode 100644 index 00000000000..20f3a7db483 --- /dev/null +++ b/packages/toasty/src/migration/snapshot.rs @@ -0,0 +1,124 @@ +use crate::{Error, Result, schema::db::Schema}; +use serde::{Deserialize, Serialize}; +use std::{fmt, path::Path, str::FromStr}; +use toml_edit::{DocumentMut, Item}; + +const SNAPSHOT_VERSION: u32 = 1; + +/// A TOML-serializable snapshot of a database schema. +/// +/// Snapshots capture the schema after a generated migration. The next +/// generation compares the most recent snapshot against the current schema to +/// build a diff. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Snapshot { + /// Snapshot format version. + version: u32, + + /// The database schema captured by this snapshot. + pub schema: Schema, +} + +impl Snapshot { + /// Create a new snapshot for `schema`. + pub fn new(schema: Schema) -> Self { + Self { + version: SNAPSHOT_VERSION, + schema, + } + } + + /// Load a snapshot from a TOML file. + pub fn load(path: impl AsRef) -> Result { + let contents = std::fs::read_to_string(path.as_ref())?; + contents.parse() + } + + /// Serialize the snapshot as TOML. + pub fn to_toml_string(&self) -> Result { + Ok(self.to_toml_document()?.to_string()) + } + + /// Save the snapshot to a TOML file. + pub fn save(&self, path: impl AsRef) -> Result<()> { + std::fs::write(path.as_ref(), self.to_toml_string()?)?; + Ok(()) + } + + fn to_toml_document(&self) -> Result { + let mut doc = toml_edit::ser::to_document(self) + .map_err(|err| Error::from_args(format_args!("{err}")))?; + + for (_key, item) in doc.as_table_mut().iter_mut() { + if item.is_inline_table() { + let mut placeholder = Item::None; + std::mem::swap(item, &mut placeholder); + let mut table = match placeholder.into_table() { + Ok(table) => table, + Err(original) => { + *item = original; + continue; + } + }; + + for (_key, item) in table.iter_mut() { + if item.is_array() { + let mut placeholder = Item::None; + std::mem::swap(item, &mut placeholder); + let mut array = match placeholder.into_array_of_tables() { + Ok(array) => array, + Err(original) => { + *item = original; + continue; + } + }; + + for table in array.iter_mut() { + for (_key, item) in table.iter_mut() { + if item.is_array() { + let mut placeholder = Item::None; + std::mem::swap(item, &mut placeholder); + match placeholder.into_array_of_tables() { + Ok(array) => *item = array.into(), + Err(original) => *item = original, + } + } + } + } + + *item = array.into(); + } + } + + *item = table.into(); + } + } + + Ok(doc) + } +} + +impl FromStr for Snapshot { + type Err = Error; + + fn from_str(s: &str) -> Result { + let snapshot: Snapshot = + toml::from_str(s).map_err(|err| Error::from_args(format_args!("{err}")))?; + + if snapshot.version != SNAPSHOT_VERSION { + return Err(Error::from_args(format_args!( + "unsupported snapshot version: {}. Expected version {}", + snapshot.version, SNAPSHOT_VERSION + ))); + } + + Ok(snapshot) + } +} + +impl fmt::Display for Snapshot { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let doc = self.to_toml_document().map_err(|_| fmt::Error)?; + write!(f, "{doc}") + } +} diff --git a/packages/toasty/src/schema.rs b/packages/toasty/src/schema.rs new file mode 100644 index 00000000000..27e3ee4cffa --- /dev/null +++ b/packages/toasty/src/schema.rs @@ -0,0 +1,49 @@ +mod auto; +pub use auto::Auto; + +mod deferred; +pub use deferred::Deferred; + +mod embed; +pub use embed::{Embed, EmbedCreate}; + +mod field; +pub use field::{Document, Field, Scalar}; + +#[cfg(feature = "jiff")] +pub(crate) mod jiff; + +#[cfg(feature = "net")] +mod net; + +mod has_many; + +pub(crate) mod lazy_slot; + +mod load; +pub use load::Load; + +mod model; +pub use model::{Model, QueryMany, QueryOne, QueryOptionOne}; + +mod option; + +mod register; +pub use register::inventory; +pub use register::{DiscoverItem, generate_unique_id}; + +mod num; + +mod relation; +pub use relation::RelationManyField; + +mod relation_one; +pub use relation_one::RelationOneField; + +mod scope; +pub use scope::Scope; + +mod via; +pub use via::{ViaMany, ViaManyField, ViaPath, ViaTarget}; + +pub use toasty_core::schema::{app, app::ModelSet, db, diff, mapping}; diff --git a/packages/toasty/src/schema/auto.rs b/packages/toasty/src/schema/auto.rs new file mode 100644 index 00000000000..cce7ca2060d --- /dev/null +++ b/packages/toasty/src/schema/auto.rs @@ -0,0 +1,69 @@ +use super::Field; +use toasty_core::schema::app::AutoStrategy; + +#[diagnostic::on_unimplemented( + message = "Toasty cannot automatically set values for type `{Self}`", + label = "Toasty cannot automatically set values for this field", + note = "Is the field annotated with #[auto]? An embedded newtype is \ + auto-able only when its inner type implements `Auto`." +)] +/// A field type that supports automatic value generation. +/// +/// Types implementing this trait can be used with the `#[auto]` attribute on +/// model fields. The database or Toasty runtime assigns values for these +/// fields automatically when a new record is created. +/// +/// Built-in implementations: +/// +/// | Type | Strategy | +/// |---|---| +/// | Integer types (`i8`..`i64`, `u8`..`u64`, `isize`, `usize`) | Auto-increment | +/// | `uuid::Uuid` | UUID v7 | +pub trait Auto: Field { + /// The strategy the runtime uses to generate values for this type. + const STRATEGY: AutoStrategy; +} + +impl Auto for i8 { + const STRATEGY: AutoStrategy = AutoStrategy::Increment; +} + +impl Auto for i16 { + const STRATEGY: AutoStrategy = AutoStrategy::Increment; +} + +impl Auto for i32 { + const STRATEGY: AutoStrategy = AutoStrategy::Increment; +} + +impl Auto for i64 { + const STRATEGY: AutoStrategy = AutoStrategy::Increment; +} + +impl Auto for u8 { + const STRATEGY: AutoStrategy = AutoStrategy::Increment; +} + +impl Auto for u16 { + const STRATEGY: AutoStrategy = AutoStrategy::Increment; +} + +impl Auto for u32 { + const STRATEGY: AutoStrategy = AutoStrategy::Increment; +} + +impl Auto for u64 { + const STRATEGY: AutoStrategy = AutoStrategy::Increment; +} + +impl Auto for isize { + const STRATEGY: AutoStrategy = AutoStrategy::Increment; +} + +impl Auto for usize { + const STRATEGY: AutoStrategy = AutoStrategy::Increment; +} + +impl Auto for uuid::Uuid { + const STRATEGY: AutoStrategy = AutoStrategy::Uuid(toasty_core::schema::app::UuidVersion::V7); +} diff --git a/packages/toasty/src/schema/deferred.rs b/packages/toasty/src/schema/deferred.rs new file mode 100644 index 00000000000..78010a66589 --- /dev/null +++ b/packages/toasty/src/schema/deferred.rs @@ -0,0 +1,235 @@ +use super::{Field, Load}; +use crate::stmt::{self, Expr, IntoExpr}; +use toasty_core::{Error, schema::app::ModelSet, stmt::Value}; + +use std::fmt; + +/// A field whose value is not loaded by default. +/// +/// `Deferred` wraps a `T` whose underlying column is excluded from default +/// queries. After a normal load the value is unloaded and accessing it via +/// [`get`](Deferred::get) panics. Use `.include()` on the query to load the +/// value. +/// +/// `Deferred>` is supported when the column is nullable. +/// +/// Equality and hashing include the load state and, when loaded, the value. +/// Two unloaded fields compare equal. An unloaded field is unequal to a loaded +/// field, including a loaded `None`. These traits require the corresponding +/// trait on `T`. +/// +/// # Serde +/// +/// With the `serde` feature a loaded `Deferred` serializes transparently as +/// its inner `T`, and any present value deserializes back to the loaded state. +/// The unloaded state has no transparent encoding of its own — skip it on the +/// way out and default it on the way in: +/// +/// ```ignore +/// #[serde(skip_serializing_if = "Deferred::is_unloaded", default)] +/// notes: Deferred>, +/// ``` +/// +/// Serializing an unloaded field without `skip_serializing_if` emits `null`, +/// which does not round-trip (it reads back as loaded), so the annotation is +/// expected on every deferred field. +#[derive(Clone, PartialEq, Eq, Hash)] +pub struct Deferred { + value: Option>, +} + +impl Deferred { + /// Returns `true` if the field has not been loaded. + pub fn is_unloaded(&self) -> bool { + self.value.is_none() + } + + /// Returns a reference to the loaded value. + /// + /// # Panics + /// + /// Panics if the field has not been loaded. + #[track_caller] + pub fn get(&self) -> &T { + self.value.as_ref().expect("deferred field not loaded") + } + + /// Clears the loaded value, returning the field to the unloaded state. + pub fn unload(&mut self) { + self.value = None; + } + + /// Consumes this `Deferred` and returns the loaded value. + /// + /// # Panics + /// + /// Panics if the field has not been loaded. + #[track_caller] + pub fn into_inner(self) -> T { + *self.value.expect("deferred field not loaded") + } +} + +impl Default for Deferred { + fn default() -> Self { + Self { value: None } + } +} + +impl From for Deferred { + /// Constructs a loaded `Deferred` from a value. + /// + /// Used in struct literals for `#[derive(Embed)]` types that contain + /// deferred sub-fields, where the user supplies the inner value + /// directly: `Metadata { author, notes: "...".into() }`. + fn from(value: T) -> Self { + Self { + value: Some(Box::new(value)), + } + } +} + +/// Forwards the inner value's expression encoding so a `Deferred` field +/// can be spliced into any site that accepts a `T`. +/// +/// `Deferred` is a load-state wrapper, not a value type, so the produced +/// expression is `Expr` (the value), never `Expr>`. Panics +/// with the standard "deferred field not loaded" error if the value is in +/// the unloaded state — matching `.get()` / `.into_inner()`. +impl> IntoExpr for Deferred { + fn into_expr(self) -> Expr { + self.into_inner().into_expr() + } + + fn by_ref(&self) -> Expr { + self.get().by_ref() + } +} + +impl> IntoExpr for &Deferred { + fn into_expr(self) -> Expr { + self.get().by_ref() + } + + fn by_ref(&self) -> Expr { + self.get().by_ref() + } +} + +impl> Load for Deferred { + type Output = Self; + + fn ty() -> toasty_core::stmt::Type { + T::ty() + } + + fn load(value: toasty_core::stmt::Value) -> crate::Result { + // The lowering wraps a loaded deferred field in a 1-element record + // and emits a bare Null when unloaded, so the two states are + // distinguishable even when the inner value is NULL (i.e. the + // `Deferred>` case). + match value { + Value::Null => Ok(Self::default()), + Value::Record(mut record) if record.fields.len() == 1 => { + Ok(Self::from(T::load(record.fields.pop().unwrap())?)) + } + value => Err(Error::from_args(format_args!( + "deferred field decoder expected Null or single-field Record, got {value:?}" + ))), + } + } + + fn reload(target: &mut Self, value: toasty_core::stmt::Value) -> crate::Result<()> { + // The caller already supplied the value as part of the update, so the + // field becomes loaded regardless of its prior state — no follow-up + // fetch is needed to read what was just written. + // + // Updates send the assigned value back unwrapped, unlike the SELECT + // lowering which wraps a loaded deferred field in a 1-element record + // to distinguish it from the unloaded case. + target.value = Some(Box::new(T::load(value)?)); + Ok(()) + } +} + +impl Field for Deferred { + const REQUIRES_EXPLICIT_COLUMN_TYPE: bool = T::REQUIRES_EXPLICIT_COLUMN_TYPE; + + type ExprTarget = T::ExprTarget; + type Path = T::Path; + type ListPath = T::ListPath; + type Update<'a> = T::Update<'a>; + type Inner = T::Inner; + const NULLABLE: bool = T::NULLABLE; + const DEFERRED: bool = true; + + fn new_path(path: stmt::Path) -> Self::Path { + T::new_path(path) + } + + fn new_list_path( + path: stmt::Path>, + ) -> Self::ListPath { + T::new_list_path(path) + } + + fn new_update<'a>( + assignments: &'a mut toasty_core::stmt::Assignments, + projection: toasty_core::stmt::Projection, + ) -> Self::Update<'a> { + T::new_update(assignments, projection) + } + + fn field_ty( + storage_ty: Option, + ) -> toasty_core::schema::app::FieldTy { + T::field_ty(storage_ty) + } + + fn key_constraint(&self, _target: stmt::Path) -> Expr { + // Foreign keys cannot reference a deferred field. + unreachable!("Deferred fields cannot be used as foreign keys") + } + + fn register(model_set: &mut ModelSet) { + T::register(model_set); + } +} + +impl fmt::Debug for Deferred { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match &self.value { + Some(value) => value.fmt(f), + None => write!(f, ""), + } + } +} + +/// Serializes transparently: a loaded `Deferred` encodes exactly as the +/// inner `T`, an unloaded one as `null`. +/// +/// The unloaded `null` is not a distinct marker — a loaded `Deferred>` +/// holding `None` also encodes as `null` — so unloaded fields are meant to be +/// skipped rather than serialized. See the type-level docs. +#[cfg(feature = "serde")] +impl serde_core::Serialize for Deferred { + fn serialize(&self, serializer: S) -> Result + where + S: serde_core::Serializer, + { + self.value.serialize(serializer) + } +} + +/// Deserializes any present value — including `null` — as loaded, mirroring the +/// transparent [`Serialize`](serde_core::Serialize) impl above. An absent field +/// is left unloaded, but only via `#[serde(default)]`; see the type-level docs. +#[cfg(feature = "serde")] +impl<'de, T: serde_core::Deserialize<'de>> serde_core::Deserialize<'de> for Deferred { + fn deserialize(deserializer: D) -> Result + where + D: serde_core::Deserializer<'de>, + { + Ok(Self::from(T::deserialize(deserializer)?)) + } +} diff --git a/packages/toasty/src/schema/embed.rs b/packages/toasty/src/schema/embed.rs new file mode 100644 index 00000000000..ab5b4308483 --- /dev/null +++ b/packages/toasty/src/schema/embed.rs @@ -0,0 +1,56 @@ +use crate::stmt::Path; +use toasty_core::schema::app::{self, ModelId}; + +/// Trait for embedded types that are flattened into their parent model's table. +/// +/// Embedded types don't have their own tables or primary keys. They can't be +/// queried independently or used as relation targets. Their fields are flattened +/// into the parent model's table columns. +/// +/// Embedded types are never registered directly by the user — they are +/// discovered transitively through the fields of the models (and other embeds) +/// that contain them, via [`Field::register`](super::Field::register). +pub trait Embed { + /// Unique identifier for this embedded type within the schema. + /// + /// Identifiers are *not* unique across schemas. + fn id() -> ModelId; + + /// Returns the schema definition for this embedded type. + fn schema() -> app::Model; + + /// An identity [`Path`] rooted at this embedded type. + /// + /// This is how `Path` recovers an embedded type's [`ModelId`] without a + /// dedicated registration trait: the type supplies its own id via + /// [`id`](Self::id). + fn path_root() -> Path + where + Self: Sized, + { + Path::from_model_id(Self::id()) + } + + /// A [`Path`] from this embedded type to the field at `index`. + fn path_field(index: usize) -> Path + where + Self: Sized, + { + Path::field_at(Self::id(), index) + } +} + +/// An embedded type that can be constructed as a write value through a +/// builder: an enum with data-carrying variants. +/// +/// `create!` and `update!` expand a variant literal +/// (`Owner::Human { human: &alice }`) into a chain on +/// [`Create`](Self::Create): `::create().human().human(&alice)`. +pub trait EmbedCreate: Embed { + /// The variant-selection builder (`OwnerCreate`), with one method per + /// data-carrying variant. + type Create; + + /// Returns the variant-selection builder. + fn create() -> Self::Create; +} diff --git a/packages/toasty/src/schema/field.rs b/packages/toasty/src/schema/field.rs new file mode 100644 index 00000000000..13400ca2fbe --- /dev/null +++ b/packages/toasty/src/schema/field.rs @@ -0,0 +1,576 @@ +use super::{Embed, Load}; +use crate::stmt::{self, Expr, List}; +use toasty_core::schema::{ + app::{Model, ModelSet}, + db, +}; + +/// Schema and runtime information for a field type. +/// +/// This trait captures the information needed to register a field's type in the +/// app schema (nullability, [`FieldTy`](toasty_core::schema::app::FieldTy)) as +/// well as runtime helpers for building field paths and update builders. +/// It is used by the `schema()` implementation that the macro expands for +/// [`Model`](super::Model) and [`Embed`](super::Embed) types. +#[diagnostic::on_unimplemented( + message = "`{Self}` cannot be used as a field type", + label = "this field's type is not storable", + note = "If `{Self}` is a Toasty model, this is a relation field and must be annotated \ + with `#[belongs_to]`, `#[has_one]`, or `#[has_many]`.", + note = "Otherwise `{Self}` must be a supported primitive (string, integer, bool, uuid, \ + …) or a wrapper around one (`Option<_>`, `Vec<_>`, `Box<_>`, `Arc<_>`, `Rc<_>`)." +)] +pub trait Field: Load { + /// Whether this field type requires `#[column(type = ...)]`. + /// + /// Generated model code evaluates this constant when a field omits an + /// explicit column type. Wrapper implementations propagate the value from + /// their inner field type, which lets the Rust type checker resolve aliases + /// and nested wrappers without the derive macro inspecting type syntax. + #[doc(hidden)] + const REQUIRES_EXPLICIT_COLUMN_TYPE: bool = false; + + /// The expression-level type of this field. + /// + /// This drives both the field's path target (the second parameter of + /// the underlying `Path`) **and** the `IntoExpr`/`Assign` bound on + /// generated setters. For most types this is `Self`. For + /// `Vec` it is [`List`] so the field accessor returns a + /// `Path<_, List>` (giving access to `contains`, `is_superset`, + /// `len`, …) and create/update setters accept any + /// `impl IntoExpr>` (Vec, slice, array literal, …). + /// + /// Decoupling this from `Self` lets the accessor macro construct the + /// path with the right target type from the start, so `new_path` is + /// identity for every primitive impl. + type ExprTarget; + + /// The type returned when accessing this field from a Fields struct. + /// For primitives, this is Path. + /// For embedded types, this is {Type}Fields. + type Path; + + /// The type returned when accessing this field from a list Fields struct. + /// For primitives, this is Path>. + /// For embedded types, this is {Type}ListFields. + type ListPath; + + /// The type of the update builder for this field. + /// For embedded types, this is {Type}Update<'a>. + /// For primitives, this will be {Type}Update<'a> once implemented. + type Update<'a>; + + /// The unwrapped type used for foreign key filter expressions. + /// Primitives map to themselves. Wrappers like `Option`, `Box`, + /// `Arc`, and `Rc` map to `T::Inner`. + type Inner; + + /// Whether or not the type is nullable + const NULLABLE: bool = false; + + /// Whether this field is omitted from default loads. + const DEFERRED: bool = false; + + /// Build a field path from a raw path of the field's + /// [`Self::ExprTarget`]. + /// + /// For primitives, returns the path as-is. + /// For embedded types, wraps the path in a Fields struct. + fn new_path(path: stmt::Path) -> Self::Path + where + Self: Sized; + + /// Build a list field path from a raw path. + /// For primitives, returns the path as-is. + /// For embedded types, wraps the path in a ListFields struct. + fn new_list_path( + path: stmt::Path>, + ) -> Self::ListPath + where + Self: Sized; + + /// Build an update builder from assignments and a projection. + /// For primitives, this returns `()` (no builder). + /// For embedded types, this is overridden to construct the {Type}Update builder. + fn new_update<'a>( + _assignments: &'a mut toasty_core::stmt::Assignments, + _projection: toasty_core::stmt::Projection, + ) -> Self::Update<'a>; + + /// Returns the app-level field type for this primitive. + /// Default implementation returns a Primitive field type. + /// Embedded types override this to return Embedded field type. + fn field_ty( + storage_ty: Option, + ) -> toasty_core::schema::app::FieldTy { + toasty_core::schema::app::FieldTy::Primitive(toasty_core::schema::app::FieldPrimitive { + ty: Self::ty(), + storage_ty, + serialize: None, + }) + } + + /// Build a boolean filter expression comparing this field value against + /// a target path. Used by generated belongs-to accessors to build FK + /// lookups. + /// + /// For `T`, returns `target.eq(self)`. For `Option`, returns + /// `target.eq(inner)` when `Some`, or `false` when `None`. + fn key_constraint(&self, target: stmt::Path) -> Expr; + + /// Register any models referenced by this field type into the given + /// [`ModelSet`]. + /// + /// The default implementation is a no-op, suitable for primitive types. + /// Embedded types override this to register themselves and recurse into + /// their own fields, which is how embeds are discovered transitively from + /// the models that contain them. + fn register(_model_set: &mut ModelSet) {} +} + +macro_rules! impl_field_primitive { + ($ty:ty) => { + impl Field for $ty { + type ExprTarget = Self; + type Path = stmt::Path; + type ListPath = stmt::Path>; + type Update<'a> = (); + type Inner = Self; + + fn new_path(path: stmt::Path) -> Self::Path { + path + } + + fn new_list_path( + path: stmt::Path>, + ) -> Self::ListPath { + path + } + + fn new_update<'a>( + _assignments: &'a mut toasty_core::stmt::Assignments, + _projection: toasty_core::stmt::Projection, + ) -> Self::Update<'a> { + } + + fn key_constraint( + &self, + target: stmt::Path, + ) -> Expr { + target.eq(self) + } + } + }; +} + +impl_field_primitive!(String); +impl_field_primitive!(uuid::Uuid); +impl_field_primitive!(bool); +impl_field_primitive!(isize); +impl_field_primitive!(usize); + +impl Field for Vec { + type ExprTarget = Self; + type Path = stmt::Path; + type ListPath = stmt::Path>; + type Update<'a> = (); + type Inner = Self; + + fn new_path(path: stmt::Path) -> Self::Path { + path + } + + fn new_list_path( + path: stmt::Path>, + ) -> Self::ListPath { + path + } + + fn new_update<'a>( + _assignments: &'a mut toasty_core::stmt::Assignments, + _projection: toasty_core::stmt::Projection, + ) -> Self::Update<'a> { + } + + fn field_ty( + storage_ty: Option, + ) -> toasty_core::schema::app::FieldTy { + toasty_core::schema::app::FieldTy::Primitive(toasty_core::schema::app::FieldPrimitive { + ty: toasty_core::stmt::Type::Bytes, + storage_ty, + serialize: None, + }) + } + + fn key_constraint(&self, target: stmt::Path) -> Expr { + target.eq(self) + } +} + +/// A `Vec` of embedded structs (`T: Embed`) is a `#[document]` collection — +/// `Primitive(List(Model(T::id())))`, stored by the schema builder as a single +/// JSON array of objects. The `field_ty` override below derives the storage +/// type configured for unit-enum discriminants; for struct embeds it matches +/// the trait default. +/// +/// This is the only *blanket* `Field for Vec<_>` impl. A blanket +/// `impl Field for Vec` cannot coexist with it: the compiler +/// can't prove `Scalar` and `Embed` are disjoint (`Scalar` is public, so a +/// downstream embed type could implement it), so the two would overlap. The +/// scalar element types therefore get *concrete* `Vec<$t>` impls (via +/// [`impl_scalar!`] below), which the orphan rule proves disjoint from this +/// blanket — exactly as the concrete [`Vec`] impl above already is. +impl Field for Vec +where + T: Embed + Field, +{ + type ExprTarget = List; + type Path = stmt::Path>; + type ListPath = stmt::Path>; + type Update<'a> = (); + type Inner = Self; + + fn new_path(path: stmt::Path>) -> Self::Path { + path + } + + fn new_list_path( + path: stmt::Path>, + ) -> Self::ListPath { + path + } + + fn new_update<'a>( + _assignments: &'a mut toasty_core::stmt::Assignments, + _projection: toasty_core::stmt::Projection, + ) -> Self::Update<'a> { + } + + /// Lifts a field-level element override into a list storage type, then + /// falls back to the unit enum's discriminant storage. + fn field_ty( + storage_ty: Option, + ) -> toasty_core::schema::app::FieldTy { + let storage_ty = storage_ty.map(db::Type::list).or_else(|| { + let Model::EmbeddedEnum(embed) = ::schema() else { + return None; + }; + + if embed.has_data_variants() { + return None; + } + + embed.discriminant.storage_ty.map(db::Type::list) + }); + toasty_core::schema::app::FieldTy::Primitive(toasty_core::schema::app::FieldPrimitive { + ty: ::ty(), + storage_ty, + serialize: None, + }) + } + + fn key_constraint(&self, _target: stmt::Path) -> Expr { + unreachable!("Vec fields cannot be used as foreign-key targets"); + } + + fn register(model_set: &mut ModelSet) { + ::register(model_set); + } +} + +/// Marks scalar (non-composite) types that are valid as the element type of a +/// model-level `Vec` collection field. The trait is implemented for every +/// primitive `Field` (string, integers, uuid, …); `Vec` is intentionally +/// not a [`Scalar`] so that bytes keep their existing scalar storage and don't +/// get re-routed through the collection-field path. +pub trait Scalar {} + +/// Marks types that `#[document]` storage accepts: a `#[derive(Embed)]` +/// struct, or a `Vec` of them. +/// +/// The `#[document]` attribute resolves the field's app-level type through +/// this trait, so applying it to anything else — a scalar, an enum embed, a +/// `Vec` — is a compile error rather than a silently ignored +/// attribute. The schema builder re-validates the resolved shape at build +/// time, but only a trait bound can catch the mistake at compile time. +#[diagnostic::on_unimplemented( + message = "`{Self}` cannot use `#[document]` storage", + label = "`#[document]` requires a `#[derive(Embed)]` struct or a `Vec` of them", + note = "enum embeds and scalar collections do not yet support document storage; \ + remove the `#[document]` attribute" +)] +pub trait Document: Field { + /// The app-level type of the document column: `Model(id)` for a bare + /// embed, `List(Model(id))` for a collection. + fn document_ty() -> toasty_core::stmt::Type { + ::ty() + } +} + +/// A `Vec` of document-capable embeds is itself document-capable — the +/// collection is stored as one JSON array of objects. Mirrors the blanket +/// `Field for Vec` impl above. +impl Document for Vec where T: Document + Embed + Field {} + +/// Registers a scalar collection element type. For each `$t` this generates +/// both `impl Scalar for $t` (opting it into the collection-path operators) +/// and a concrete `impl Field for Vec<$t>` (the collection field impl). The +/// `Vec` impl is per-type rather than a blanket `impl` so it stays +/// coherent with the blanket `impl Field for Vec` above — see that +/// impl's docs. A single list keeps the two in sync. +macro_rules! impl_scalar { + ( $( $t:ty ),* $(,)? ) => { + $( + impl Scalar for $t {} + + impl Field for Vec<$t> { + // The `List` marker is the expression target so the field's + // path is `Path<_, List>` (giving `contains`, `is_superset`, + // `len`, …) and create/update setters bind through + // `IntoExpr>` (Vec, slice, array literal). `new_path` + // stays identity; `field_ty` uses the default + // `Primitive(::ty())` = `Primitive(List($t))`. + type ExprTarget = List<$t>; + type Path = stmt::Path>; + type ListPath = stmt::Path>; + type Update<'a> = (); + type Inner = Self; + + fn new_path(path: stmt::Path>) -> Self::Path { + path + } + + fn new_list_path( + path: stmt::Path>, + ) -> Self::ListPath { + path + } + + fn new_update<'a>( + _assignments: &'a mut toasty_core::stmt::Assignments, + _projection: toasty_core::stmt::Projection, + ) -> Self::Update<'a> { + } + + fn key_constraint( + &self, + _target: stmt::Path, + ) -> Expr { + // A foreign key cannot reference a `Vec` field — + // collections aren't valid key types. + unreachable!("Vec fields cannot be used as foreign-key targets"); + } + } + )* + }; +} + +impl_scalar!( + String, + bool, + i8, + i16, + i32, + i64, + u16, + u32, + u64, + isize, + usize, + f32, + f64, + uuid::Uuid +); + +#[cfg(feature = "rust_decimal")] +impl_scalar!(rust_decimal::Decimal); + +#[cfg(feature = "bigdecimal")] +impl_scalar!(bigdecimal::BigDecimal); + +#[cfg(feature = "jiff")] +impl_scalar!( + jiff::Timestamp, + jiff::Zoned, + jiff::civil::Date, + jiff::civil::Time, + jiff::civil::DateTime, + jiff::tz::TimeZone +); + +#[cfg(feature = "net")] +impl_scalar!(stmt::IpCidr, stmt::IpInet, stmt::MacAddr6, stmt::MacAddr8); + +impl Field for Option { + const REQUIRES_EXPLICIT_COLUMN_TYPE: bool = T::REQUIRES_EXPLICIT_COLUMN_TYPE; + + type ExprTarget = Self; + type Path = stmt::Path; + type ListPath = stmt::Path>; + type Update<'a> = (); + type Inner = T::Inner; + const NULLABLE: bool = true; + + fn new_path(path: stmt::Path) -> Self::Path { + path + } + + fn new_list_path( + path: stmt::Path>, + ) -> Self::ListPath { + path + } + + fn new_update<'a>( + _assignments: &'a mut toasty_core::stmt::Assignments, + _projection: toasty_core::stmt::Projection, + ) -> Self::Update<'a> { + } + + /// Delegate the field-type description to `T` so wrappers carrying + /// schema-relevant metadata (e.g. `Json::serialize = Some(Json)`) + /// propagate through `Option<_>` instead of getting flattened to the + /// default `serialize: None`. + fn field_ty( + storage_ty: Option, + ) -> toasty_core::schema::app::FieldTy { + T::field_ty(storage_ty) + } + + fn key_constraint(&self, target: stmt::Path) -> Expr { + match self { + Some(value) => T::key_constraint(value, target), + None => Expr::from_value(toasty_core::stmt::Value::Bool(false)), + } + } + + fn register(model_set: &mut ModelSet) { + T::register(model_set); + } +} + +impl Field for std::sync::Arc { + const REQUIRES_EXPLICIT_COLUMN_TYPE: bool = T::REQUIRES_EXPLICIT_COLUMN_TYPE; + + type ExprTarget = Self; + type Path = stmt::Path; + type ListPath = stmt::Path>; + type Update<'a> = (); + type Inner = T::Inner; + + fn new_path(path: stmt::Path) -> Self::Path { + path + } + + fn new_list_path( + path: stmt::Path>, + ) -> Self::ListPath { + path + } + + fn new_update<'a>( + _assignments: &'a mut toasty_core::stmt::Assignments, + _projection: toasty_core::stmt::Projection, + ) -> Self::Update<'a> { + } + + fn field_ty( + storage_ty: Option, + ) -> toasty_core::schema::app::FieldTy { + T::field_ty(storage_ty) + } + + fn key_constraint(&self, target: stmt::Path) -> Expr { + T::key_constraint(self, target) + } + + fn register(model_set: &mut ModelSet) { + T::register(model_set); + } +} + +impl Field for std::rc::Rc { + const REQUIRES_EXPLICIT_COLUMN_TYPE: bool = T::REQUIRES_EXPLICIT_COLUMN_TYPE; + + type ExprTarget = Self; + type Path = stmt::Path; + type ListPath = stmt::Path>; + type Update<'a> = (); + type Inner = T::Inner; + + fn new_path(path: stmt::Path) -> Self::Path { + path + } + + fn new_list_path( + path: stmt::Path>, + ) -> Self::ListPath { + path + } + + fn new_update<'a>( + _assignments: &'a mut toasty_core::stmt::Assignments, + _projection: toasty_core::stmt::Projection, + ) -> Self::Update<'a> { + } + + fn field_ty( + storage_ty: Option, + ) -> toasty_core::schema::app::FieldTy { + T::field_ty(storage_ty) + } + + fn key_constraint(&self, target: stmt::Path) -> Expr { + T::key_constraint(self, target) + } + + fn register(model_set: &mut ModelSet) { + T::register(model_set); + } +} + +impl Field for Box { + const REQUIRES_EXPLICIT_COLUMN_TYPE: bool = T::REQUIRES_EXPLICIT_COLUMN_TYPE; + + type ExprTarget = Self; + type Path = stmt::Path; + type ListPath = stmt::Path>; + type Update<'a> = (); + type Inner = T::Inner; + + fn new_path(path: stmt::Path) -> Self::Path { + path + } + + fn new_list_path( + path: stmt::Path>, + ) -> Self::ListPath { + path + } + + fn new_update<'a>( + _assignments: &'a mut toasty_core::stmt::Assignments, + _projection: toasty_core::stmt::Projection, + ) -> Self::Update<'a> { + } + + fn field_ty( + storage_ty: Option, + ) -> toasty_core::schema::app::FieldTy { + T::field_ty(storage_ty) + } + + fn key_constraint(&self, target: stmt::Path) -> Expr { + T::key_constraint(self, target) + } + + fn register(model_set: &mut ModelSet) { + T::register(model_set); + } +} + +#[cfg(feature = "rust_decimal")] +impl_field_primitive!(rust_decimal::Decimal); + +#[cfg(feature = "bigdecimal")] +impl_field_primitive!(bigdecimal::BigDecimal); diff --git a/packages/toasty/src/schema/has_many.rs b/packages/toasty/src/schema/has_many.rs new file mode 100644 index 00000000000..8178b1168a7 --- /dev/null +++ b/packages/toasty/src/schema/has_many.rs @@ -0,0 +1,65 @@ +use super::{Deferred, Load, Model, RelationManyField}; + +use toasty_core::schema::Name; +use toasty_core::schema::app::{self, FieldId, FieldTy, ModelId}; +use toasty_core::stmt; + +impl RelationManyField for Vec { + type Target = M; + + const DEFERRED: bool = false; + + fn reload(target: &mut Self, value: stmt::Value) -> crate::Result<()> { + ::reload(target, value) + } + + fn many_relation_field_ty( + singular: Name, + pair: Option, + via: Option, + ) -> FieldTy { + many_relation_field_ty::(singular, pair, via) + } +} + +impl RelationManyField for Deferred> { + type Target = M; + + const DEFERRED: bool = true; + + fn reload(target: &mut Self, _value: stmt::Value) -> crate::Result<()> { + target.unload(); + Ok(()) + } + + fn many_relation_field_ty( + singular: Name, + pair: Option, + via: Option, + ) -> FieldTy { + many_relation_field_ty::(singular, pair, via) + } +} + +fn many_relation_field_ty( + singular: Name, + pair: Option, + via: Option, +) -> FieldTy { + let target = ::id(); + let expr_ty = stmt::Type::List(Box::new(stmt::Type::Model(target))); + let cardinality = app::Cardinality::Many { singular }; + + match via { + Some(path) => FieldTy::Via(app::Via::new(target, expr_ty, cardinality, path, None)), + None => FieldTy::Has(app::Has { + target, + expr_ty, + cardinality, + pair_id: pair.unwrap_or(FieldId { + model: ModelId(usize::MAX), + index: usize::MAX, + }), + }), + } +} diff --git a/packages/toasty/src/schema/jiff.rs b/packages/toasty/src/schema/jiff.rs new file mode 100644 index 00000000000..c4961c1f96a --- /dev/null +++ b/packages/toasty/src/schema/jiff.rs @@ -0,0 +1,152 @@ +use super::{Field, Load}; +use crate::stmt::{Expr, List, Path}; +use toasty_core::{ + Result, + stmt::{Type, Value}, +}; + +macro_rules! impl_jiff_field { + ($ty:ty, $name:ident, $lit:literal) => { + impl Load for $ty { + type Output = Self; + + fn ty() -> Type { + Type::$name + } + + fn load(value: Value) -> Result { + match value { + Value::$name(v) => Ok(v), + _ => Err(toasty_core::Error::type_conversion(value, $lit)), + } + } + + fn reload(target: &mut Self, value: Value) -> Result<()> { + *target = Self::load(value)?; + Ok(()) + } + } + + impl Field for $ty { + type ExprTarget = Self; + type Path = Path; + type ListPath = Path>; + type Update<'a> = (); + type Inner = Self; + + fn new_path(path: Path) -> Self::Path { + path + } + + fn new_list_path( + path: Path>, + ) -> Self::ListPath { + path + } + + fn new_update<'a>( + _assignments: &'a mut toasty_core::stmt::Assignments, + _projection: toasty_core::stmt::Projection, + ) -> Self::Update<'a> { + } + + fn key_constraint(&self, target: Path) -> Expr { + target.eq(self) + } + } + }; +} + +impl_jiff_field!(jiff::Timestamp, Timestamp, "jiff::Timestamp"); +impl_jiff_field!(jiff::Zoned, Zoned, "jiff::Zoned"); +impl_jiff_field!(jiff::civil::Date, Date, "jiff::civil::Date"); +impl_jiff_field!(jiff::civil::Time, Time, "jiff::civil::Time"); +impl_jiff_field!(jiff::civil::DateTime, DateTime, "jiff::civil::DateTime"); + +/// No backend has a time zone column type, so a `TimeZone` has no `stmt::Type` +/// of its own — it is a `String` end to end, and these two functions are the +/// only place the encoding is defined. +/// +/// The encoding is the RFC 9557 form a `Zoned` carries in its bracketed +/// annotation: an IANA identifier, a fixed offset, a POSIX TZ string, or +/// `Etc/Unknown`. jiff gives `TimeZone` neither `Display` nor `FromStr`. +static PRINTER: ::jiff::fmt::temporal::DateTimePrinter = + ::jiff::fmt::temporal::DateTimePrinter::new(); + +static PARSER: ::jiff::fmt::temporal::DateTimeParser = ::jiff::fmt::temporal::DateTimeParser::new(); + +/// Printing canonicalizes `america/NEW_YORK` to `America/New_York`, so equal +/// zones store identical text. +/// +/// Fails only for a zone with no text form, which jiff produces just for a +/// `TimeZone::system()` read from a non-symlinked `/etc/localtime`. +pub(crate) fn time_zone_to_text(tz: &::jiff::tz::TimeZone) -> Result { + let mut text = String::new(); + + PRINTER.print_time_zone(tz, &mut text).map_err(|_| { + toasty_core::Error::unsupported_feature( + "this `jiff::tz::TimeZone` has no IANA identifier, fixed offset, \ + or POSIX rule string, so it cannot be stored", + ) + })?; + + Ok(text) +} + +/// IANA identifiers resolve against jiff's global time zone database, so a +/// zone renamed between tzdb releases fails to load on a host carrying only +/// the other name. +fn time_zone_from_text(text: &str) -> Result<::jiff::tz::TimeZone> { + PARSER + .parse_time_zone(text) + .map_err(|e| toasty_core::Error::from_args(format_args!("invalid time zone {text:?}: {e}"))) +} + +impl Load for ::jiff::tz::TimeZone { + type Output = Self; + + fn ty() -> Type { + Type::String + } + + fn load(value: Value) -> Result { + match value { + Value::String(text) => time_zone_from_text(&text), + _ => Err(toasty_core::Error::type_conversion( + value, + "jiff::tz::TimeZone", + )), + } + } + + fn reload(target: &mut Self, value: Value) -> Result<()> { + *target = Self::load(value)?; + Ok(()) + } +} + +impl Field for ::jiff::tz::TimeZone { + type ExprTarget = Self; + type Path = Path; + type ListPath = Path>; + type Update<'a> = (); + type Inner = Self; + + fn new_path(path: Path) -> Self::Path { + path + } + + fn new_list_path(path: Path>) -> Self::ListPath { + path + } + + fn new_update<'a>( + _assignments: &'a mut toasty_core::stmt::Assignments, + _projection: toasty_core::stmt::Projection, + ) -> Self::Update<'a> { + } + + fn key_constraint(&self, target: Path) -> Expr { + target.eq(self) + } +} diff --git a/packages/toasty/src/schema/lazy_slot.rs b/packages/toasty/src/schema/lazy_slot.rs new file mode 100644 index 00000000000..88e289bd6d6 --- /dev/null +++ b/packages/toasty/src/schema/lazy_slot.rs @@ -0,0 +1,6 @@ +use toasty_core::stmt; + +/// Wrap a loaded value to distinguish loaded NULL from an unloaded field. +pub(crate) fn loaded_expr(value: stmt::Expr) -> stmt::Expr { + stmt::Expr::record([value]) +} diff --git a/packages/toasty/src/schema/load.rs b/packages/toasty/src/schema/load.rs new file mode 100644 index 00000000000..dae8cb06101 --- /dev/null +++ b/packages/toasty/src/schema/load.rs @@ -0,0 +1,333 @@ +use crate::stmt::List; +use toasty_core::Error; +use toasty_core::stmt; + +/// Load an instance of a type from a [`Value`][stmt::Value]. +/// +/// The value is expected to be a `Value::Record` containing the type's fields. +/// This trait is implemented by both root models and any other types that can +/// be deserialized from the database value representation. +/// +/// The associated `Output` type allows marker types (like `List`) to +/// specify a concrete return type. For sized types `Output = Self`; for +/// `List`, `Output = Vec`. +pub trait Load { + /// The concrete type produced by [`load`](Self::load). + /// + /// For sized types this is `Self`. For marker types like [`List`](crate::stmt::List), + /// it is `Vec`. + type Output; + + /// Returns the [`stmt::Type`] that describes values of this type. + fn ty() -> stmt::Type; + + /// Returns the [`stmt::Type`] used when this type appears as a relation + /// target. The default delegates to [`ty()`](Load::ty). + fn ty_relation() -> stmt::Type { + Self::ty() + } + + /// Deserialize a database [`Value`](stmt::Value) into `Self::Output`. + /// + /// Returns an error if the value cannot be converted to this type. + fn load(value: stmt::Value) -> Result; + + /// Deserialize a value that was loaded as a relation target. + /// + /// The default delegates to [`load`](Self::load). Override this when + /// relation values use a different encoding. + fn load_relation(value: stmt::Value) -> Result { + Self::load(value) + } + + /// Reload the value in-place from a value returned by the database. + /// + /// The value may be a `SparseRecord` for partial embedded updates, in which + /// case only the specified fields should be updated. Embedded types must + /// override this method to handle partial updates correctly. + /// + /// Takes `&mut Self::Output` rather than `&mut self` so that wrapper types + /// like `Option` can implement reload generically regardless of whether + /// `T::Output == T`. + /// + /// The default implementation panics. Types that support reloading (i.e., + /// types that implement [`Field`]) should override this. + fn reload(target: &mut Self::Output, value: stmt::Value) -> Result<(), Error> { + let _ = (target, value); + unimplemented!("reload is not supported for this type") + } +} + +impl Load for () { + type Output = (); + + fn ty() -> stmt::Type { + stmt::Type::Unit + } + + fn load(_value: stmt::Value) -> Result { + Ok(()) + } +} + +impl> Load for Vec { + type Output = Vec; + + fn ty() -> stmt::Type { + stmt::Type::list(T::ty()) + } + + fn load(value: stmt::Value) -> Result { + match value { + stmt::Value::List(items) => items.into_iter().map(T::load).collect(), + // Records are produced by dynamic batch queries (Vec/array inputs) + // where each field in the record is one query's result. + stmt::Value::Record(record) => record.into_iter().map(T::load).collect(), + // Bytes are a compact representation; load each byte individually. + stmt::Value::Bytes(bytes) => bytes + .into_iter() + .map(|b| T::load(stmt::Value::U8(b))) + .collect(), + _ => Err(Error::type_conversion(value, "Vec")), + } + } + + fn reload(target: &mut Self::Output, value: stmt::Value) -> Result<(), Error> { + *target = Self::load(value)?; + Ok(()) + } +} + +/// List type encoding: `List` loads as `Vec`. +impl Load for List { + type Output = Vec; + + fn ty() -> stmt::Type { + stmt::Type::list(M::ty()) + } + + fn load(value: stmt::Value) -> Result { + match value { + stmt::Value::List(items) => items.into_iter().map(M::load).collect(), + stmt::Value::Record(record) => record.into_iter().map(M::load).collect(), + _ => Err(Error::type_conversion(value, "List")), + } + } +} + +macro_rules! impl_load_for_tuple { + ( $( $T:ident ),+ ; $( $idx:tt ),+ ) => { + impl< $( $T: Load ),+ > Load for ( $( $T, )+ ) { + type Output = ( $( $T::Output, )+ ); + + fn ty() -> stmt::Type { + stmt::Type::Record(vec![ $( $T::ty() ),+ ]) + } + + fn load(value: stmt::Value) -> Result { + match value { + stmt::Value::Record(mut record) => Ok(( + $( $T::load(record[$idx].take())?, )+ + )), + _ => Err(Error::type_conversion(value, "tuple")), + } + } + } + }; +} + +impl_load_for_tuple!(A, B; 0, 1); +impl_load_for_tuple!(A, B, C; 0, 1, 2); +impl_load_for_tuple!(A, B, C, D; 0, 1, 2, 3); +impl_load_for_tuple!(A, B, C, D, E; 0, 1, 2, 3, 4); +impl_load_for_tuple!(A, B, C, D, E, F; 0, 1, 2, 3, 4, 5); +impl_load_for_tuple!(A, B, C, D, E, F, G; 0, 1, 2, 3, 4, 5, 6); +impl_load_for_tuple!(A, B, C, D, E, F, G, H; 0, 1, 2, 3, 4, 5, 6, 7); + +impl Load for String { + type Output = Self; + + fn ty() -> stmt::Type { + stmt::Type::String + } + + fn load(value: stmt::Value) -> Result { + match value { + stmt::Value::String(v) => Ok(v), + _ => Err(Error::type_conversion(value, "String")), + } + } + + fn reload(target: &mut Self, value: stmt::Value) -> Result<(), Error> { + *target = Self::load(value)?; + Ok(()) + } +} + +impl Load for uuid::Uuid { + type Output = Self; + + fn ty() -> stmt::Type { + stmt::Type::Uuid + } + + fn load(value: stmt::Value) -> Result { + match value { + stmt::Value::Uuid(v) => Ok(v), + _ => Err(Error::type_conversion(value, "uuid::Uuid")), + } + } + + fn reload(target: &mut Self, value: stmt::Value) -> Result<(), Error> { + *target = Self::load(value)?; + Ok(()) + } +} + +impl Load for bool { + type Output = Self; + + fn ty() -> stmt::Type { + stmt::Type::Bool + } + + fn load(value: stmt::Value) -> Result { + match value { + stmt::Value::Bool(v) => Ok(v), + _ => Err(Error::type_conversion(value, "bool")), + } + } + + fn reload(target: &mut Self, value: stmt::Value) -> Result<(), Error> { + *target = Self::load(value)?; + Ok(()) + } +} + +impl> Load for std::sync::Arc { + type Output = Self; + + fn ty() -> stmt::Type { + T::ty() + } + + fn load(value: stmt::Value) -> Result { + ::load(value).map(std::sync::Arc::new) + } + + fn reload(target: &mut Self, value: stmt::Value) -> Result<(), Error> { + *target = Self::load(value)?; + Ok(()) + } +} + +impl> Load for std::rc::Rc { + type Output = Self; + + fn ty() -> stmt::Type { + T::ty() + } + + fn load(value: stmt::Value) -> Result { + ::load(value).map(std::rc::Rc::new) + } + + fn reload(target: &mut Self, value: stmt::Value) -> Result<(), Error> { + *target = Self::load(value)?; + Ok(()) + } +} + +impl> Load for Box { + type Output = Self; + + fn ty() -> stmt::Type { + T::ty() + } + + fn load(value: stmt::Value) -> Result { + ::load(value).map(Box::new) + } + + fn reload(target: &mut Self, value: stmt::Value) -> Result<(), Error> { + *target = Self::load(value)?; + Ok(()) + } +} + +// Pointer-sized integers map to fixed-size types internally +impl Load for isize { + type Output = Self; + + fn ty() -> stmt::Type { + stmt::Type::I64 + } + + fn load(value: stmt::Value) -> Result { + value.try_into() + } + + fn reload(target: &mut Self, value: stmt::Value) -> Result<(), Error> { + *target = Self::load(value)?; + Ok(()) + } +} + +impl Load for usize { + type Output = Self; + + fn ty() -> stmt::Type { + stmt::Type::U64 + } + + fn load(value: stmt::Value) -> Result { + value.try_into() + } + + fn reload(target: &mut Self, value: stmt::Value) -> Result<(), Error> { + *target = Self::load(value)?; + Ok(()) + } +} + +#[cfg(feature = "rust_decimal")] +impl Load for rust_decimal::Decimal { + type Output = Self; + + fn ty() -> stmt::Type { + stmt::Type::Decimal + } + + fn load(value: stmt::Value) -> Result { + match value { + stmt::Value::Decimal(v) => Ok(v), + _ => Err(Error::type_conversion(value, "rust_decimal::Decimal")), + } + } + + fn reload(target: &mut Self, value: stmt::Value) -> Result<(), Error> { + *target = Self::load(value)?; + Ok(()) + } +} + +#[cfg(feature = "bigdecimal")] +impl Load for bigdecimal::BigDecimal { + type Output = Self; + + fn ty() -> stmt::Type { + stmt::Type::BigDecimal + } + + fn load(value: stmt::Value) -> Result { + match value { + stmt::Value::BigDecimal(v) => Ok(v), + _ => Err(Error::type_conversion(value, "bigdecimal::BigDecimal")), + } + } + + fn reload(target: &mut Self, value: stmt::Value) -> Result<(), Error> { + *target = Self::load(value)?; + Ok(()) + } +} diff --git a/packages/toasty/src/schema/model.rs b/packages/toasty/src/schema/model.rs new file mode 100644 index 00000000000..59dffe0a82b --- /dev/null +++ b/packages/toasty/src/schema/model.rs @@ -0,0 +1,155 @@ +use super::Load; +use crate::stmt::{Expr, IntoExpr, IntoInsert, List, Path}; + +use toasty_core::schema::app::{self, FieldId, ModelId, ModelSet}; + +/// Trait for root models that map to database tables and can be queried. +/// +/// Root models have primary keys, can be queried independently, and support +/// full CRUD operations. They carry queryability and deserialization +/// capabilities along with the relation-target metadata +/// that the [`RelationManyField`](super::RelationManyField) and +/// [`RelationOneField`](super::RelationOneField) traits project through when +/// describing a field that references this model. +pub trait Model: Load + Sized { + /// Query builder for this model, parameterized by what it executes to. + /// + /// The single generic `T` selects the result shape: + /// + /// - `Query>` — list query, executes to `Vec`. + /// - `Query` — single-row query, executes to `Self`, erroring if no + /// row matches. Returned by non-nullable relation accessors. + /// - `Query>` — optional single-row query, executes to + /// `Option`. Returned by nullable relation accessors. + type Query; + + /// Create builder type for this model + type Create: Default + IntoInsert + IntoExpr; + + /// Update builder type for this model + type Update<'a>; + + /// Update by query builder type for this model + type UpdateQuery; + + /// A typed path from `Origin` into this model. + type Path; + + /// The model's primary key type. + /// + /// For a single-column key, this is the column's Rust type (e.g. + /// `Uuid`, `i64`). For a composite key, this is a tuple of the + /// column types in declaration order. + /// + /// Generic code can bind on this to write functions that accept any + /// model identified by a particular PK type — for example, a request + /// extractor for any `M: Model`. + type PrimaryKey; + + /// The field accessor type used when this model appears as the "many" side + /// of a has-many relation, parameterized by the origin model. + type ManyField; + + /// The field accessor for a singular relation to this model, + /// parameterized by the origin and path target. + /// + /// `Target` is `Self` for required relations and `Option` for + /// optional relations, as resolved by + /// [`RelationOneField::Expr`](super::RelationOneField::Expr). + type OneField; + + /// Unique identifier for this model within the schema. + /// + /// Identifiers are *not* unique across schemas. + fn id() -> ModelId; + + /// Returns the schema definition for this model. + fn schema() -> app::Model; + + /// Register this model and all models reachable through its fields into + /// the given [`ModelSet`]. + /// + /// If this model is already present in the set (checked via + /// [`ModelSet::contains`]), the method returns immediately. Otherwise it + /// inserts the model and recursively registers any models referenced by + /// embedded or relation fields. This is the entry point used by + /// [`models!`](crate::models) to register root models; embedded types are + /// discovered transitively through their containing fields. + fn register(model_set: &mut ModelSet); + + /// Construct a model path from a [`Path`] targeting this model. + fn new_path(path: Path) -> Self::Path; + + /// Construct a path rooted at this model. + fn new_root_path() -> Self::Path { + Self::new_path(Self::path_root()) + } + + /// An identity [`Path`] rooted at this model. + /// + /// This is how `Path` recovers a model's [`ModelId`] without a dedicated + /// registration trait: the model supplies its own id via [`id`](Self::id). + fn path_root() -> Path { + Path::from_model_id(Self::id()) + } + + /// A [`Path`] from this model to the field at `index`. + fn path_field(index: usize) -> Path { + Path::field_at(Self::id(), index) + } + + /// An identity [`Path`] rooted at a list of this model, used as the root of + /// has-many relation scopes. + fn path_model_list() -> Path, List> { + Path::from_model_id(Self::id()) + } + + /// Return a fresh, default-initialized create builder. + fn new_create() -> Self::Create { + Self::Create::default() + } + + /// Construct a [`ManyField`](Self::ManyField) from a path targeting a list + /// of this model. + fn new_many_field(path: Path>) -> Self::ManyField; + + /// Map a field name string to its [`FieldId`]. + /// + /// Panics if `name` does not match any field on the model. + fn field_name_to_id(name: &str) -> FieldId; + + /// Build a query that filters this model by its primary key. + /// + /// `id` takes any expression that evaluates to the model's + /// [`PrimaryKey`](Self::PrimaryKey) type — bare PK values via their + /// `IntoExpr` impl, subqueries that return a PK, or any other + /// [`Expr`] of the correct type. + fn find_by_primary_key(id: Expr) -> Self::Query>; + + /// Wrap a raw statement-level [`Query`](crate::stmt::Query) in this model's + /// query builder, preserving the result shape `T`. This is the single + /// constructor generic code uses to build any query shape; callers narrow + /// at the statement level (`.one()`, `.first()`) before wrapping. + fn wrap_query(stmt: crate::stmt::Query) -> Self::Query; + + /// Narrow a list query to a single-row query (errors at exec time if no + /// row matches). Used by the codegen for non-nullable relation accessors. + fn query_one(query: Self::Query>) -> Self::Query; + + /// Narrow a list query to an optional single-row query. Used by the + /// codegen for nullable relation accessors. + fn query_first(query: Self::Query>) -> Self::Query>; +} + +/// List query builder for model `M` (executes to `Vec`). Names the +/// `::Query>` instantiation of the [`Model::Query`] GAT so +/// the long projection stays out of relation-accessor signatures and the +/// errors they produce. +pub type QueryMany = ::Query>; + +/// Single-row query builder for model `M` (executes to `M`). See [`QueryMany`]. +pub type QueryOne = ::Query; + +/// Optional single-row query builder for model `M` (executes to `Option`). +/// See [`QueryMany`]. +pub type QueryOptionOne = ::Query>; diff --git a/packages/toasty/src/schema/net.rs b/packages/toasty/src/schema/net.rs new file mode 100644 index 00000000000..38732d5e80a --- /dev/null +++ b/packages/toasty/src/schema/net.rs @@ -0,0 +1,63 @@ +use super::{Field, Load}; +use crate::stmt::{Expr, IpCidr, IpInet, List, MacAddr6, MacAddr8, Path}; +use toasty_core::{ + Result, + stmt::{Type, Value}, +}; + +macro_rules! impl_net_field { + ($ty:ty, $name:ident, $lit:literal) => { + impl Load for $ty { + type Output = Self; + + fn ty() -> Type { + Type::$name + } + + fn load(value: Value) -> Result { + match value { + Value::$name(v) => Ok(v), + _ => Err(toasty_core::Error::type_conversion(value, $lit)), + } + } + + fn reload(target: &mut Self, value: Value) -> Result<()> { + *target = Self::load(value)?; + Ok(()) + } + } + + impl Field for $ty { + type ExprTarget = Self; + type Path = Path; + type ListPath = Path>; + type Update<'a> = (); + type Inner = Self; + + fn new_path(path: Path) -> Self::Path { + path + } + + fn new_list_path( + path: Path>, + ) -> Self::ListPath { + path + } + + fn new_update<'a>( + _assignments: &'a mut toasty_core::stmt::Assignments, + _projection: toasty_core::stmt::Projection, + ) -> Self::Update<'a> { + } + + fn key_constraint(&self, target: Path) -> Expr { + target.eq(self) + } + } + }; +} + +impl_net_field!(IpCidr, Cidr, "cidr::IpCidr"); +impl_net_field!(IpInet, Inet, "cidr::IpInet"); +impl_net_field!(MacAddr6, MacAddr, "macaddr::MacAddr6"); +impl_net_field!(MacAddr8, MacAddr8, "macaddr::MacAddr8"); diff --git a/packages/toasty/src/schema/num.rs b/packages/toasty/src/schema/num.rs new file mode 100644 index 00000000000..d39efe504d6 --- /dev/null +++ b/packages/toasty/src/schema/num.rs @@ -0,0 +1,76 @@ +use crate::{ + Result, + schema::{Field, Load}, + stmt::{Expr, List, Path}, +}; + +use toasty_core::stmt; + +/// Macro to generate Load and Field implementations for numeric types that use `try_into()` +macro_rules! impl_field_numeric { + ($($ty:ty => $stmt_ty:ident),* $(,)?) => { + $( + impl Load for $ty { + type Output = Self; + + fn ty() -> stmt::Type { + stmt::Type::$stmt_ty + } + + fn load(value: stmt::Value) -> Result { + value.try_into() + } + + fn reload(target: &mut Self, value: stmt::Value) -> Result<()> { + *target = Self::load(value)?; + Ok(()) + } + } + + impl Field for $ty { + type ExprTarget = Self; + type Path = Path; + type ListPath = Path>; + type Update<'a> = (); + type Inner = Self; + + fn new_path(path: Path) -> Self::Path { + path + } + + fn new_list_path( + path: Path>, + ) -> Self::ListPath { + path + } + + fn new_update<'a>( + _assignments: &'a mut toasty_core::stmt::Assignments, + _projection: toasty_core::stmt::Projection, + ) -> Self::Update<'a> { + } + + fn key_constraint( + &self, + target: Path, + ) -> Expr { + target.eq(self) + } + } + )* + }; +} + +// Generate implementations for all numeric types +impl_field_numeric! { + i8 => I8, + i16 => I16, + i32 => I32, + i64 => I64, + u8 => U8, + u16 => U16, + u32 => U32, + u64 => U64, + f32 => F32, + f64 => F64, +} diff --git a/packages/toasty/src/schema/option.rs b/packages/toasty/src/schema/option.rs new file mode 100644 index 00000000000..395fac6d70c --- /dev/null +++ b/packages/toasty/src/schema/option.rs @@ -0,0 +1,37 @@ +use super::Load; +use toasty_core::stmt::{self, Value}; + +impl Load for Option { + type Output = Option; + + fn ty() -> stmt::Type { + T::ty() + } + + fn ty_relation() -> stmt::Type { + T::ty() + } + + fn load(value: Value) -> Result { + match value { + Value::Null => Ok(None), + // Any other value is the raw model record (from INSERT or + // SELECT+include when a matching row exists). + v => Ok(Some(T::load(v)?)), + } + } + + fn load_relation(value: Value) -> Result { + match value { + Value::Null => Ok(None), + // Any other value is the raw model record (from INSERT or + // SELECT+include when a matching row exists). + v => Ok(Some(T::load(v)?)), + } + } + + fn reload(target: &mut Self::Output, value: Value) -> Result<(), crate::Error> { + *target = Self::load(value)?; + Ok(()) + } +} diff --git a/packages/toasty/src/schema/register.rs b/packages/toasty/src/schema/register.rs new file mode 100644 index 00000000000..d34ce327415 --- /dev/null +++ b/packages/toasty/src/schema/register.rs @@ -0,0 +1,144 @@ +use toasty_core::schema::app::{ModelId, ModelSet}; + +/// Generate a unique model ID at runtime. +/// +/// This function uses a global atomic counter to ensure each call returns +/// a unique ModelId. IDs start at 0 and increment with each call. +/// This is thread-safe and can be called concurrently. +pub fn generate_unique_id() -> ModelId { + use std::sync::atomic::{AtomicUsize, Ordering}; + + static NEXT_MODEL_ID: AtomicUsize = AtomicUsize::new(0); + + let id = NEXT_MODEL_ID.fetch_add(1, Ordering::Relaxed); + ModelId(id) +} + +/// An item discovered at compile time by the `#[derive(Model)]` macro. +/// +/// Each root model emits an `inventory::submit!` call that creates a +/// `DiscoverItem` carrying the originating crate name (via +/// `env!("CARGO_PKG_NAME")`) and a registration function. +/// [`toasty::models!`] iterates over all submitted items filtered by crate +/// name. Embedded types are not submitted directly; they are discovered +/// transitively through the fields of the models that contain them. +#[doc(hidden)] +pub struct DiscoverItem { + crate_name: &'static str, + add_fn: fn(&mut ModelSet), +} + +impl DiscoverItem { + pub const fn new(crate_name: &'static str, add_fn: fn(&mut ModelSet)) -> Self { + Self { crate_name, add_fn } + } + + pub fn crate_name(&self) -> &'static str { + self.crate_name + } + + pub fn add_to(&self, model_set: &mut ModelSet) { + (self.add_fn)(model_set) + } + + pub fn add_all_from_crate_to(model_set: &mut ModelSet, crate_name: &str) { + // Normalize all crate names to use `_` instead of `-` for `models!` invocation + // which has to use `_` to be a valid crate identifier. + let crate_name = crate_name.replace("-", "_"); + for item in inventory::iter:: { + if item.crate_name().replace("-", "_") == crate_name { + item.add_to(model_set); + } + } + } +} + +// Collect all `DiscoverItem` instances submitted by derive macros so they +// can be iterated by [`toasty::models!`]. +inventory::collect!(DiscoverItem); + +// Re-exported so that generated `inventory::submit!` calls can reference +// the crate without requiring users to depend on it directly. +pub use inventory; + +/// Creates a [`ModelSet`] containing the specified models. +/// +/// The resulting `ModelSet` is typically passed to +/// [`Builder::models`](crate::Db::builder) when setting up a database +/// connection: +/// +/// ```ignore +/// let db = toasty::Db::builder() +/// .models(toasty::models!(User, Todo)) +/// .connect("sqlite::memory:") +/// .await?; +/// ``` +/// +/// Each registered model automatically discovers any models reachable through +/// its fields — embedded types, `BelongsTo`, `HasMany`, and `HasOne` targets +/// are all registered transitively. You only need to list the entry-point +/// models. +/// +/// # Syntax +/// +/// The macro accepts a comma-separated list of any combination of: +/// +/// - **Individual models** — a type path to a struct that derives `Model`. +/// Module paths are supported (e.g. `my_module::MyModel`). Embedded types are +/// not listed here; they are discovered through the fields of the models that +/// contain them. +/// - **`crate::*`** — registers every model discovered in the current crate. +/// - **`some_crate::*`** — registers every model discovered in the named +/// external crate. +/// +/// These forms can be freely combined: +/// +/// ```ignore +/// toasty::models!( +/// // All models from the current crate +/// crate::*, +/// // All models from an external crate +/// third_party_models::*, +/// // Individual models, with or without a module path +/// MyModel, +/// other::SpecificModel, +/// ) +/// ``` +#[allow(clippy::crate_in_macro_def)] +#[macro_export] +macro_rules! models { + // Register all models from current crate with `models!(crate::*)` + (@internal $set:ident crate::* $(,$($rest:tt)*)?) => {{ + ::toasty::codegen_support::DiscoverItem::add_all_from_crate_to(&mut $set, env!("CARGO_PKG_NAME")); + $( + $crate::models!(@internal $set $($rest)*); + )? + }}; + + // Register all models from a third party crate with `models!(third_party::*)` + (@internal $set:ident $crate_name:ident::* $(,$($rest:tt)*)?) => {{ + // Make sure the provided crate actually exists. + { use ::$crate_name; } + ::toasty::codegen_support::DiscoverItem::add_all_from_crate_to(&mut $set, stringify!($crate_name)); + $( + $crate::models!(@internal $set $($rest)*); + )? + }}; + + // Register single model with `models!(ModelName)` + (@internal $set:ident $model:ty $(,$($rest:tt)*)?) => {{ + <$model as ::toasty::schema::Model>::register(&mut $set); + $( + $crate::models!(@internal $set $($rest)*); + )? + }}; + + // Empty list + (@internal $set:ident) => {}; + + ($($tokens:tt)*) => {{ + let mut model_set = ::toasty::schema::ModelSet::new(); + $crate::models!(@internal model_set $($tokens)*); + model_set + }}; +} diff --git a/packages/toasty/src/schema/relation.rs b/packages/toasty/src/schema/relation.rs new file mode 100644 index 00000000000..7015062cfcf --- /dev/null +++ b/packages/toasty/src/schema/relation.rs @@ -0,0 +1,44 @@ +use super::{Load, Model}; + +use toasty_core::schema::Name; +use toasty_core::schema::app::{FieldId, FieldTy}; +use toasty_core::stmt; + +/// A Rust field type that represents a `#[has_many]` relation. +/// +/// Implemented by [`Vec`](Vec) (eager) and +/// [`Deferred>`](super::Deferred) (lazy) where `M: Model`. The set of +/// impls is the source of truth for which Rust shapes are valid as a +/// has-many field: anything outside those two combinations does not satisfy +/// the trait. +/// +/// A direct `#[has_many]` pairs with a `BelongsTo` on its target. A +/// `#[has_many(via = ...)]` follows a relation path instead. Many-to-many +/// models use a direct field for join-model records and a `via` field for the +/// opposite endpoints. +pub trait RelationManyField: Load { + /// The target model that this field references. + type Target: Model; + + /// Whether the field stores its value in a deferred load slot. + const DEFERRED: bool; + + /// Reloads this relation field from a returned value. + fn reload(target: &mut Self, value: stmt::Value) -> crate::Result<()>; + + /// Build the [`FieldTy`] for a `HasMany` relation field, given the + /// singular name derived from the field identifier and an optional + /// paired `BelongsTo` field on the target model resolved from + /// `#[has_many(pair = )]`. When `None`, the linker selects the + /// pair by searching the target for a unique `BelongsTo` back to the + /// source. + /// + /// `via` carries the fully resolved [`stmt::Path`] of a + /// `#[has_many(via = a.b)]` multi-step relation, rooted at the declaring + /// model. A `via` relation has no pair. + fn many_relation_field_ty( + singular: Name, + pair: Option, + via: Option, + ) -> FieldTy; +} diff --git a/packages/toasty/src/schema/relation_one.rs b/packages/toasty/src/schema/relation_one.rs new file mode 100644 index 00000000000..68648b951f1 --- /dev/null +++ b/packages/toasty/src/schema/relation_one.rs @@ -0,0 +1,152 @@ +use super::{Deferred, Load, Model, QueryMany, QueryOne, QueryOptionOne}; + +use toasty_core::schema::app::ModelId; +use toasty_core::schema::app::{self, FieldId, FieldTy, ForeignKey}; +use toasty_core::stmt; + +/// A Rust field type that represents a `#[has_one]` or `#[belongs_to]` +/// relation. +/// +/// Implemented by `M`, `Option`, `Deferred`, and `Deferred>` +/// where `M: Model`. The `Option<...>` wrappers carry nullability; the +/// `Deferred<...>` wrappers carry deferred loading. Anything outside this +/// shape does not satisfy the trait. +pub trait RelationOneField: Load { + /// The target model that this field references. + type Target: Model; + + /// The query type produced by the relation accessor. For non-nullable + /// impls this is `::Query`; for nullable impls it is + /// `::Query>`. + type One; + + /// The path target and expression type used in create/update setters. + /// Resolves to `Self::Target` for non-nullable impls and + /// `Option` for nullable impls. + type Expr; + + /// Whether the field stores its value in a deferred load slot. + const DEFERRED: bool; + + /// Whether the field is nullable (i.e. wrapped in `Option`). + const NULLABLE: bool; + + /// Reloads this relation field from a returned value. + fn reload(target: &mut Self, value: stmt::Value) -> crate::Result<()> { + ::reload(target, value) + } + + /// Narrow a list query targeting the related model into the appropriate + /// "one" query — `Query` for non-nullable impls and + /// `Query>` for nullable impls. + /// + /// Relation accessors build the list query (from a filter or by wrapping a + /// singular association via [`Model::wrap_query`]) and pass it here; the + /// association's path is preserved through the wrap so generated mutators + /// (insert, remove, create) can read it. + fn make_one(query: QueryMany) -> Self::One; + + /// Build the [`FieldTy`] for a `HasOne` relation field, given an + /// optional paired `BelongsTo` field on the target model resolved + /// from `#[has_one(pair = )]`. When `None`, the linker selects + /// the pair by searching the target for a unique `BelongsTo` back to + /// the source. + /// + /// `via` carries the fully resolved [`stmt::Path`] of a + /// `#[has_one(via = a.b)]` multi-step relation, rooted at the declaring + /// model. A `via` relation has no pair. + fn has_one_relation_field_ty(pair: Option, via: Option) -> FieldTy { + let target = ::id(); + let expr_ty = stmt::Type::Model(target); + let cardinality = app::Cardinality::One; + + match via { + Some(path) => FieldTy::Via(app::Via::new(target, expr_ty, cardinality, path, None)), + None => FieldTy::Has(app::Has { + target, + expr_ty, + cardinality, + pair_id: pair.unwrap_or(FieldId { + model: ModelId(usize::MAX), + index: usize::MAX, + }), + }), + } + } + + /// Build the [`FieldTy`] for a `BelongsTo` relation field, given the + /// foreign key resolved from the field's `#[belongs_to(...)]` attribute. + fn belongs_to_relation_field_ty(foreign_key: ForeignKey) -> FieldTy { + let target = ::id(); + + FieldTy::BelongsTo(app::BelongsTo { + target, + expr_ty: stmt::Type::Model(target), + // The pair is populated at runtime. + pair: None, + foreign_key, + }) + } +} + +impl RelationOneField for M { + type Target = M; + type One = QueryOne; + type Expr = M; + + const DEFERRED: bool = false; + const NULLABLE: bool = false; + + fn make_one(query: QueryMany) -> Self::One { + ::query_one(query) + } +} + +impl RelationOneField for Option { + type Target = M; + type One = QueryOptionOne; + type Expr = Option; + + const DEFERRED: bool = false; + const NULLABLE: bool = true; + + fn make_one(query: QueryMany) -> Self::One { + ::query_first(query) + } +} + +impl RelationOneField for Deferred { + type Target = M; + type One = QueryOne; + type Expr = M; + + const DEFERRED: bool = true; + const NULLABLE: bool = false; + + fn reload(target: &mut Self, _value: stmt::Value) -> crate::Result<()> { + target.unload(); + Ok(()) + } + + fn make_one(query: QueryMany) -> Self::One { + ::query_one(query) + } +} + +impl RelationOneField for Deferred> { + type Target = M; + type One = QueryOptionOne; + type Expr = Option; + + const DEFERRED: bool = true; + const NULLABLE: bool = true; + + fn reload(target: &mut Self, _value: stmt::Value) -> crate::Result<()> { + target.unload(); + Ok(()) + } + + fn make_one(query: QueryMany) -> Self::One { + ::query_first(query) + } +} diff --git a/packages/toasty/src/schema/scope.rs b/packages/toasty/src/schema/scope.rs new file mode 100644 index 00000000000..45f5f0409bb --- /dev/null +++ b/packages/toasty/src/schema/scope.rs @@ -0,0 +1,44 @@ +use crate::stmt::Path; + +/// A scope represents a context that contains items of a particular type. +/// +/// Generated query types are scopes whose items are instances of the target +/// model. The trait provides associated types for building typed paths and +/// create builders within the scope. +/// +/// Whether a particular scope can satisfy a scoped create is decided at +/// execution time: `create_in_scope` always produces a create builder and +/// forwards the scope to the engine, which validates whether it can populate +/// the required fields (for example, by reading a foreign key from a single- +/// step relation traversal). +#[diagnostic::on_unimplemented( + message = "this expression cannot be used as a Toasty create scope", + label = "this expression does not support scoped creation" +)] +pub trait Scope { + /// The item type contained in this scope. + type Item; + + /// A typed path from `Origin` into this scope. + type Path; + + /// The create builder for items in this scope. + type Create; + + /// Construct a scope path from a [`Path`] targeting the item type. + fn new_path(path: Path) -> Self::Path; + + /// Return a fresh, default-initialized create builder for this scope. + fn new_create() -> Self::Create; + + /// Return a root path for this scope, anchored at the scope's model. + /// + /// This is used by the `create!` macro to obtain field accessors for + /// nested builders without needing to know the concrete model type. + fn new_path_root() -> Self::Path; + + /// Build a create builder scoped to this expression. Forwards any + /// relation-association metadata to the engine, which decides at exec + /// time whether the scope can be satisfied. + fn create_in_scope(self) -> Self::Create; +} diff --git a/packages/toasty/src/schema/via.rs b/packages/toasty/src/schema/via.rs new file mode 100644 index 00000000000..f66a74a40de --- /dev/null +++ b/packages/toasty/src/schema/via.rs @@ -0,0 +1,197 @@ +use super::{Deferred, Load}; +use crate::stmt::{Association, List, Path, Query}; + +use toasty_core::schema::Name; +use toasty_core::schema::app::{self, FieldTy, ModelId, Via}; +use toasty_core::stmt; + +/// Placeholder via target for a scalar terminal, overwritten when `toasty-core` +/// resolves the relation chain while linking. It must never reach a query. +const UNRESOLVED_TARGET: ModelId = ModelId(usize::MAX); + +/// Implemented by a `#[has_many(via = …)]` field's Rust type, exposing the +/// via's [`Target`](Self::Target) — the terminal type the path projects. +/// +/// Blanket-implemented over `Vec` and [`Deferred>`](Deferred) for +/// every `E`, so the derive can route a via field through [`ViaTarget`] +/// without requiring `E: Model`. A single blanket impl per outer shape keeps +/// it coherent with the per-target `ViaTarget` impls. +/// +/// A join-model many-to-many field is a model-terminal via. For example, +/// `#[has_many(via = memberships.group)] groups: Deferred>` +/// follows the declaring model's `memberships` relation and then each +/// membership's `group` relation. +pub trait ViaManyField { + /// The via's terminal type — the model or scalar the path projects. + type Target: ViaTarget; + + /// Whether the field stores its value in a deferred load slot. + const DEFERRED: bool; +} + +impl ViaManyField for Vec { + type Target = E; + const DEFERRED: bool = false; +} + +impl ViaManyField for Deferred> { + type Target = E; + const DEFERRED: bool = true; +} + +/// The query builder a `#[has_many(via = …)]` navigation method returns for a +/// field of type `F` (e.g. `Vec`, `Deferred>`). +/// +/// Resolves to the terminal type's [`ViaTarget::Query`]: `QueryMany` for a +/// model terminal, `Query>` for a scalar. Mirrors +/// [`QueryMany`](super::QueryMany), which aliases a model's query builder the +/// same way. +pub type ViaMany = <::Target as ViaTarget>::Query; + +/// The typed path handle a `#[has_many(via = …)]` field's accessor returns for +/// a field of type `F` (e.g. `Vec`, `Deferred>`), rooted +/// at `Origin`. +/// +/// Resolves to the terminal type's [`ViaTarget::Path`]: the model's chainable +/// `ManyField` for a model terminal, `Path>` for a scalar. +/// Mirrors [`ViaMany`], which aliases the navigation query the same way. +pub type ViaPath = <::Target as ViaTarget>::Path; + +/// A type that can be the terminal element of a `#[has_many(via = …)]` +/// relation. +/// +/// A `via` relation reaches its terminal by following a path of existing +/// relations. The terminal is usually another model (`via = comments.article`), +/// but it may also be a **scalar field** (`via = todos.tags.name`), which +/// projects that field across the relation path to yield `Vec`. +/// +/// This trait is keyed on the *terminal element type*, with disjoint concrete +/// impls so they never overlap: +/// +/// - The `#[derive(Model)]` macro emits an impl for each model, setting +/// [`Query`](Self::Query) to that model's generated query builder (so a +/// relation-terminal via keeps its rich `QueryMany` API). +/// - This module emits an impl for each scalar primitive, setting +/// [`Query`](Self::Query) to the plain [`Query>`]. +pub trait ViaTarget { + /// The query builder returned by the generated has-many-via navigation + /// method. + type Query; + + /// The typed path handle a `#[has_many(via = …)]` field's accessor returns, + /// parameterized by the origin model. Mirrors [`Model::Path`] for the + /// via-many case. + /// + /// A **model** terminal returns that model's chainable + /// [`ManyField`](crate::schema::Model::ManyField) — a `Path` wrapper that + /// adds field accessors, so navigation can continue *through* a via step + /// (e.g. `organizations.todos.title`, where `organizations.todos` is itself + /// a via). A **scalar** terminal returns a plain list [`Path`]: a leaf with + /// nothing further to chain. + /// + /// [`Model::Path`]: crate::schema::Model::Path + type Path; + + /// Build the [`FieldTy::Via`] for a `#[has_many(via = …)]` field whose + /// terminal element type is `Self`. + /// + /// `path` is the fully-resolved field path (rooted at the declaring + /// model), including the terminal step. The via target is `Self::id()` for + /// a model terminal; for a scalar terminal it is the model the relation + /// chain reaches, which the path alone can't name here — so the scalar + /// impls leave it unset and `toasty-core` fills it in while linking. + fn via_field_ty(singular: Name, path: stmt::Path) -> FieldTy; + + /// Build the [`Path`](Self::Path) handle from the typed `path` reaching this + /// terminal. A model terminal wraps the path in its chainable `ManyField`; a + /// scalar terminal is a leaf, so the path is the handle. + fn new_path(path: Path>) -> Self::Path + where + Self: Sized; + + /// Wrap a via-field association into the navigation query. + /// + /// A scalar terminal yields a query whose source model and terminal + /// projection are filled in during lowering ([`RewriteVia`]) from the + /// core-resolved schema, so neither is needed here. + /// + /// [`RewriteVia`]: ../../engine/lower/association/struct.RewriteVia.html + fn make_via_query(assoc: Association>) -> Self::Query + where + Self: Sized; +} + +/// Emit `ViaTarget` for scalar primitives. Each impl is concrete, so it +/// stays disjoint from the per-model impls the derive macro generates. +macro_rules! impl_via_many_scalar { + ( $( $t:ty ),* $(,)? ) => { + $( + impl ViaTarget for $t { + type Query = Query>; + type Path = Path>; + + fn new_path(path: Path>) -> Self::Path { + // A scalar terminal is a leaf — the path is the handle. + path + } + + fn via_field_ty(singular: Name, path: stmt::Path) -> FieldTy { + // The terminal scalar field is the path's last step. + let terminal = *path + .projection + .as_slice() + .last() + .expect("via path has at least one step"); + let expr_ty = stmt::Type::List(Box::new(<$t as Load>::ty())); + // `target` (the model the relation chain reaches) is resolved + // in core's link phase; leave a placeholder it overwrites. + FieldTy::Via(Via::new( + UNRESOLVED_TARGET, + expr_ty, + app::Cardinality::Many { singular }, + path, + Some(terminal), + )) + } + + fn make_via_query(assoc: Association>) -> Self::Query { + // A via-as-source query whose source model and terminal + // projection RewriteVia fills in from the resolved schema + // (see its scalar branch). The placeholder source id is + // overwritten there. `distinct` collapses duplicate terminal + // values, matching the include path. + let mut query = stmt::Query::builder(stmt::SourceModel { + id: UNRESOLVED_TARGET, + via: Some(assoc.untyped), + }) + .build(); + query.body.as_select_mut_unwrap().distinct = true; + Query::from_untyped(query) + } + } + )* + }; +} + +impl_via_many_scalar!( + String, + bool, + i8, + i16, + i32, + i64, + u16, + u32, + u64, + isize, + usize, + f32, + f64, + uuid::Uuid, +); + +#[cfg(feature = "rust_decimal")] +impl_via_many_scalar!(rust_decimal::Decimal); + +#[cfg(feature = "bigdecimal")] +impl_via_many_scalar!(bigdecimal::BigDecimal); diff --git a/packages/toasty/src/sql.rs b/packages/toasty/src/sql.rs new file mode 100644 index 00000000000..01e4d99d8df --- /dev/null +++ b/packages/toasty/src/sql.rs @@ -0,0 +1,288 @@ +//! Raw SQL execution helpers for SQL backends. +//! +//! Use [`statement`] for SQL that returns an affected-row count and [`query`] +//! for SQL that returns rows. Both builders execute through +//! [`Executor`](crate::Executor), so they work with [`Db`](crate::Db), +//! [`Connection`](crate::Connection), and [`Transaction`](crate::Transaction). +//! +//! Raw SQL is backend SQL. Toasty does not rewrite placeholders, quote +//! identifiers, or hydrate models from raw query results. Use the placeholder +//! syntax of the active driver. The syntax is exposed at runtime through +//! [`Capability::sql_placeholder`](crate::Capability::sql_placeholder): +//! +//! | Backend | [`SqlPlaceholder`](crate::SqlPlaceholder) | +//! |---|---| +//! | SQLite | `NumberedQuestionMark` (`?1`, `?2`, ...) | +//! | Turso | `NumberedQuestionMark` (`?1`, `?2`, ...) | +//! | PostgreSQL | `DollarNumber` (`$1`, `$2`, ...) | +//! | MySQL | `QuestionMark` (`?`, `?`, ...) | +//! +//! # Statements +//! +//! ```ignore +//! let updated = toasty::sql::statement( +//! "UPDATE users SET name = ?1 WHERE id = ?2", +//! ) +//! .bind("Alice") +//! .bind(1_i64) +//! .exec(&mut db) +//! .await?; +//! +//! assert_eq!(updated, 1); +//! ``` +//! +//! # Queries +//! +//! Queries return dynamic [`Value`] rows. Each row is a [`Value::Record`] with +//! fields in selected-column order. +//! +//! ```ignore +//! let rows = toasty::sql::query( +//! "SELECT id, name FROM users WHERE active = ?1", +//! ) +//! .bind(true) +//! .exec(&mut db) +//! .await?; +//! ``` +//! +//! The driver infers result value types from database metadata by default. Use +//! [`Query::column_types`] when a result column is ambiguous, such as a SQLite +//! boolean stored as an integer or a UUID stored as bytes. + +use crate::{Executor, Result}; +use toasty_core::{ + Error, + driver::{ + Capability, + operation::{RawSql, RawSqlRet, TypedValue}, + }, + schema::db, + stmt::{self, Value}, +}; + +/// A raw SQL statement that returns an affected-row count. +/// +/// Create one with [`statement`]. +pub struct Statement { + sql: String, + params: Vec, +} + +/// A raw SQL query that returns dynamic rows. +/// +/// Create one with [`query`]. +pub struct Query { + sql: String, + params: Vec, + ret: RawSqlRet, +} + +#[derive(Debug, Clone)] +enum Param { + Infer(Value), + Typed(TypedValue), +} + +/// Create a raw SQL statement. +/// +/// Statements are executed for their affected-row count. Use +/// [`query`] for SQL that returns rows. +/// +/// The SQL string must use the placeholder syntax reported by +/// [`Capability::sql_placeholder`](crate::Capability::sql_placeholder). +/// +/// ```ignore +/// let count = toasty::sql::statement( +/// "DELETE FROM users WHERE archived = ?1", +/// ) +/// .bind(true) +/// .exec(&mut db) +/// .await?; +/// ``` +pub fn statement(sql: impl Into) -> Statement { + Statement { + sql: sql.into(), + params: vec![], + } +} + +/// Create a raw SQL query. +/// +/// Query rows are returned as dynamic [`Value`] records. +/// +/// ```ignore +/// let rows = toasty::sql::query( +/// "SELECT id, name FROM users WHERE active = ?1", +/// ) +/// .bind(true) +/// .exec(&mut db) +/// .await?; +/// ``` +pub fn query(sql: impl Into) -> Query { + Query { + sql: sql.into(), + params: vec![], + ret: RawSqlRet::Infer, + } +} + +impl Statement { + /// Bind a value using the executor driver's default database type for + /// that value. + /// + /// Values are bound in the order this method is called. Use + /// [`bind_typed`](Self::bind_typed) for `NULL`, empty lists, or values + /// whose database type cannot be inferred from the value alone. + pub fn bind(mut self, value: impl Into) -> Self { + self.params.push(Param::Infer(value.into())); + self + } + + /// Bind a value with an explicit database type. + /// + /// This is useful for `NULL`, empty lists, or backend-specific column + /// types. + /// + /// ```ignore + /// use toasty::schema::db; + /// + /// toasty::sql::statement( + /// "UPDATE users SET archived_at = ?1 WHERE id = ?2", + /// ) + /// .bind_typed(toasty::stmt::Value::Null, db::Type::Timestamp(6)) + /// .bind(1_i64) + /// .exec(&mut db) + /// .await?; + /// ``` + pub fn bind_typed(mut self, value: impl Into, ty: db::Type) -> Self { + self.params.push(Param::Typed(TypedValue { + value: value.into(), + ty, + })); + self + } + + /// Execute the statement and return the affected-row count. + /// + /// The executor may be a [`Db`](crate::Db), [`Connection`](crate::Connection), + /// or [`Transaction`](crate::Transaction). + pub async fn exec(self, executor: &mut dyn Executor) -> Result { + let raw = into_raw_sql( + self.sql, + self.params, + RawSqlRet::None, + executor.capability(), + )?; + let response = executor.exec_raw_sql(raw).await?; + + Ok(response.values.into_count()) + } +} + +impl Query { + /// Bind a value using the executor driver's default database type for + /// that value. + /// + /// Values are bound in the order this method is called. Use + /// [`bind_typed`](Self::bind_typed) for `NULL`, empty lists, or values + /// whose database type cannot be inferred from the value alone. + pub fn bind(mut self, value: impl Into) -> Self { + self.params.push(Param::Infer(value.into())); + self + } + + /// Bind a value with an explicit database type. + /// + /// This is useful for `NULL`, empty lists, or backend-specific column + /// types. + pub fn bind_typed(mut self, value: impl Into, ty: db::Type) -> Self { + self.params.push(Param::Typed(TypedValue { + value: value.into(), + ty, + })); + self + } + + /// Provide result column type hints for ambiguous backend values. + /// + /// Hints affect result decoding only. They do not change the SQL sent to + /// the database. Use this when backend metadata is not enough to choose the + /// desired Toasty value type. + /// + /// ```ignore + /// use toasty::stmt; + /// + /// let rows = toasty::sql::query( + /// "SELECT id, enabled FROM users WHERE id = ?1", + /// ) + /// .bind(1_i64) + /// .column_types([stmt::Type::I64, stmt::Type::Bool]) + /// .exec(&mut db) + /// .await?; + /// ``` + pub fn column_types(mut self, types: impl IntoIterator) -> Self { + self.ret = RawSqlRet::Types(types.into_iter().collect()); + self + } + + /// Execute the query and return dynamic rows. + /// + /// Each row is a [`Value::Record`] with fields in selected-column order. + /// The executor may be a [`Db`](crate::Db), [`Connection`](crate::Connection), + /// or [`Transaction`](crate::Transaction). + pub async fn exec(self, executor: &mut dyn Executor) -> Result> { + let raw = into_raw_sql(self.sql, self.params, self.ret, executor.capability())?; + let response = executor.exec_raw_sql(raw).await?; + + match response.values.collect_as_value().await? { + Value::List(items) => Ok(items), + value => Err(toasty_core::Error::invalid_result(format!( + "raw SQL query expected a list of rows, got {value:?}" + ))), + } + } +} + +fn into_raw_sql( + sql: String, + params: Vec, + ret: RawSqlRet, + capability: &Capability, +) -> Result { + if !capability.sql() { + return Err(Error::unsupported_feature(format!( + "{} does not support raw SQL", + capability.driver_name + ))); + } + + Ok(RawSql { + sql, + params: params + .into_iter() + .map(|param| param.into_typed(capability)) + .collect::>()?, + ret, + }) +} + +impl Param { + fn into_typed(self, capability: &Capability) -> Result { + match self { + Param::Infer(value) => infer_typed_value(value, capability), + Param::Typed(value) => Ok(value), + } + } +} + +fn infer_typed_value(value: Value, capability: &Capability) -> Result { + let ty = value + .infer_db_ty(&capability.storage_types) + .map_err(|err| { + err.context(Error::invalid_statement( + "cannot infer raw SQL bind type; use bind_typed", + )) + })?; + + Ok(TypedValue { value, ty }) +} diff --git a/packages/toasty/src/stmt.rs b/packages/toasty/src/stmt.rs new file mode 100644 index 00000000000..ec340bce30c --- /dev/null +++ b/packages/toasty/src/stmt.rs @@ -0,0 +1,275 @@ +mod association; +pub use association::Association; + +mod batch; +pub use batch::{Batch, batch}; + +mod create_many; +pub use create_many::CreateMany; + +mod delete; +pub use delete::Delete; + +mod expr; +pub use expr::Expr; + +mod include; +pub use include::Include; + +mod insert; +pub use insert::Insert; + +mod assignment; +pub use assignment::{ + Assign, Assignment, add, apply, clear, decrement, extend, increment, insert, patch, pop, push, + remove, remove_at, set, subtract, +}; + +mod numeric; +pub use numeric::Numeric; + +mod into_expr; +pub use into_expr::IntoExpr; + +mod into_insert; +pub use into_insert::IntoInsert; + +mod into_statement; +pub use into_statement::IntoStatement; + +#[cfg(feature = "serde")] +mod json; +#[cfg(feature = "serde")] +pub use json::Json; + +mod list; +pub use list::List; + +mod page; +pub use page::Page; + +mod paginate; +pub use paginate::Paginate; + +mod path; +pub use path::Path; + +pub use crate::schema::Auto; +use crate::{Executor, schema::Load}; + +mod query; +pub use query::Query; + +mod relation_insert; +pub use relation_insert::RelationInsert; + +mod relation_remove; +pub use relation_remove::RelationRemove; + +mod scope; +pub use scope::IntoScope; + +mod update; +pub use update::Update; + +mod upsert; +pub use upsert::Upsert; + +#[cfg(feature = "jiff")] +pub use jiff::tz::TimeZone; +#[cfg(feature = "bigdecimal")] +pub use toasty_core::stmt::BigDecimal; +#[cfg(feature = "rust_decimal")] +pub use toasty_core::stmt::Decimal; +#[cfg(feature = "jiff")] +pub use toasty_core::stmt::{Date, DateTime, Time, Timestamp, Zoned}; +#[cfg(feature = "net")] +pub use toasty_core::stmt::{IpCidr, IpInet, MacAddr6, MacAddr8}; +pub use toasty_core::stmt::{OrderBy, OrderByExpr, Projection, Type, Uuid, Value}; + +use toasty_core::stmt; + +use std::{fmt, marker::PhantomData}; + +/// A typed wrapper around an untyped [`stmt::Statement`](toasty_core::stmt::Statement). +/// +/// The type parameter `T` is the **returning type** — the type produced when the +/// statement is executed — not the model the statement operates on. For example, +/// a query that selects `User` records returns `List`, so its statement +/// type is `Statement>`, not `Statement`. +/// +/// Common returning types: +/// +/// | Statement kind | `T` | `exec()` produces | +/// |---|---|---| +/// | Multi-row query | [`List`] | `Vec` | +/// | Single-row query (`.one()`) | `M` | `M` | +/// | Optional query (`.first()`) | `Option` | `Option` | +/// | Insert (create) | `M` | `M` | +/// | Upsert (create or update) | `M` | `M` | +/// | Upsert with `or_ignore()` | `Option` | `Option` | +/// | Delete | `()` | `()` | +/// | Update | `()` | `()` | +/// +/// You rarely construct `Statement` directly. Instead, use the [`From`] +/// implementations to convert from [`Query`], [`Insert`], [`Update`], +/// [`Upsert`], or [`Delete`], or call [`IntoStatement::into_statement`] on a +/// query builder. +pub struct Statement { + pub(crate) untyped: stmt::Statement, + _p: PhantomData, +} + +impl Statement { + /// Wrap a raw untyped [`stmt::Statement`](toasty_core::stmt::Statement), + /// tagging it with returning type `T`. + /// + /// # Examples + /// + /// ``` + /// # use toasty::stmt::Statement; + /// # use toasty_core::stmt as core_stmt; + /// let raw: core_stmt::Statement = core_stmt::Query::unit().into(); + /// let _typed: Statement<()> = Statement::from_untyped_stmt(raw); + /// ``` + pub fn from_untyped_stmt(untyped: stmt::Statement) -> Self { + Self { + untyped, + _p: PhantomData, + } + } + + /// Consume this wrapper and return the underlying untyped statement. + /// + /// This is the inverse of [`from_untyped_stmt`](Self::from_untyped_stmt). + /// + /// # Examples + /// + /// ``` + /// # use toasty::stmt::Statement; + /// # use toasty_core::stmt as core_stmt; + /// let raw: core_stmt::Statement = core_stmt::Query::unit().into(); + /// let typed: Statement<()> = Statement::from_untyped_stmt(raw.clone()); + /// let back: core_stmt::Statement = typed.into_untyped(); + /// assert_eq!(raw, back); + /// ``` + pub fn into_untyped(self) -> stmt::Statement { + self.untyped + } + + pub(crate) fn into_untyped_query(self) -> stmt::Query { + match self.untyped { + stmt::Statement::Query(q) => q, + _ => panic!("expected query statement"), + } + } + + /// Try to extract the inner [`Query`] from this statement. + /// + /// Returns `Some(query)` if the statement is a query, or `None` for + /// inserts, upserts, updates, and deletes. + /// + /// # Examples + /// + /// ``` + /// # use toasty::stmt::{Query, Statement, List}; + /// # use toasty_core::stmt as core_stmt; + /// let query_stmt: Statement> = Statement::from_untyped_stmt( + /// core_stmt::Query::unit().into(), + /// ); + /// assert!(query_stmt.into_query().is_some()); + /// ``` + pub fn into_query(self) -> Option> { + match self.untyped { + stmt::Statement::Query(q) => Some(Query::from_untyped(q)), + _ => None, + } + } +} + +impl Statement { + /// Execute this statement against the given executor and return the + /// deserialized result. + /// + /// This is a convenience wrapper around + /// [`Executor::exec`](crate::Executor::exec). + /// + /// # Examples + /// + /// ``` + /// # tokio::runtime::Runtime::new().unwrap().block_on(async { + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// # let driver = toasty_driver_sqlite::Sqlite::in_memory(); + /// # let mut db = toasty::Db::builder().models(toasty::models!(User)).build(driver).await.unwrap(); + /// # db.push_schema().await.unwrap(); + /// use toasty::stmt::{IntoStatement, List, Query}; + /// + /// let stmt = Query::>::all().into_statement(); + /// let users: Vec = stmt.exec(&mut db).await.unwrap(); + /// # }); + /// ``` + pub async fn exec(self, executor: &mut dyn Executor) -> crate::Result { + executor.exec(self).await + } +} + +impl From> for Statement { + fn from(value: Query) -> Self { + Self { + untyped: value.untyped.into(), + _p: PhantomData, + } + } +} + +impl From> for Statement { + fn from(value: Insert) -> Self { + Self { + untyped: value.untyped.into(), + _p: PhantomData, + } + } +} + +impl From> for Statement { + fn from(value: Update) -> Self { + Self { + untyped: value.untyped.into(), + _p: PhantomData, + } + } +} + +impl fmt::Debug for Statement { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + self.untyped.fmt(fmt) + } +} + +/// Returns an expression that tests whether `lhs` is contained in `rhs`. +/// +/// This works for both single fields and tuples of fields (composite keys): +/// +/// # Examples +/// +/// ``` +/// # #[derive(Debug, toasty::Model)] +/// # struct User { +/// # #[key] +/// # id: i64, +/// # name: String, +/// # } +/// // Single field — test if a user's id is in a list +/// let filter = toasty::stmt::in_list(User::fields().id(), [1_i64, 2, 3]); +/// ``` +pub fn in_list(lhs: impl IntoExpr, rhs: impl IntoExpr>) -> Expr { + Expr::from_untyped(stmt::Expr::in_list( + lhs.into_expr().untyped, + rhs.into_expr().untyped, + )) +} diff --git a/packages/toasty/src/stmt/assignment.rs b/packages/toasty/src/stmt/assignment.rs new file mode 100644 index 00000000000..37b39b3caf6 --- /dev/null +++ b/packages/toasty/src/stmt/assignment.rs @@ -0,0 +1,568 @@ +use super::{IntoExpr, List, Numeric, Path}; +use std::marker::PhantomData; +use toasty_core::stmt; + +/// Apply a field mutation to an update statement's [`Assignments`] map. +/// +/// Every update builder setter accepts `impl Assign`. All types that +/// implement [`IntoExpr`] also implement `Assign` with set semantics, +/// so update setters accept the same types as create setters. +/// +/// For has-many fields, arrays and [`Vec`]s implement +/// `Assign>` with set (replace) semantics. Use [`insert`], +/// [`remove`], or [`apply`] for incremental mutations. +/// +/// [`Assignments`]: toasty_core::stmt::Assignments +pub trait Assign { + /// Convert into an [`Assignment`] value. + fn into_assignment(self) -> Assignment; + + /// Record one or more assignments into the given [`Assignments`] map at + /// the specified projection. + fn assign(self, assignments: &mut stmt::Assignments, projection: stmt::Projection) + where + Self: Sized, + { + self.into_assignment().kind.apply(assignments, projection); + } +} + +/// A typed assignment produced by the [`insert`], [`remove`], [`set`], and +/// [`patch`] combinators. +/// +/// `Assignment` implements `Assign`, so it can be passed directly +/// to any update builder setter that accepts `impl Assign`. +pub struct Assignment { + kind: AssignmentKind, + _p: PhantomData, +} + +enum AssignmentKind { + Set(stmt::Expr), + Insert(stmt::Expr), + Remove(stmt::Expr), + Append(stmt::Expr), + Pop, + RemoveAt(stmt::Expr), + Add(stmt::Expr), + Subtract(stmt::Expr), + Patch { + path_projection: stmt::Projection, + inner: Box, + }, + Apply(Vec), + /// An operation the builder API accepts but Toasty does not support. + /// Applying it records the reason on the [`Assignments`] map instead of + /// an entry; the statement then fails verification with an + /// `unsupported_feature` error before anything is written. + /// + /// [`Assignments`]: toasty_core::stmt::Assignments + Unsupported(String), +} + +impl AssignmentKind { + fn apply(self, assignments: &mut stmt::Assignments, projection: stmt::Projection) { + match self { + AssignmentKind::Set(expr) => assignments.set(projection, expr), + AssignmentKind::Insert(expr) => assignments.insert(projection, expr), + AssignmentKind::Remove(expr) => assignments.remove(projection, expr), + AssignmentKind::Append(expr) => assignments.append(projection, expr), + AssignmentKind::Pop => assignments.pop(projection), + AssignmentKind::RemoveAt(expr) => assignments.remove_at(projection, expr), + AssignmentKind::Add(expr) => assignments.add(projection, expr), + AssignmentKind::Subtract(expr) => assignments.subtract(projection, expr), + AssignmentKind::Patch { + path_projection, + inner, + } => { + let mut projection = projection; + for &step in path_projection.as_slice() { + projection.push(step); + } + inner.apply(assignments, projection); + } + AssignmentKind::Apply(ops) => { + for op in ops { + op.apply(assignments, projection.clone()); + } + } + AssignmentKind::Unsupported(reason) => assignments.reject_unsupported(reason), + } + } +} + +// Assignment implements Assign +impl Assign for Assignment { + fn into_assignment(self) -> Assignment { + self + } +} + +/// Helper macro: generates `impl Assign<$target> for $source` with set +/// semantics by delegating to `IntoExpr`. Used alongside every `IntoExpr` +/// impl to keep the two traits in sync. +macro_rules! impl_assign_via_expr { + // Simple: impl Assign for S + ($source:ty => $target:ty) => { + impl super::assignment::Assign<$target> for $source { + fn into_assignment(self) -> super::assignment::Assignment<$target> { + $crate::stmt::set( + super::IntoExpr::<$target>::into_expr(self), + ) + } + } + }; + // Generic: impl Assign for Source where bounds + // Uses { } instead of [ ] to avoid parsing ambiguity with array types. + ({ $($gen:tt)* } $source:ty => $target:ty) => { + impl<$($gen)*> super::assignment::Assign<$target> for $source { + fn into_assignment(self) -> super::assignment::Assignment<$target> { + $crate::stmt::set( + super::IntoExpr::<$target>::into_expr(self), + ) + } + } + }; +} + +// Make the macro available to into_expr.rs (sibling module) +pub(super) use impl_assign_via_expr; + +/// Insert a value into a collection field. +/// +/// Takes an expression of `T` (a single item) and produces an assignment for +/// `List` (the collection). The returned [`Assignment`] can be passed to any +/// update builder setter that accepts `impl Assign>`. +/// +/// [`Assignments`]: toasty_core::stmt::Assignments +/// +/// # Examples +/// +/// ```ignore +/// user.update() +/// .todos(stmt::insert(Todo::create().title("Buy groceries"))) +/// .exec(&mut db) +/// .await?; +/// ``` +pub fn insert(expr: impl IntoExpr) -> Assignment> { + Assignment { + kind: AssignmentKind::Insert(expr.into_expr().untyped), + _p: PhantomData, + } +} + +/// Remove a value from a collection field. +/// +/// Takes an expression of `T` (the item to remove) and produces an assignment +/// for `List` (the collection). +/// +/// What "remove" means depends on the field kind: +/// - **`Vec` field**: every element equal to the value is removed, +/// atomically against the existing column value. Backends that advertise +/// [`Capability::vec_remove`](toasty_core::driver::Capability::vec_remove) +/// support this — currently PostgreSQL only; other backends return an +/// error. +/// - **Has-many relation**: the related record is dissociated. With an +/// optional foreign key the FK is set to `NULL`; with a required foreign +/// key the related record is deleted. +/// +/// # Examples +/// +/// ```ignore +/// // Vec field +/// user.update() +/// .tags(stmt::remove("admin")) +/// .exec(&mut db) +/// .await?; +/// +/// // Has-many relation +/// user.update() +/// .todos(stmt::remove(&todo_a)) +/// .exec(&mut db) +/// .await?; +/// ``` +pub fn remove(expr: impl IntoExpr) -> Assignment> { + Assignment { + kind: AssignmentKind::Remove(expr.into_expr().untyped), + _p: PhantomData, + } +} + +/// Replace a field's value entirely. +/// +/// For collection fields, `set` replaces the entire collection: all current +/// members are disassociated (following the same optional/required foreign key +/// rules as [`remove`]), then the new set is associated. +/// +/// Pass an empty slice to clear the collection: +/// +/// ```ignore +/// user.update() +/// .todos(stmt::set::>([])) +/// .exec(&mut db) +/// .await?; +/// ``` +/// +/// For scalar fields, `set` is equivalent to passing a plain value (the +/// setter already defaults to set semantics). +/// +/// # Examples +/// +/// ```ignore +/// // Replace all todos +/// user.update() +/// .todos(stmt::set([ +/// Todo::create().title("Only todo"), +/// ])) +/// .exec(&mut db) +/// .await?; +/// ``` +pub fn set(expr: impl IntoExpr) -> Assignment { + Assignment { + kind: AssignmentKind::Set(expr.into_expr().untyped), + _p: PhantomData, + } +} + +/// Append one element to an ordered collection field (e.g. `Vec`). +/// +/// Takes an expression of `T` (the element to append) and produces an +/// assignment for `List` (the collection). The append is atomic +/// against the existing column value on every supported backend. +/// +/// After `.exec()`, the instance's field reflects the post-update value +/// (old contents followed by the appended element). +/// +/// # Examples +/// +/// ```ignore +/// user.update() +/// .tags(stmt::push("admin")) +/// .exec(&mut db) +/// .await?; +/// ``` +pub fn push(expr: impl IntoExpr) -> Assignment> { + let element = expr.into_expr().untyped; + Assignment { + kind: AssignmentKind::Append(stmt::Expr::list([element])), + _p: PhantomData, + } +} + +/// Append every element of a list to an ordered collection field. +/// +/// Takes a list-shaped expression (anything that converts to `List` +/// — `Vec`, `[T; N]`, `&[T]`, …) and produces an assignment for +/// `List`. Elements are appended in order and the operation is +/// atomic against the existing column value, same as [`push`]. +/// +/// After `.exec()`, the instance's field reflects the post-update value. +/// `stmt::extend(iter)` of an empty iterator is a no-op. +/// +/// # Examples +/// +/// ```ignore +/// user.update() +/// .tags(stmt::extend(["admin", "verified"])) +/// .exec(&mut db) +/// .await?; +/// ``` +pub fn extend(items: impl IntoExpr>) -> Assignment> { + Assignment { + kind: AssignmentKind::Append(items.into_expr().untyped), + _p: PhantomData, + } +} + +/// Remove every element from an ordered collection field. +/// +/// Produces an assignment for `List` that replaces the column with an +/// empty list. Equivalent to passing an empty Vec to the field setter, +/// just more explicit at the call site. +/// +/// # Examples +/// +/// ```ignore +/// user.update() +/// .tags(stmt::clear()) +/// .exec(&mut db) +/// .await?; +/// ``` +pub fn clear() -> Assignment> { + Assignment { + kind: AssignmentKind::Set(stmt::Expr::list(Vec::::new())), + _p: PhantomData, + } +} + +/// Drop the last element of an ordered collection field. +/// +/// Produces an assignment for `List` that removes the trailing element +/// of the existing column value. The popped element is discarded — the +/// assignment API does not return values. Empty-collection is a no-op +/// rather than an error; for failure-on-empty semantics, filter the +/// collection first. +/// +/// Atomic against the existing column value. Backends that advertise +/// [`Capability::vec_pop`](toasty_core::driver::Capability::vec_pop) +/// support this — currently PostgreSQL only; other backends return an +/// error. +/// +/// # Examples +/// +/// ```ignore +/// user.update() +/// .tags(stmt::pop()) +/// .exec(&mut db) +/// .await?; +/// ``` +pub fn pop() -> Assignment> { + Assignment { + kind: AssignmentKind::Pop, + _p: PhantomData, + } +} + +/// Drop the element at the given index from an ordered collection field. +/// +/// Takes a `usize`-typed index and produces an assignment for `List` +/// that removes the element at that position. Out-of-bounds indices are +/// a no-op rather than an error — per-row failure semantics on a bulk +/// update are rarely useful. +/// +/// Atomic against the existing column value. Backends that advertise +/// [`Capability::vec_remove_at`](toasty_core::driver::Capability::vec_remove_at) +/// support this — currently PostgreSQL only; other backends return an +/// error. +/// +/// # Examples +/// +/// ```ignore +/// user.update() +/// .tags(stmt::remove_at(0)) +/// .exec(&mut db) +/// .await?; +/// ``` +pub fn remove_at(idx: impl IntoExpr) -> Assignment> { + Assignment { + kind: AssignmentKind::RemoveAt(idx.into_expr().untyped), + _p: PhantomData, + } +} + +/// Partially update a sub-field of an embedded struct. +/// +/// Takes a [`Path`] (identifying which sub-field to update) and a +/// value (`impl Assign` — either a plain value or a nested +/// [`Assignment`] for deeper patching). Returns an [`Assignment`] +/// that can be passed to the parent field's setter. +/// +/// The path may step through embedded structs, and it may end at an +/// embedded enum field, which the patch then replaces as a whole. It may +/// not enter an enum variant: setting or incrementing a field inside a +/// variant is not supported on any backend. `patch` itself cannot fail, so +/// the update statement carrying such a patch is rejected as a whole with an +/// `unsupported_feature` error when it is executed, before any write. Change +/// a variant's field by assigning a whole enum value instead. +/// +/// # Examples +/// +/// ```ignore +/// // Update a single sub-field +/// user.update() +/// .address(stmt::patch(Address::fields().city(), "Portland")) +/// .exec(&mut db) +/// .await?; +/// +/// // Nested patching +/// company.update() +/// .headquarters(stmt::patch( +/// Office::fields().address().into(), +/// stmt::patch(Address::fields().city(), "Boston"), +/// )) +/// .exec(&mut db) +/// .await?; +/// +/// // Replace an enum field nested inside a struct +/// task.update() +/// .meta(stmt::patch( +/// Meta::fields().status().into(), +/// Status::Failed { reason: "flaky".to_string() }, +/// )) +/// .exec(&mut db) +/// .await?; +/// ``` +pub fn patch(path: Path, value: impl Assign) -> Assignment { + let inner = value.into_assignment(); + + // A variant root is dropped below along with the rest of the path root, + // which would leave the variant-local steps to name whichever field the + // parent type has at those indices. Reject the patch instead. + let kind = match path.untyped.root { + stmt::PathRoot::Model(_) => AssignmentKind::Patch { + path_projection: path.untyped.projection, + inner: Box::new(inner.kind), + }, + stmt::PathRoot::Variant { .. } => AssignmentKind::Unsupported( + "patching a field inside an enum variant is not supported; \ + assign a whole enum value instead" + .to_string(), + ), + }; + + Assignment { + kind, + _p: PhantomData, + } +} + +/// Apply multiple operations to a single field. +/// +/// Takes an array or [`Vec`] of [`Assignment`] and applies each in order. +/// +/// # Examples +/// +/// ```ignore +/// user.update() +/// .todos(stmt::apply([ +/// stmt::insert(Todo::create().title("Buy groceries")), +/// stmt::insert(Todo::create().title("Walk the dog")), +/// stmt::remove(&old_todo), +/// ])) +/// .exec(&mut db) +/// .await?; +/// ``` +pub fn apply(ops: impl IntoIterator>) -> Assignment { + let ops: Vec = ops.into_iter().map(|a| a.kind).collect(); + Assignment { + kind: AssignmentKind::Apply(ops), + _p: PhantomData, + } +} + +/// Add a value to a numeric field (`col = col + value`). +/// +/// The [`Numeric`] bound restricts `T` to types Toasty knows how to do +/// database arithmetic on, so misuse on a non-numeric column (e.g. +/// `String`) is a compile error rather than a runtime database error. +/// +/// The update is atomic against the existing column value on every backend. +/// Use this when you want a relative update — e.g. crediting a balance — +/// without a read-modify-write round trip from the client. +/// +/// # Examples +/// +/// ```ignore +/// user.update() +/// .balance(stmt::add(10)) +/// .exec(&mut db) +/// .await?; +/// ``` +pub fn add(value: impl IntoExpr) -> Assignment { + Assignment { + kind: AssignmentKind::Add(value.into_expr().untyped), + _p: PhantomData, + } +} + +/// Subtract a value from a numeric field (`col = col - value`). +/// +/// Mirror of [`add`]. See its docs for the [`Numeric`] bound. Atomic +/// against the existing column value on every backend. +/// +/// # Examples +/// +/// ```ignore +/// user.update() +/// .balance(stmt::subtract(5)) +/// .exec(&mut db) +/// .await?; +/// ``` +pub fn subtract(value: impl IntoExpr) -> Assignment { + Assignment { + kind: AssignmentKind::Subtract(value.into_expr().untyped), + _p: PhantomData, + } +} + +/// Increment a numeric field by one. +/// +/// The [`Numeric`] bound encodes the `1` literal in a value variant +/// matching the field type, so `stmt::increment()` works on every +/// supported numeric column (`i8`–`i64`, `u8`–`u64`, `f32`, `f64`) and +/// on user newtypes that implement [`Numeric`]. +/// +/// Atomic against the existing column value on every backend, same as +/// [`add`]. +/// +/// # Examples +/// +/// ```ignore +/// user.update() +/// .login_count(stmt::increment()) +/// .exec(&mut db) +/// .await?; +/// ``` +pub fn increment() -> Assignment { + Assignment { + kind: AssignmentKind::Add(stmt::Expr::Value(T::one())), + _p: PhantomData, + } +} + +/// Decrement a numeric field by one. +/// +/// Mirror of [`increment`]. See its docs for the [`Numeric`] bound and +/// the set of supported field types. +/// +/// # Examples +/// +/// ```ignore +/// user.update() +/// .lives_left(stmt::decrement()) +/// .exec(&mut db) +/// .await?; +/// ``` +pub fn decrement() -> Assignment { + Assignment { + kind: AssignmentKind::Subtract(stmt::Expr::Value(T::one())), + _p: PhantomData, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use toasty_core::schema::app::{ModelId, VariantId}; + + /// `field 1` of model 0, then field 0 of variant 0 of the enum at model 1. + fn path_into_variant() -> Path<(), i64> { + Path::<(), ()>::field_at(ModelId(0), 1) + .into_variant(VariantId { + model: ModelId(1), + index: 0, + }) + .chain(Path::<(), i64>::field_at(ModelId(1), 0)) + } + + #[test] + fn patch_into_variant_records_unsupported() { + let mut assignments = stmt::Assignments::new(); + patch(path_into_variant(), increment::()) + .assign(&mut assignments, stmt::Projection::from_index(2)); + + assert!(assignments.is_empty()); + assert_eq!(assignments.unsupported().len(), 1); + } + + #[test] + fn patch_into_variant_nested_in_struct_patch_records_unsupported() { + let mut assignments = stmt::Assignments::new(); + let outer = Path::<(), ()>::field_at(ModelId(0), 3); + patch(outer, patch(path_into_variant(), 9_i64)) + .assign(&mut assignments, stmt::Projection::from_index(2)); + + assert!(assignments.is_empty()); + assert_eq!(assignments.unsupported().len(), 1); + } +} diff --git a/packages/toasty/src/stmt/association.rs b/packages/toasty/src/stmt/association.rs new file mode 100644 index 00000000000..b26ca034d46 --- /dev/null +++ b/packages/toasty/src/stmt/association.rs @@ -0,0 +1,319 @@ +use super::{IntoExpr, IntoScope, IntoStatement, List, Path, Statement}; +use crate::schema::Model; +use std::{fmt, marker::PhantomData}; +use toasty_core::stmt; + +/// A typed handle to a model association (relation). +/// +/// `Association` represents a link between a source model and a target model, +/// such as a has-many or belongs-to relation. It wraps an untyped +/// [`stmt::Association`](toasty_core::stmt::Association) and carries a type `T` +/// that encodes the **returning type** — what executing the association query +/// produces: +/// +/// - `Association>` — a has-many relation, returns `Vec`. +/// - `Association` — a has-one or belongs-to relation, returns `M`. +/// +/// Associations are constructed by generated code (see [`many`](Association::many) +/// and [`one`](Association::one)). +/// They implement [`IntoStatement`] so they can be passed directly to +/// [`Db::exec`](crate::Db::exec). +pub struct Association { + pub(crate) untyped: stmt::Association, + _p: PhantomData, +} + +impl Association { + /// Construct a typed association from a raw untyped one. Used by + /// generated code that re-types an association after carrying it through + /// an untyped storage slot. + #[doc(hidden)] + pub fn from_untyped(untyped: stmt::Association) -> Self { + Self { + untyped, + _p: PhantomData, + } + } + + /// Construct an association from `source` following `path`, without + /// requiring the returning type `T` to be a model. + /// + /// Used by generated `#[has_many(via = …)]` navigation methods, whose + /// terminal may be a scalar (`Path>`). The [`many`](Self::many) / + /// [`one`](Self::one) constructors bound the element on [`Model`]; this one + /// only bounds the *source* model `S`, so it works for both relation- and + /// scalar-terminal vias. + /// + /// # Panics + /// + /// Panics if the root of `path` does not match the model id of `S`. + #[doc(hidden)] + pub fn from_source_and_path(source: super::Query>, path: Path) -> Self { + assert_eq!(path.untyped.root.as_model_unwrap(), S::id()); + + Self { + untyped: stmt::Association { + source: Box::new(source.untyped), + path: path.untyped, + }, + _p: PhantomData, + } + } +} + +impl Association> { + /// Create a has-many association from `source` following `path`. + /// + /// # Panics + /// + /// Panics if the root of `path` does not match the model id of `T`. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// # #[derive(Debug, toasty::Model)] + /// # struct Todo { + /// # #[key] + /// # id: i64, + /// # user_id: i64, + /// # title: String, + /// # } + /// use toasty::stmt::{Association, List, Query}; + /// use toasty::schema::Model; + /// + /// let source = Query::>::all().filter(User::fields().id().eq(1)); + /// let path = User::path_field::>(2); + /// let _assoc = Association::many(source, path); + /// ``` + pub fn many(source: super::Query>, path: Path>) -> Self { + assert_eq!(path.untyped.root.as_model_unwrap(), T::id()); + + Self { + untyped: stmt::Association { + source: Box::new(source.untyped), + path: path.untyped, + }, + _p: PhantomData, + } + } + + /// Create a has-many association through a singular (has-one / belongs-to) + /// path. Because the source is a query that may match multiple rows, the + /// result is still a list. + /// + /// # Panics + /// + /// Panics if the root of `path` does not match the model id of `T`. + /// + pub(crate) fn many_via_one(source: super::Query>, path: Path) -> Self { + assert_eq!(path.untyped.root.as_model_unwrap(), T::id()); + + Self { + untyped: stmt::Association { + source: Box::new(source.untyped), + path: path.untyped, + }, + _p: PhantomData, + } + } + + /// Insert an associated record into this has-many relation. + /// + /// Converts the association into an update statement that adds `expr` to + /// the relation's field on the source model. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// # #[derive(Debug, toasty::Model)] + /// # struct Todo { + /// # #[key] + /// # id: i64, + /// # user_id: i64, + /// # title: String, + /// # } + /// use toasty::stmt::{Association, Expr, List, Query}; + /// use toasty::schema::Model; + /// + /// let source = Query::>::all().filter(User::fields().id().eq(1)); + /// let path = User::path_field::>(2); + /// let assoc = Association::many(source, path); + /// + /// let todo_expr = Expr::::from_untyped( + /// toasty_core::stmt::Value::from(42_i64), + /// ); + /// let _stmt = assoc.insert(todo_expr); + /// ``` + pub fn insert(self, expr: impl IntoExpr) -> Statement<()> { + let [index] = self.untyped.path.projection.as_slice() else { + todo!() + }; + + let mut stmt = self.untyped.source.update(); + stmt.assignments.insert(*index, expr.into_expr().untyped); + + Statement { + untyped: stmt.into(), + _p: PhantomData, + } + } + + /// Remove an associated record from this has-many relation. + /// + /// Converts the association into an update statement that removes `expr` + /// from the relation's field on the source model. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// # #[derive(Debug, toasty::Model)] + /// # struct Todo { + /// # #[key] + /// # id: i64, + /// # user_id: i64, + /// # title: String, + /// # } + /// use toasty::stmt::{Association, Expr, List, Query}; + /// use toasty::schema::Model; + /// + /// let source = Query::>::all().filter(User::fields().id().eq(1)); + /// let path = User::path_field::>(2); + /// let assoc = Association::many(source, path); + /// + /// // Remove a todo by its expression + /// let todo_expr = Expr::::from_untyped( + /// toasty_core::stmt::Value::from(42_i64), + /// ); + /// let _stmt = assoc.remove(todo_expr); + /// ``` + pub fn remove(self, expr: impl IntoExpr) -> Statement<()> { + let [index] = self.untyped.path.projection.as_slice() else { + todo!() + }; + let mut stmt = self.untyped.source.update(); + stmt.assignments.remove(*index, expr.into_expr().untyped); + + Statement { + untyped: stmt.into(), + _p: PhantomData, + } + } + + /// Append a single field step to this association's path, retargeting it + /// to `NewTarget`. Used by macro-generated chain methods on the `Many` + /// struct — `field_index` must identify a relation field on `M`. + #[doc(hidden)] + pub fn chain_field(mut self, field_index: usize) -> Association> { + self.untyped.path.projection.push(field_index); + Association { + untyped: self.untyped, + _p: PhantomData, + } + } +} + +impl IntoStatement for Association> { + type Returning = List; + + fn into_statement(self) -> Statement> { + let query = stmt::Query::builder(stmt::SourceModel { + id: T::id(), + via: Some(self.untyped), + }) + .build(); + Statement::from_untyped_stmt(query.into()) + } +} + +impl IntoScope for Association> { + fn into_scope(self) -> Statement> { + self.into_statement() + } +} + +impl Association { + /// Create a has-one or belongs-to association from `source` following + /// `path`. + /// + /// # Panics + /// + /// Panics if the root of `path` does not match the model id of `T`. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// # #[derive(Debug, toasty::Model)] + /// # struct Todo { + /// # #[key] + /// # id: i64, + /// # user_id: i64, + /// # title: String, + /// # } + /// use toasty::stmt::{Association, List, Query}; + /// use toasty::schema::Model; + /// + /// let source = Query::>::all().filter(Todo::fields().id().eq(1)); + /// let path = Todo::path_field::(1); + /// let _assoc = Association::one(source, path); + /// ``` + pub fn one(source: super::Query>, path: Path) -> Self { + assert_eq!(path.untyped.root.as_model_unwrap(), T::id()); + + Self { + untyped: stmt::Association { + source: Box::new(source.untyped), + path: path.untyped, + }, + _p: PhantomData, + } + } +} + +impl IntoStatement for Association { + type Returning = List; + + fn into_statement(self) -> Statement> { + let query = stmt::Query::builder(stmt::SourceModel { + id: T::id(), + via: Some(self.untyped), + }) + .build(); + Statement::from_untyped_stmt(query.into()) + } +} + +impl IntoScope for Association { + fn into_scope(self) -> Statement> { + self.into_statement() + } +} + +impl fmt::Debug for Association { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + self.untyped.fmt(fmt) + } +} diff --git a/packages/toasty/src/stmt/batch.rs b/packages/toasty/src/stmt/batch.rs new file mode 100644 index 00000000000..8ebafd4b6d8 --- /dev/null +++ b/packages/toasty/src/stmt/batch.rs @@ -0,0 +1,58 @@ +use super::{Expr, IntoExpr, IntoStatement, Statement}; +use crate::schema::Load; +use crate::{Executor, Result}; + +use toasty_core::stmt; + +/// A batch of queries composed into a single statement. +/// +/// Created by [`batch()`]. The composed statement flows through +/// the standard engine pipeline and the result is deserialized via [`Load`]. +pub struct Batch { + stmt: Statement, +} + +/// Compose multiple independent queries into a single batched statement. +/// +/// The queries are sent to the database in a single round-trip and the results +/// come back as a typed tuple matching the input queries. +/// +/// # Examples +/// +/// ```ignore +/// let (active_users, recent_posts) = toasty::batch(( +/// User::find_by_active(true), +/// Post::find_recent(100), +/// )).exec(&mut db).await?; +/// ``` +pub fn batch(queries: Q) -> Batch +where + Q::Returning: Load, +{ + Batch { + stmt: queries.into_statement(), + } +} + +impl Batch { + /// Execute the batched queries and return the deserialized results. + pub async fn exec(self, executor: &mut dyn Executor) -> Result { + executor.exec(self.stmt).await + } +} + +impl From> for Batch { + fn from(stmt: Statement) -> Self { + Batch { stmt } + } +} + +impl IntoExpr for Batch { + fn into_expr(self) -> Expr { + Expr::from_untyped(stmt::Expr::stmt(self.stmt.untyped)) + } + + fn by_ref(&self) -> Expr { + Expr::from_untyped(stmt::Expr::stmt(self.stmt.untyped.clone())) + } +} diff --git a/packages/toasty/src/stmt/create_many.rs b/packages/toasty/src/stmt/create_many.rs new file mode 100644 index 00000000000..132b9b3b60a --- /dev/null +++ b/packages/toasty/src/stmt/create_many.rs @@ -0,0 +1,136 @@ +use crate::{ + Executor, Result, + schema::Model, + stmt::{Expr, Insert, IntoExpr, IntoInsert, List}, +}; +use toasty_core::stmt as core_stmt; + +/// A builder for inserting multiple records of the same model in a single +/// statement. +/// +/// Records are accumulated with [`item`](CreateMany::item) or +/// [`with_item`](CreateMany::with_item), then executed with +/// [`exec`](CreateMany::exec). `CreateMany` also implements [`IntoExpr`] so a +/// batch insert can be embedded inside another statement. +/// +/// # Examples +/// +/// ```no_run +/// # tokio::runtime::Runtime::new().unwrap().block_on(async { +/// # #[derive(Debug, toasty::Model)] +/// # struct User { +/// # #[key] +/// # #[auto] +/// # id: i64, +/// # name: String, +/// # } +/// # let driver = toasty_driver_sqlite::Sqlite::in_memory(); +/// # let mut db = toasty::Db::builder().models(toasty::models!(User)).build(driver).await.unwrap(); +/// # db.push_schema().await.unwrap(); +/// let users = User::create_many() +/// .item(User::create().name("Alice")) +/// .item(User::create().name("Bob")) +/// .exec(&mut db) +/// .await +/// .unwrap(); +/// assert_eq!(users.len(), 2); +/// # }); +/// ``` +pub struct CreateMany { + stmts: Vec>, +} + +impl CreateMany { + /// Append a record from any value that implements [`IntoInsert`] + /// for model `M`. + /// + /// Returns `self` for method chaining. + pub fn item(mut self, item: impl IntoInsert) -> Self { + let stmt = item.into_insert(); + assert!( + stmt.untyped.source.single, + "BUG: insert statement should have `single` flag set" + ); + self.stmts.push(stmt); + self + } + + /// Append a record using a closure that configures the model's generated + /// create builder. + /// + /// `f` receives a default create builder and returns it after setting the + /// desired fields. + /// + /// Returns `self` for method chaining. + pub fn with_item( + mut self, + f: impl FnOnce(::Create) -> ::Create, + ) -> Self { + let create = f(M::new_create()); + let stmt = create.into_insert(); + assert!( + stmt.untyped.source.single, + "BUG: insert statement should have `single` flag set" + ); + self.stmts.push(stmt); + self + } + + /// Convert the collected inserts into a list expression suitable for + /// embedding in a parent insert statement (e.g., as a nested HasMany value). + /// + /// Unlike `exec`, this does not run any database query. + fn into_expr(self) -> Expr> { + if self.stmts.is_empty() { + return Expr::from_untyped( + core_stmt::Expr::list(std::iter::empty::()), + ); + } + let mut stmts = self.stmts.into_iter(); + let mut merged = stmts.next().unwrap(); + for stmt in stmts { + merged.merge(stmt); + } + // Clear the single flag so the engine handles multi-row inserts correctly. + merged.untyped.source.single = false; + merged.into_list_expr() + } + + /// Execute the batch insert and return the created records. + /// + /// Returns an empty `Vec` if no records were queued. + pub async fn exec(self, executor: &mut dyn Executor) -> Result> { + if self.stmts.is_empty() { + return Ok(vec![]); + } + + let mut stmts = self.stmts.into_iter(); + let mut merged = stmts.next().unwrap(); + + for stmt in stmts { + merged.merge(stmt); + } + + merged.untyped.source.single = false; + + let stmt: crate::Statement> = + crate::Statement::from_untyped_stmt(merged.untyped.into()); + executor.exec(stmt).await + } +} + +impl IntoExpr> for CreateMany { + fn into_expr(self) -> Expr> { + self.into_expr() + } + + fn by_ref(&self) -> Expr> { + todo!() + } +} + +impl Default for CreateMany { + fn default() -> Self { + Self { stmts: vec![] } + } +} diff --git a/packages/toasty/src/stmt/delete.rs b/packages/toasty/src/stmt/delete.rs new file mode 100644 index 00000000000..cf67e639fe4 --- /dev/null +++ b/packages/toasty/src/stmt/delete.rs @@ -0,0 +1,129 @@ +use super::{IntoStatement, Statement}; +use crate::{Executor, Result, schema::Load}; +use std::marker::PhantomData; +use toasty_core::stmt; + +/// A typed delete statement. +/// +/// `Delete` removes records that match the selection built by the +/// originating [`Query`]. Obtain one by calling [`Query::delete`]. +/// +/// The type parameter `T` is the **returning type**, not the model being +/// deleted. Currently `Query::delete` always produces `Delete<()>` because +/// deletes do not return the removed records. +/// +/// # Execution +/// +/// Call [`exec`](Delete::exec) to run the delete, or convert it into a +/// [`Statement`] with [`IntoStatement`] for batch execution. +/// +/// ``` +/// # tokio::runtime::Runtime::new().unwrap().block_on(async { +/// # #[derive(Debug, toasty::Model)] +/// # struct User { +/// # #[key] +/// # id: i64, +/// # name: String, +/// # } +/// # let driver = toasty_driver_sqlite::Sqlite::in_memory(); +/// # let mut db = toasty::Db::builder().models(toasty::models!(User)).build(driver).await.unwrap(); +/// # db.push_schema().await.unwrap(); +/// use toasty::stmt::{List, Query}; +/// +/// Query::>::all().filter(User::fields().id().eq(1)) +/// .delete() +/// .exec(&mut db) +/// .await +/// .unwrap(); +/// # }); +/// ``` +pub struct Delete { + pub(crate) untyped: stmt::Delete, + _p: PhantomData, +} + +impl Delete { + /// Wrap a raw untyped [`stmt::Delete`](toasty_core::stmt::Delete). + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// use toasty::stmt::{Delete, List, Query}; + /// + /// // Build a delete from a query + /// let delete: Delete<()> = Query::>::all().delete(); + /// ``` + pub const fn from_untyped(untyped: stmt::Delete) -> Self { + Self { + untyped, + _p: PhantomData, + } + } +} + +impl Delete { + /// Attach a condition to this delete. The condition is evaluated after the + /// filter; if it fails, the operation returns an error (unlike a filter + /// failure, which silently produces count 0). + pub fn set_condition(mut self, condition: toasty_core::stmt::Condition) -> Self { + self.untyped.condition = condition; + self + } +} + +impl Delete { + /// Execute this delete statement against the given executor. + /// + /// Returns `Ok(())` on success. Any matching records are removed from the + /// database. + /// + /// # Examples + /// + /// ``` + /// # tokio::runtime::Runtime::new().unwrap().block_on(async { + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// # let driver = toasty_driver_sqlite::Sqlite::in_memory(); + /// # let mut db = toasty::Db::builder().models(toasty::models!(User)).build(driver).await.unwrap(); + /// # db.push_schema().await.unwrap(); + /// User::filter(User::fields().id().eq(1)) + /// .delete() + /// .exec(&mut db) + /// .await + /// .unwrap(); + /// # }); + /// ``` + pub async fn exec(self, executor: &mut dyn Executor) -> Result { + executor.exec(self.into()).await + } +} + +impl IntoStatement for Delete { + type Returning = T; + + fn into_statement(self) -> Statement { + Statement { + untyped: self.untyped.into(), + _p: PhantomData, + } + } +} + +impl From> for Statement { + fn from(value: Delete) -> Self { + Self { + untyped: value.untyped.into(), + _p: PhantomData, + } + } +} diff --git a/packages/toasty/src/stmt/expr.rs b/packages/toasty/src/stmt/expr.rs new file mode 100644 index 00000000000..04edf816648 --- /dev/null +++ b/packages/toasty/src/stmt/expr.rs @@ -0,0 +1,323 @@ +use super::{Insert, IntoExpr, List}; +use std::marker::PhantomData; +use std::ops::Not; +use toasty_core::stmt; + +/// A typed expression in the Toasty query language. +/// +/// `Expr` wraps an untyped AST expression node and tags it with a Rust type +/// `T` that represents the expression's value type. Common instantiations: +/// +/// - `Expr` — a boolean filter expression (comparisons, `and`, `or`, `not`). +/// - `Expr`, `Expr`, etc. — scalar value expressions. +/// - `Expr>` — a nullable expression with [`is_none`](Expr::is_none) +/// and [`is_some`](Expr::is_some) helpers. +/// - `Expr>` — a list expression (see [`Expr::list`]). +/// +/// Expressions are built from [`Path`] comparisons, literal values via +/// [`IntoExpr`], and combinators like [`and`](Expr::and) and [`or`](Expr::or). +#[derive(Debug)] +pub struct Expr { + pub(crate) untyped: stmt::Expr, + pub(crate) _p: PhantomData, +} + +impl Expr { + /// Create an expression from the given value. + pub(crate) fn from_value(value: stmt::Value) -> Self { + Self { + untyped: stmt::Expr::Value(value), + _p: PhantomData, + } + } + + /// Wrap a raw untyped expression, tagging it with type `T`. + /// + /// # Examples + /// + /// ``` + /// # use toasty::stmt::Expr; + /// # use toasty_core::stmt as core_stmt; + /// let expr = Expr::::from_untyped(core_stmt::Expr::Value( + /// core_stmt::Value::from(42_i64), + /// )); + /// ``` + pub fn from_untyped(untyped: impl Into) -> Self { + Self { + untyped: untyped.into(), + _p: PhantomData, + } + } + + /// Re-tag this expression with a different type `U`. + /// + /// This performs no runtime conversion — the underlying AST node is + /// unchanged. Use this when the type system needs a different type tag but + /// the expression itself is compatible (e.g., widening `Expr` to + /// `Expr>`). + /// + /// # Examples + /// + /// ``` + /// # use toasty::stmt::{Expr, IntoExpr}; + /// let expr: Expr = 42_i64.into_expr(); + /// let optional: Expr> = expr.cast(); + /// ``` + pub fn cast(self) -> Expr { + Expr { + untyped: self.untyped, + _p: PhantomData, + } + } + + /// Test whether this expression equals `rhs`. + /// + /// When either side selects an enum variant (a path such as + /// `contact().email().address()`), the filter also requires that + /// variant: the engine attaches the check when it normalizes the + /// statement. + pub fn eq(self, rhs: impl IntoExpr) -> Expr { + let rhs = rhs.into_expr().untyped; + Expr::from_untyped(stmt::Expr::eq(self.untyped, rhs)) + } + + /// Test whether this expression does not equal `rhs`. + /// + /// Like [`eq`](Expr::eq), the filter requires any variant either side + /// selects: rows of other variants do not match. + pub fn ne(self, rhs: impl IntoExpr) -> Expr { + let rhs = rhs.into_expr().untyped; + Expr::from_untyped(stmt::Expr::ne(self.untyped, rhs)) + } + + /// Test whether this expression is greater than `rhs`. + pub fn gt(self, rhs: impl IntoExpr) -> Expr { + let rhs = rhs.into_expr().untyped; + Expr::from_untyped(stmt::Expr::gt(self.untyped, rhs)) + } + + /// Test whether this expression is greater than or equal to `rhs`. + pub fn ge(self, rhs: impl IntoExpr) -> Expr { + let rhs = rhs.into_expr().untyped; + Expr::from_untyped(stmt::Expr::ge(self.untyped, rhs)) + } + + /// Test whether this expression is less than `rhs`. + pub fn lt(self, rhs: impl IntoExpr) -> Expr { + let rhs = rhs.into_expr().untyped; + Expr::from_untyped(stmt::Expr::lt(self.untyped, rhs)) + } + + /// Test whether this expression is less than or equal to `rhs`. + pub fn le(self, rhs: impl IntoExpr) -> Expr { + let rhs = rhs.into_expr().untyped; + Expr::from_untyped(stmt::Expr::le(self.untyped, rhs)) + } +} + +impl Expr> { + /// Build a list expression from an iterator of items. + /// + /// Each item is converted to an `Expr` via [`IntoExpr`]. The resulting + /// expression represents a literal list value. + /// + /// # Examples + /// + /// ``` + /// # use toasty::stmt::{Expr, List}; + /// let ids = Expr::>::list([1_i64, 2, 3]); + /// ``` + pub fn list(items: impl IntoIterator) -> Self + where + I: IntoExpr, + { + Self::from_untyped(stmt::Expr::list( + items.into_iter().map(|item| item.into_expr().untyped), + )) + } +} + +impl Expr { + /// Combine two boolean expressions with logical AND. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # age: i64, + /// # } + /// let filter = User::fields().name().eq("Alice") + /// .and(User::fields().age().gt(18)); + /// ``` + pub fn and(self, rhs: impl IntoExpr) -> Self { + Self::from_untyped(stmt::Expr::and(self.untyped, rhs.into_expr().untyped)) + } + + /// Combine an iterator of boolean expressions with logical AND. + /// + /// Returns `true` (no filter) when the iterator is empty. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # age: i64, + /// # } + /// use toasty::stmt::Expr; + /// + /// let conditions = vec![ + /// User::fields().name().eq("Alice"), + /// User::fields().age().gt(18), + /// ]; + /// let combined = Expr::and_all(conditions); + /// ``` + pub fn and_all(exprs: impl IntoIterator) -> Self + where + E: IntoExpr, + { + exprs + .into_iter() + .map(|expr| expr.into_expr().untyped) + .reduce(stmt::Expr::and) + .map(Self::from_untyped) + .unwrap_or_else(|| Self::from_untyped(true)) + } + + /// Combine two boolean expressions with logical OR. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// let filter = User::fields().name().eq("Alice") + /// .or(User::fields().name().eq("Bob")); + /// ``` + pub fn or(self, rhs: impl IntoExpr) -> Self { + Self::from_untyped(stmt::Expr::or(self.untyped, rhs.into_expr().untyped)) + } + + /// Negate this boolean expression. + /// + /// Equivalent to the `!` operator (which is also implemented via [`Not`]). + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// let not_alice = User::fields().name().eq("Alice").not(); + /// ``` + #[allow(clippy::should_implement_trait)] + pub fn not(self) -> Self { + !self + } + + /// Test whether `lhs` is contained in `rhs`. + /// + /// This is the associated-function form of [`in_list`](super::in_list). + /// Both single values and tuples (composite keys) are supported. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// use toasty::stmt::Expr; + /// + /// let filter = Expr::in_list(User::fields().id(), [1_i64, 2, 3]); + /// ``` + pub fn in_list(lhs: L, rhs: R) -> Self + where + L: IntoExpr, + R: IntoExpr>, + { + Self::from_untyped(stmt::Expr::in_list( + lhs.into_expr().untyped, + rhs.into_expr().untyped, + )) + } +} + +impl Not for Expr { + type Output = Self; + + fn not(self) -> Self { + Self::from_untyped(stmt::Expr::not(self.untyped)) + } +} + +impl Expr> { + /// Test whether this optional expression is `NULL`. + /// + /// # Examples + /// + /// ``` + /// # use toasty::stmt::{Expr, IntoExpr}; + /// let expr: Expr> = None::.into_expr(); + /// let _is_null: Expr = expr.is_none(); + /// ``` + pub fn is_none(self) -> Expr { + Expr::from_untyped(stmt::Expr::is_null(self.untyped)) + } + + /// Test whether this optional expression is not `NULL`. + /// + /// # Examples + /// + /// ``` + /// # use toasty::stmt::{Expr, IntoExpr}; + /// let expr: Expr> = Some(42_i64).into_expr(); + /// let _is_not_null: Expr = expr.is_some(); + /// ``` + pub fn is_some(self) -> Expr { + Expr::from_untyped(stmt::Expr::is_not_null(self.untyped)) + } +} + +impl Clone for Expr { + fn clone(&self) -> Self { + Self { + untyped: self.untyped.clone(), + _p: PhantomData, + } + } +} + +impl From> for stmt::Expr { + fn from(value: Expr) -> Self { + value.untyped + } +} + +impl From> for Expr { + fn from(value: Insert) -> Self { + Self::from_untyped(stmt::Expr::Stmt(value.untyped.into())) + } +} + +impl From> for Expr> { + fn from(value: Insert) -> Self { + Self::from_untyped(stmt::Expr::Stmt(value.untyped.into())) + } +} diff --git a/packages/toasty/src/stmt/include.rs b/packages/toasty/src/stmt/include.rs new file mode 100644 index 00000000000..d1e31b4604b --- /dev/null +++ b/packages/toasty/src/stmt/include.rs @@ -0,0 +1,71 @@ +use std::marker::PhantomData; + +use toasty_core::stmt; + +use super::{Expr, Path, Query}; + +/// A typed wrapper around an untyped [`stmt::Include`] that carries the +/// origin model and the relation target type. +/// +/// Produced by passing a relation path — optionally with a `.filter(...)` +/// predicate — to `.include(...)`. +pub struct Include { + pub(crate) untyped: stmt::Include, + _p: PhantomData<(Origin, T)>, +} + +impl Include { + #[doc(hidden)] + pub fn from_path_and_query(path: Path, query: Query) -> Self { + Self { + untyped: stmt::Include::with_query(path.untyped, query.untyped), + _p: PhantomData, + } + } +} + +impl Include { + /// Restricts the related rows loaded by this include. + pub fn filter(mut self, predicate: Expr) -> Self { + self.query_mut().add_filter(predicate.untyped); + self + } + + /// Orders the related rows loaded by this include. + pub fn order_by(mut self, order_by: impl Into) -> Self { + let order_by = order_by.into(); + match &mut self.query_mut().order_by { + Some(existing) => existing.exprs.extend(order_by.exprs), + slot @ None => *slot = Some(order_by), + } + self + } + + fn query_mut(&mut self) -> &mut stmt::Query { + self.untyped + .query + .as_mut() + .expect("include modifiers require a related-model query") + } +} + +impl From> for Include { + fn from(path: Path) -> Self { + Self { + untyped: stmt::Include::new(path.untyped), + _p: PhantomData, + } + } +} + +impl From> for stmt::Include { + fn from(value: Include) -> Self { + value.untyped + } +} + +impl From> for stmt::Include { + fn from(path: Path) -> Self { + stmt::Include::new(path.untyped) + } +} diff --git a/packages/toasty/src/stmt/insert.rs b/packages/toasty/src/stmt/insert.rs new file mode 100644 index 00000000000..41b85625865 --- /dev/null +++ b/packages/toasty/src/stmt/insert.rs @@ -0,0 +1,206 @@ +use super::{Expr, IntoScope, List}; +use crate::schema::Model; +use std::{fmt, marker::PhantomData}; +use toasty_core::stmt; + +/// A typed insert statement for model `M`. +/// +/// `Insert` represents one or more records to be created. Generated +/// create-builders (e.g., `User::create()`) produce `Insert` values under the +/// hood. +/// +/// The type parameter `M` is the **returning type** — `exec()` produces the +/// newly created `M` instance(s). For inserts, the returning type always +/// matches the model being inserted. +/// +/// Field values are set with [`set`](Insert::set). Collection fields can be +/// extended with [`insert_all`](Insert::insert_all). +pub struct Insert { + pub(crate) untyped: stmt::Insert, + _p: PhantomData, +} + +impl Insert { + /// Create an insert statement with a single blank record. + /// + /// Fields marked `#[auto]` are set to [`Expr::Default`](toasty_core::stmt::Expr::Default) + /// (the database generates the value). All other fields are initialized to + /// `NULL`. The caller must fill in required fields with [`set`](Insert::set) + /// before executing; the blank record is not guaranteed to be valid on its + /// own. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// use toasty::stmt::Insert; + /// + /// let mut insert = Insert::::blank_single(); + /// // Fill in the required fields + /// insert.set(0, toasty_core::stmt::Value::from(1_i64)); + /// insert.set(1, toasty_core::stmt::Value::from("Alice")); + /// ``` + pub fn blank_single() -> Self { + Self { + untyped: stmt::Insert { + target: stmt::InsertTarget::Model(M::id()), + source: stmt::Query::new_single(vec![ + stmt::ExprRecord::from_vec( + M::schema() + .as_root_unwrap() + .fields + .iter() + .map(|field| match field.auto() { + Some(_) => stmt::Expr::Default, + None => stmt::Expr::Value(stmt::Value::Null), + }) + .collect(), + ) + .into(), + ]), + upsert: None, + returning: Some(stmt::Returning::Model { include: vec![] }), + }, + _p: PhantomData, + } + } + + /// Set the scope of the insert. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// use toasty::stmt::Insert; + /// + /// let mut insert = Insert::::blank_single(); + /// // Scope the insert to all users (used by association inserts) + /// insert.set_scope(User::all()); + /// ``` + pub fn set_scope(&mut self, scope: S) + where + S: IntoScope, + { + self.untyped.target = + stmt::InsertTarget::Scope(Box::new(scope.into_scope().into_untyped_query())); + } + + /// Set the value of the field at `field` index in the current record. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// use toasty::stmt::Insert; + /// + /// let mut insert = Insert::::blank_single(); + /// insert.set(0, toasty_core::stmt::Value::from(1_i64)); + /// insert.set(1, toasty_core::stmt::Value::from("Alice")); + /// ``` + pub fn set(&mut self, field: usize, expr: impl Into) { + *self.expr_mut(field) = expr.into(); + } + + /// Merge a list expression into the list field at `field` index, + /// extending any existing list. + /// + /// If the field is currently `NULL`, it is replaced with `expr`. If both + /// are lists, the items from `expr` are appended to the existing list. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// use toasty::stmt::Insert; + /// + /// let mut insert = Insert::::blank_single(); + /// let list = toasty_core::stmt::Expr::list([ + /// toasty_core::stmt::Expr::Value(toasty_core::stmt::Value::from("a")), + /// toasty_core::stmt::Expr::Value(toasty_core::stmt::Value::from("b")), + /// ]); + /// insert.insert_all(1, list); + /// ``` + pub fn insert_all(&mut self, field: usize, expr: impl Into) { + let target = self.expr_mut(field); + let incoming = expr.into(); + + match target { + stmt::Expr::Value(stmt::Value::Null) => { + *target = incoming; + } + stmt::Expr::List(existing) => { + if let stmt::Expr::List(incoming_list) = incoming { + existing.items.extend(incoming_list.items); + } else { + existing.items.push(incoming); + } + } + _ => todo!("existing={target:#?}; expr={:#?}", incoming), + } + } + + pub(crate) fn merge(&mut self, stmt: Self) { + self.untyped.merge(stmt.untyped); + } + + fn expr_mut(&mut self, field: usize) -> &mut stmt::Expr { + &mut self.current_mut()[field] + } + + /// Returns the current record being updated + fn current_mut(&mut self) -> &mut stmt::ExprRecord { + let values = self.untyped.source.body.as_values_mut_unwrap(); + values.rows.last_mut().unwrap().as_record_mut_unwrap() + } + + /// Convert this insert into a list expression. + /// + /// The resulting [`Expr>`] wraps the insert as a sub-statement, + /// which can be used as the right-hand side of an association + /// [`insert`](Association::insert) call or embedded in other expressions. + /// + pub(crate) fn into_list_expr(self) -> Expr> { + Expr::from_untyped(stmt::Expr::Stmt(self.untyped.into())) + } +} + +impl Clone for Insert { + fn clone(&self) -> Self { + Self { + untyped: self.untyped.clone(), + _p: PhantomData, + } + } +} + +impl From> for stmt::Expr { + fn from(value: Insert) -> Self { + Self::stmt(value.untyped) + } +} + +impl fmt::Debug for Insert { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.untyped.fmt(f) + } +} diff --git a/packages/toasty/src/stmt/into_expr.rs b/packages/toasty/src/stmt/into_expr.rs new file mode 100644 index 00000000000..ab39b20650b --- /dev/null +++ b/packages/toasty/src/stmt/into_expr.rs @@ -0,0 +1,481 @@ +use std::{rc::Rc, sync::Arc}; + +use super::assignment::impl_assign_via_expr; +use super::{Expr, List, Value}; +#[cfg(feature = "net")] +use super::{IpCidr, IpInet, MacAddr6, MacAddr8}; +use toasty_core::stmt; + +/// Convert a value into an [`Expr`]. +/// +/// This trait is the primary way Toasty coerces Rust values into query +/// expressions. It is implemented for all scalar types (`i64`, `String`, +/// `bool`, `uuid::Uuid`, …), `Option`, tuples, slices, `Vec`, and arrays. +/// +/// Generated code uses `IntoExpr` bounds on filter and setter methods so that +/// callers can pass either a raw value or an already-constructed [`Expr`]: +/// +/// ``` +/// # use toasty::stmt::{Expr, IntoExpr}; +/// // Both &str and Expr implement IntoExpr: +/// let _e1: Expr = "Alice".into_expr(); +/// let _e2: Expr = Expr::::from_untyped( +/// toasty_core::stmt::Value::from("Bob"), +/// ).into_expr(); +/// ``` +/// +/// # Implementing for custom types +/// +/// If you have a newtype that wraps a supported scalar, implement `IntoExpr` +/// by converting to the inner type first, then calling [`Expr::from_value`]. +pub trait IntoExpr { + /// Consume `self` and produce an [`Expr`]. + /// + /// # Examples + /// + /// ``` + /// use toasty::stmt::{Expr, IntoExpr}; + /// + /// let _expr: Expr = 42_i64.into_expr(); + /// let _expr: Expr = "hello".into_expr(); + /// ``` + fn into_expr(self) -> Expr; + + /// Produce an [`Expr`] from a reference without consuming `self`. + /// + /// For [`Copy`] types this clones the value. For non-`Copy` types like + /// `String` this clones the underlying data. + /// + /// # Examples + /// + /// ``` + /// use toasty::stmt::{Expr, IntoExpr}; + /// + /// let value = 42_i64; + /// let _expr: Expr = value.by_ref(); + /// // `value` is still usable + /// assert_eq!(value, 42); + /// ``` + fn by_ref(&self) -> Expr; +} + +macro_rules! impl_into_expr_for_copy { + ( $( $var:ident($t:ty) ;)* ) => { + $( + impl IntoExpr<$t> for $t { + fn into_expr(self) -> Expr<$t> { + Expr::from_value(Value::from(self)) + } + + fn by_ref(&self) -> Expr<$t> { + Expr::from_value(Value::from(self.clone())) + } + } + + impl_assign_via_expr!($t => $t); + )* + }; +} + +impl_into_expr_for_copy! { + Bool(bool); + I8(i8); + I16(i16); + I32(i32); + I64(i64); + U8(u8); + U16(u16); + U32(u32); + U64(u64); + F32(f32); + F64(f64); + Uuid(uuid::Uuid); +} + +#[cfg(feature = "jiff")] +impl_into_expr_for_copy! { + Timestamp(jiff::Timestamp); + Zoned(jiff::Zoned); + Date(jiff::civil::Date); + Time(jiff::civil::Time); + DateTime(jiff::civil::DateTime); +} + +/// A time zone binds as its text form (see `schema::jiff`). Unlike the other +/// conversions here this one can fail, and `into_expr` has no error channel — +/// carrying the zone to a fallible cast instead would mean a `Value::TimeZone` +/// variant and the core plumbing behind it. +#[cfg(feature = "jiff")] +impl IntoExpr for jiff::tz::TimeZone { + fn into_expr(self) -> Expr { + self.by_ref() + } + + fn by_ref(&self) -> Expr { + let text = crate::schema::jiff::time_zone_to_text(self) + .expect("time zone cannot be stored: no IANA identifier, fixed offset, or POSIX rule"); + Expr::from_value(Value::String(text)) + } +} + +#[cfg(feature = "jiff")] +impl_assign_via_expr!(jiff::tz::TimeZone => jiff::tz::TimeZone); + +#[cfg(feature = "net")] +impl_into_expr_for_copy! { + Cidr(IpCidr); + Inet(IpInet); + MacAddr(MacAddr6); + MacAddr8(MacAddr8); +} + +// Pointer-sized integers convert through their fixed-size equivalents +impl IntoExpr for isize { + fn into_expr(self) -> Expr { + Expr::from_value(Value::from(self as i64)) + } + + fn by_ref(&self) -> Expr { + Expr::from_value(Value::from(*self as i64)) + } +} +impl_assign_via_expr!(isize => isize); + +impl IntoExpr for usize { + fn into_expr(self) -> Expr { + Expr::from_value(Value::from(self as u64)) + } + + fn by_ref(&self) -> Expr { + Expr::from_value(Value::from(*self as u64)) + } +} +impl_assign_via_expr!(usize => usize); + +impl IntoExpr for Expr { + fn into_expr(self) -> Self { + self + } + + fn by_ref(&self) -> Self { + self.clone() + } +} +impl_assign_via_expr!({T} Expr => T); + +impl> IntoExpr for &T { + fn into_expr(self) -> Expr { + self.by_ref() + } + + fn by_ref(&self) -> Expr { + (*self).by_ref() + } +} +impl_assign_via_expr!({T: IntoExpr} &T => T); + +impl> IntoExpr for Option { + fn into_expr(self) -> Expr { + match self { + Some(value) => value.into_expr().cast(), + None => Expr::from_value(Value::Null), + } + } + + fn by_ref(&self) -> Expr { + match self { + Some(value) => value.by_ref().cast(), + None => Expr::from_value(Value::Null), + } + } +} +impl_assign_via_expr!({T: IntoExpr} Option => Option); + +impl> IntoExpr> for T { + fn into_expr(self) -> Expr> { + self.into_expr().cast() + } + + fn by_ref(&self) -> Expr> { + self.by_ref().cast() + } +} +impl_assign_via_expr!({T: IntoExpr} T => Option); + +impl> IntoExpr> for &T { + fn into_expr(self) -> Expr> { + self.by_ref().cast() + } + + fn by_ref(&self) -> Expr> { + (*self).by_ref().cast() + } +} +impl_assign_via_expr!({T: IntoExpr} &T => Option); + +impl IntoExpr for &str { + fn into_expr(self) -> Expr { + Expr::from_value(Value::from(self)) + } + + fn by_ref(&self) -> Expr { + Expr::from_value(Value::from(*self)) + } +} +impl_assign_via_expr!(&str => String); + +impl IntoExpr> for &str { + fn into_expr(self) -> Expr> { + Expr::from_value(Value::from(self)) + } + + fn by_ref(&self) -> Expr> { + Expr::from_value(Value::from(*self)) + } +} +impl_assign_via_expr!(&str => Option); + +impl IntoExpr for String { + fn into_expr(self) -> Expr { + Expr::from_value(self.into()) + } + + fn by_ref(&self) -> Expr { + Expr::from_value(self.into()) + } +} +impl_assign_via_expr!(String => String); + +impl IntoExpr for Vec { + fn into_expr(self) -> Expr { + Expr::from_value(self.into()) + } + + fn by_ref(&self) -> Expr { + Expr::from_value(self.clone().into()) + } +} +impl_assign_via_expr!(Vec => Vec); + +#[cfg(feature = "rust_decimal")] +impl IntoExpr for rust_decimal::Decimal { + fn into_expr(self) -> Expr { + Expr::from_value(self.into()) + } + + fn by_ref(&self) -> Expr { + Expr::from_value((*self).into()) + } +} +#[cfg(feature = "rust_decimal")] +impl_assign_via_expr!(rust_decimal::Decimal => rust_decimal::Decimal); + +#[cfg(feature = "bigdecimal")] +impl IntoExpr for bigdecimal::BigDecimal { + fn into_expr(self) -> Expr { + Expr::from_value(self.into()) + } + + fn by_ref(&self) -> Expr { + Expr::from_value(self.clone().into()) + } +} +#[cfg(feature = "bigdecimal")] +impl_assign_via_expr!(bigdecimal::BigDecimal => bigdecimal::BigDecimal); + +impl IntoExpr> for [U; N] +where + U: IntoExpr, +{ + fn into_expr(self) -> Expr> { + Expr::from_untyped(stmt::Expr::list( + self.into_iter().map(|item| item.into_expr().untyped), + )) + } + + fn by_ref(&self) -> Expr> { + Expr::from_untyped(stmt::Expr::list( + self.iter().map(|item| U::by_ref(item).untyped), + )) + } +} +impl_assign_via_expr!({T, U: IntoExpr, const N: usize} [U; N] => List); + +impl IntoExpr> for &[U; N] +where + U: IntoExpr, +{ + fn into_expr(self) -> Expr> { + Expr::from_untyped(stmt::Expr::list( + self.iter().map(|item| U::by_ref(item).untyped), + )) + } + + fn by_ref(&self) -> Expr> { + Expr::from_untyped(stmt::Expr::list( + self.iter().map(|item| U::by_ref(item).untyped), + )) + } +} +impl_assign_via_expr!({T, U: IntoExpr, const N: usize} &[U; N] => List); + +impl> IntoExpr> for &[E] { + fn into_expr(self) -> Expr> { + Expr::from_untyped(stmt::Expr::list( + self.iter().map(|item| E::by_ref(item).untyped), + )) + } + + fn by_ref(&self) -> Expr> { + Expr::from_untyped(stmt::Expr::list( + self.iter().map(|item| E::by_ref(item).untyped), + )) + } +} +impl_assign_via_expr!({T, E: IntoExpr} &[E] => List); + +impl IntoExpr> for Vec +where + U: IntoExpr, +{ + fn into_expr(self) -> Expr> { + Expr::from_untyped(stmt::Expr::list( + self.into_iter().map(|item| item.into_expr().untyped), + )) + } + + fn by_ref(&self) -> Expr> { + Expr::from_untyped(stmt::Expr::list( + self.iter().map(|item| item.by_ref().untyped), + )) + } +} +impl_assign_via_expr!({T, U: IntoExpr} Vec => List); + +// `Vec` model fields bind through the existing `IntoExpr>` +// impls (slice, array, `Vec`, …) — `Field::ExprTarget = List` for +// `Vec`, so the create/update setter macros emit +// `IntoExpr>` bounds without any type parsing. `Vec` keeps +// `ExprTarget = Self` and goes through its scalar `IntoExpr>` impl +// above. + +macro_rules! forward_impl { + ( $( $ty:ty ,) *) => { + $( + impl IntoExpr<$ty> for T + where + T: IntoExpr, + { + fn into_expr(self) -> Expr<$ty> { + >::into_expr(self).cast() + } + + fn by_ref(&self) -> Expr<$ty> { + >::by_ref(self).cast() + } + } + ) * + }; +} + +forward_impl!(Arc, Box, Rc,); +impl_assign_via_expr!({T: IntoExpr} T => Arc); +impl_assign_via_expr!({T: IntoExpr} T => Box); +impl_assign_via_expr!({T: IntoExpr} T => Rc); + +macro_rules! ref_smart_ptr_impl { + ( $( $ptr:ident ,)* ) => { + $( + impl> IntoExpr<$ptr> for $ptr { + fn into_expr(self) -> Expr<$ptr> { + T::by_ref(&self).cast() + } + + fn by_ref(&self) -> Expr<$ptr> { + T::by_ref(self).cast() + } + } + + impl> IntoExpr for &$ptr { + fn into_expr(self) -> Expr { + T::by_ref(self) + } + + fn by_ref(&self) -> Expr { + T::by_ref(self) + } + } + )* + }; +} + +ref_smart_ptr_impl!(Arc, Box, Rc,); + +macro_rules! impl_into_expr_for_tuple { + (! $( $n:tt $t:ident $e:ident )* ) => { + impl<$( $t, $e ),*> IntoExpr<($( $t, )*)> for ($( $e, )*) + where + $( $e: IntoExpr<$t>, )* + { + fn into_expr(self) -> Expr<($( $t, )*)> { + let record = stmt::ExprRecord::from_vec(vec![ + $( self.$n.into_expr().untyped, )* + ]); + let untyped = stmt::Expr::Record(record); + Expr::from_untyped(untyped) + } + + fn by_ref(&self) -> Expr<($( $t, )*)> { + let record = stmt::ExprRecord::from_vec(vec![ + $( self.$n.by_ref().untyped, )* + ]); + let untyped = stmt::Expr::Record(record); + Expr::from_untyped(untyped) + } + } + + impl_assign_via_expr!({$( $t, $e: IntoExpr<$t> ),*} ($( $e, )*) => ($( $t, )*)); + }; + + ( + ( $( $n_base:tt $t_base:ident $e_base:ident )* ) + $n:tt $t:ident $e:ident + $( $rest:tt )* + ) => { + // Implement for tuples at this level + impl_into_expr_for_tuple!(! $( $n_base $t_base $e_base )* $n $t $e); + + // Recurse + impl_into_expr_for_tuple!( + ( $( $n_base $t_base $e_base )* $n $t $e ) + $( $rest )* + ); + }; + + ( ( $( $n:tt $t:ident $e:ident )* ) ) => {} +} + +impl_into_expr_for_tuple! { + () + 0 T0 E0 + 1 T1 E1 + 2 T2 E2 + 3 T3 E3 + 4 T4 E4 + 5 T5 E5 + 6 T6 E6 + 7 T7 E7 + 8 T8 E8 + 9 T9 E9 +} + +#[test] +fn assert_bounds() { + fn assert_into_expr>() {} + + assert_into_expr::(); + assert_into_expr::<(String, String), (&String, &String)>(); + assert_into_expr::, &[(&String, &String)]>(); + assert_into_expr::, [(&String, &String); 3]>(); + assert_into_expr::, &[(&String, &String); 3]>(); +} diff --git a/packages/toasty/src/stmt/into_insert.rs b/packages/toasty/src/stmt/into_insert.rs new file mode 100644 index 00000000000..e4e9df03d37 --- /dev/null +++ b/packages/toasty/src/stmt/into_insert.rs @@ -0,0 +1,18 @@ +use super::Insert; +use crate::schema::Model; + +/// Convert a value into an [`Insert`] statement. +/// +/// Generated create-builders implement this trait so that they can be passed +/// anywhere an insert is expected (e.g., association [`insert`](Association::insert) +/// calls or batch operations). +/// +/// The associated type [`Model`](IntoInsert::Model) identifies which model the +/// insert targets. +pub trait IntoInsert { + /// The model this insert targets. + type Model: Model; + + /// Consume `self` and produce the [`Insert`] statement. + fn into_insert(self) -> Insert; +} diff --git a/packages/toasty/src/stmt/into_statement.rs b/packages/toasty/src/stmt/into_statement.rs new file mode 100644 index 00000000000..9aa01307aaa --- /dev/null +++ b/packages/toasty/src/stmt/into_statement.rs @@ -0,0 +1,116 @@ +use super::{List, Statement}; + +use std::marker::PhantomData; + +use toasty_core::stmt; + +/// Convert a value into a [`Statement`]. +/// +/// This trait bridges query builders to [`Statement`](Statement). The +/// associated [`Returning`](IntoStatement::Returning) type encodes what the +/// statement produces when executed: +/// +/// | Builder | `Returning` | +/// |---|---| +/// | [`Query`] | [`List`] | +/// | [`Delete`] | `()` | +/// | [`Association`] | [`List`] | +/// | Create builders (generated) | `M` | +/// | Tuples of builders | `(R1, R2, …)` | +/// | `Vec` / `[Q; N]` | [`List`](List) | +/// +/// Tuples, `Vec`, and arrays batch multiple statements into a single +/// round-trip. Each sub-statement runs independently; the combined result is +/// a tuple or list of individual results. +pub trait IntoStatement { + /// The type this statement produces when executed. + type Returning; + + /// Consume `self` and produce the [`Statement`]. + fn into_statement(self) -> Statement; +} + +macro_rules! impl_into_statement_for_tuple { + ( $( $Q:ident ),+ ; $n:tt ; $( $idx:tt ),+ ) => { + impl< $( $Q: IntoStatement ),+ > IntoStatement for ( $( $Q, )+ ) { + type Returning = ( $( $Q::Returning, )+ ); + + fn into_statement(self) -> Statement { + let exprs: Vec = vec![ + $( { + let mut untyped = self.$idx.into_statement().untyped; + ensure_batch_returning(&mut untyped); + stmt::Expr::stmt(untyped) + }, )+ + ]; + + let query = stmt::Query::new_single( + stmt::Values::from(stmt::Expr::record(exprs)), + ); + + Statement { + untyped: query.into(), + _p: PhantomData, + } + } + } + }; +} + +impl_into_statement_for_tuple!(Q1, Q2; 2; 0, 1); +impl_into_statement_for_tuple!(Q1, Q2, Q3; 3; 0, 1, 2); +impl_into_statement_for_tuple!(Q1, Q2, Q3, Q4; 4; 0, 1, 2, 3); +impl_into_statement_for_tuple!(Q1, Q2, Q3, Q4, Q5; 5; 0, 1, 2, 3, 4); +impl_into_statement_for_tuple!(Q1, Q2, Q3, Q4, Q5, Q6; 6; 0, 1, 2, 3, 4, 5); +impl_into_statement_for_tuple!(Q1, Q2, Q3, Q4, Q5, Q6, Q7; 7; 0, 1, 2, 3, 4, 5, 6); +impl_into_statement_for_tuple!(Q1, Q2, Q3, Q4, Q5, Q6, Q7, Q8; 8; 0, 1, 2, 3, 4, 5, 6, 7); + +/// Ensure a sub-statement has a returning clause for batch composition. +/// +/// Mutation statements (delete, update) without a returning clause default to +/// returning a count. In a batch, every sub-statement must produce a value, so +/// we set an empty record returning (`Returning::Expr(Expr::record([]))`) +/// which represents unit. +fn ensure_batch_returning(stmt: &mut stmt::Statement) { + if !stmt.is_query() && stmt.returning().is_none() { + stmt.set_returning_expr(stmt::Expr::record::([])); + } +} + +/// Helper to build a batched statement from an iterator of queries. +fn batch_from_iter( + iter: impl Iterator, +) -> Statement> { + let exprs: Vec = iter + .map(|q| { + let mut untyped = q.into_statement().untyped; + ensure_batch_returning(&mut untyped); + stmt::Expr::stmt(untyped) + }) + .collect(); + + let query = stmt::Query::new_single(stmt::Values::from(stmt::Expr::record(exprs))); + + Statement { + untyped: query.into(), + _p: PhantomData, + } +} + +/// Dynamic batch via `Vec`: all queries have the same type, returns `List`. +impl IntoStatement for Vec { + type Returning = List; + + fn into_statement(self) -> Statement { + batch_from_iter(self.into_iter()) + } +} + +/// Dynamic batch via `[Q; N]`: fixed-size array of typed queries, returns `List`. +impl IntoStatement for [Q; N] { + type Returning = List; + + fn into_statement(self) -> Statement { + batch_from_iter(self.into_iter()) + } +} diff --git a/packages/toasty/src/stmt/json.rs b/packages/toasty/src/stmt/json.rs new file mode 100644 index 00000000000..4a15b8be97b --- /dev/null +++ b/packages/toasty/src/stmt/json.rs @@ -0,0 +1,457 @@ +use super::{Expr, IntoExpr, List, Path, Value as StmtValue}; +use crate::schema::{Field, Load}; +use serde_json::Value as JsonValue; +use toasty_core::schema::app::{FieldPrimitive, FieldTy, SerializeFormat}; +use toasty_core::{schema::db, stmt}; + +use std::fmt; + +fn load_json(value: stmt::Value) -> crate::Result +where + T: for<'de> serde_core::Deserialize<'de>, +{ + let json = ::load(value)?; + serde_json::from_str(&json).map_err(|e| { + toasty_core::Error::from_args(format_args!("failed to deserialize JSON field: {e}")) + }) +} + +fn json_field_ty(storage_ty: Option, missing_type_message: &'static str) -> FieldTy { + FieldTy::Primitive(FieldPrimitive { + ty: stmt::Type::String, + storage_ty: Some(storage_ty.expect(missing_type_message)), + serialize: Some(SerializeFormat::Json), + }) +} + +fn json_expr(value: &(impl serde_core::Serialize + ?Sized)) -> Expr { + let json = serde_json::to_string(value).expect("failed to serialize JSON field"); + Expr::::from_value(StmtValue::from(json)).cast() +} + +/// A field wrapper that serializes `T` as JSON in a database column. +/// +/// Use `Json` as a model field type when the column has no native database +/// representation for `T` — for example, a serde-derived struct, a `HashMap`, +/// or any other type that round-trips through +/// [`serde::Serialize`](serde::Serialize) and +/// [`serde::Deserialize`](serde::Deserialize). The value is JSON-encoded on +/// insert and update, and decoded on read. +/// +/// Every `Json` field must select its database column type with +/// `#[column(type = ...)]`. Use `text` or `varchar(...)` for text-backed JSON, +/// `json` for PostgreSQL or MySQL native JSON, and `jsonb` for PostgreSQL JSONB. +/// A field whose Rust type is already [`serde_json::Value`] does not need the +/// `Json` wrapper and supports the same column types. +/// +/// # Two nullable variants +/// +/// `Json` composes with `Option` in two distinct, both-useful ways: +/// +/// | Field type | `None` is stored as | +/// |----------------------|--------------------------| +/// | `Option>` | SQL `NULL` | +/// | `Json>` | JSON literal `"null"` | +/// +/// # Setter ergonomics +/// +/// The crate provides an `IntoExpr>` impl for any +/// `T: serde::Serialize`, so create / update setters and the `create!` +/// macro accept the bare inner value without an explicit `Json(...)` +/// wrapper. Both forms produce the same expression: +/// +/// ```ignore +/// // for a `payload: Json` (or `Deferred>`) field +/// Repository::create().payload(my_payload.clone()).exec(&mut db).await?; +/// Repository::create().payload(Json(my_payload.clone())).exec(&mut db).await?; +/// ``` +/// +/// The `Json(...)` form is still useful when type inference needs a +/// nudge (e.g., comparison expressions like `.eq(Json("hello"))`). +/// +/// # Composition with `Deferred` +/// +/// `Json` is the only wrapper allowed inside [`Deferred`](crate::Deferred) +/// for a serde-typed field, since a deferred column needs both lazy +/// loading and a serializer the macro can drive through trait dispatch: +/// +/// ```ignore +/// #[derive(Debug, toasty::Model)] +/// struct Repository { +/// #[key] #[auto] +/// id: uuid::Uuid, +/// #[column(type = text)] +/// schema: toasty::Deferred>, +/// } +/// ``` +/// +/// # Examples +/// +/// ```ignore +/// use serde::{Serialize, Deserialize}; +/// +/// #[derive(Debug, Clone, Serialize, Deserialize)] +/// struct Tags(Vec); +/// +/// #[derive(Debug, toasty::Model)] +/// struct Item { +/// #[key] #[auto] +/// id: i64, +/// #[column(type = text)] +/// tags: toasty::Json, +/// } +/// ``` +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash)] +pub struct Json(pub T); + +impl From for Json { + fn from(value: T) -> Self { + Json(value) + } +} + +impl std::ops::Deref for Json { + type Target = T; + + fn deref(&self) -> &T { + &self.0 + } +} + +impl std::ops::DerefMut for Json { + fn deref_mut(&mut self) -> &mut T { + &mut self.0 + } +} + +impl AsRef for Json { + fn as_ref(&self) -> &T { + &self.0 + } +} + +impl AsMut for Json { + fn as_mut(&mut self) -> &mut T { + &mut self.0 + } +} + +impl Load for Json +where + T: for<'de> serde_core::Deserialize<'de>, +{ + type Output = Self; + + fn ty() -> stmt::Type { + stmt::Type::String + } + + fn load(value: stmt::Value) -> crate::Result { + // SQL `NULL` never reaches here: a nullable JSON column is typed + // `Option>`, and `Option: Load` intercepts `Value::Null` + // before delegating to the inner type. `Json>` stores the + // JSON literal `"null"` as a non-null string, so its `None` case + // comes through as `Value::String("null")` and `serde_json` decodes + // it. A bare `Value::Null` at this point is a driver bug. + load_json(value).map(Json) + } + + fn reload(target: &mut Self, value: stmt::Value) -> crate::Result<()> { + *target = Self::load(value)?; + Ok(()) + } +} + +impl Field for Json +where + T: serde_core::Serialize + for<'de> serde_core::Deserialize<'de>, +{ + const REQUIRES_EXPLICIT_COLUMN_TYPE: bool = true; + + type ExprTarget = Self; + type Path = Path; + type ListPath = Path>; + type Update<'a> = (); + type Inner = Self; + + fn new_path(path: Path) -> Self::Path { + path + } + + fn new_list_path(path: Path>) -> Self::ListPath { + path + } + + fn new_update<'a>( + _assignments: &'a mut toasty_core::stmt::Assignments, + _projection: toasty_core::stmt::Projection, + ) -> Self::Update<'a> { + } + + /// Tag the schema entry as JSON-serialized with explicit storage. + /// + /// The macro never inspects the wrapper at the AST level; the + /// `serialize: Some(Json)` metadata flows through trait dispatch + /// alongside the storage type, so external schema consumers can still + /// see that the column is JSON-typed even though the encoding itself is + /// invisible to the macro. + fn field_ty(storage_ty: Option) -> FieldTy { + json_field_ty( + storage_ty, + "`toasty::Json` fields require `#[column(type = ...)]`; use \ + `#[column(type = text)]` for text-backed JSON storage", + ) + } + + fn key_constraint(&self, _target: Path) -> Expr { + // JSON columns are not valid foreign-key targets — the comparison + // would happen at the serialized-string level, which is rarely what + // the user wants. The trait method exists for every `Field`; we + // satisfy it with a panic rather than admitting nonsense semantics. + unreachable!("Json fields cannot be used as foreign-key targets") + } +} + +impl IntoExpr> for Json +where + T: serde_core::Serialize, +{ + fn into_expr(self) -> Expr> { + json_expr(&self.0) + } + + fn by_ref(&self) -> Expr> { + json_expr(&self.0) + } +} + +// `IntoExpr>` for `&Json` comes from the blanket +// `impl> IntoExpr for &T` in `stmt::into_expr`. + +/// Accept a bare `T` wherever the API expects `IntoExpr>`, so +/// callers don't have to spell `Json(value)` at setter sites: +/// +/// ```ignore +/// // both forms work for a `payload: Json` field +/// Repository::create().payload(Json(payload.clone())).exec(&mut db).await?; +/// Repository::create().payload(payload.clone()).exec(&mut db).await?; +/// ``` +/// +/// The blanket only fires when `T: serde::Serialize`; it doesn't overlap +/// the explicit `IntoExpr> for Json` impl because `Json` +/// itself is not `Serialize` (no derive on the wrapper). +impl IntoExpr> for T +where + T: serde_core::Serialize, +{ + fn into_expr(self) -> Expr> { + json_expr(&self) + } + + fn by_ref(&self) -> Expr> { + json_expr(self) + } +} + +/// Accept a borrowed `T` wherever the API expects `IntoExpr>`, so a +/// caller that still needs the value afterwards doesn't have to clone it. +/// The value is serialized out of the borrow. +/// +/// This doesn't overlap the `IntoExpr> for T` blanket: unifying the +/// two would require `T == &T`. +impl IntoExpr> for &T +where + T: serde_core::Serialize, +{ + fn into_expr(self) -> Expr> { + json_expr(self) + } + + fn by_ref(&self) -> Expr> { + json_expr(*self) + } +} + +/// Same, for the explicit `Json(&value)` form. +impl IntoExpr> for Json<&T> +where + T: serde_core::Serialize, +{ + fn into_expr(self) -> Expr> { + json_expr(self.0) + } + + fn by_ref(&self) -> Expr> { + json_expr(self.0) + } +} + +/// `Json(&value)` on an `Option>` field, mirroring the +/// `IntoExpr> for T` blanket that lets the owned `Json(value)` +/// skip the `Some(...)`. +impl IntoExpr>> for Json<&T> +where + T: serde_core::Serialize, +{ + fn into_expr(self) -> Expr>> { + json_expr::>(self.0).cast() + } + + fn by_ref(&self) -> Expr>> { + json_expr::>(self.0).cast() + } +} + +impl super::assignment::Assign> for Json +where + T: serde_core::Serialize, +{ + fn into_assignment(self) -> super::assignment::Assignment> { + super::set(>>::into_expr(self)) + } +} + +/// Mirrors the `IntoExpr> for T` blanket on the assignment side so +/// update builders accept a bare value too: +/// +/// ```ignore +/// repo.update().payload(payload.clone()).exec(&mut db).await?; +/// ``` +impl super::assignment::Assign> for T +where + T: serde_core::Serialize, +{ + fn into_assignment(self) -> super::assignment::Assignment> { + super::set(>>::into_expr(self)) + } +} + +/// Mirrors the borrowed `IntoExpr` impls on the assignment side, so the +/// update builders take a reference too. +impl super::assignment::Assign> for &T +where + T: serde_core::Serialize, +{ + fn into_assignment(self) -> super::assignment::Assignment> { + super::set(>>::into_expr(self)) + } +} + +/// Same, for the explicit `Json(&value)` form. +impl super::assignment::Assign> for Json<&T> +where + T: serde_core::Serialize, +{ + fn into_assignment(self) -> super::assignment::Assignment> { + super::set(>>::into_expr(self)) + } +} + +/// `Json(&value)` assigned to an `Option>` field. +impl super::assignment::Assign>> for Json<&T> +where + T: serde_core::Serialize, +{ + fn into_assignment(self) -> super::assignment::Assignment>> { + super::set(>>>::into_expr(self)) + } +} + +impl Load for JsonValue { + type Output = Self; + + fn ty() -> stmt::Type { + stmt::Type::String + } + + fn load(value: stmt::Value) -> crate::Result { + load_json(value) + } + + fn reload(target: &mut Self, value: stmt::Value) -> crate::Result<()> { + *target = Self::load(value)?; + Ok(()) + } +} + +impl Field for JsonValue { + const REQUIRES_EXPLICIT_COLUMN_TYPE: bool = true; + + type ExprTarget = Self; + type Path = Path; + type ListPath = Path>; + type Update<'a> = (); + type Inner = Self; + + fn new_path(path: Path) -> Self::Path { + path + } + + fn new_list_path(path: Path>) -> Self::ListPath { + path + } + + fn new_update<'a>( + _assignments: &'a mut toasty_core::stmt::Assignments, + _projection: toasty_core::stmt::Projection, + ) -> Self::Update<'a> { + } + + fn field_ty(storage_ty: Option) -> FieldTy { + json_field_ty( + storage_ty, + "`serde_json::Value` fields require `#[column(type = ...)]`; use \ + `#[column(type = text)]` for text-backed JSON storage", + ) + } + + fn key_constraint(&self, _target: Path) -> Expr { + unreachable!("serde_json::Value fields cannot be used as foreign-key targets") + } +} + +impl IntoExpr for JsonValue { + fn into_expr(self) -> Expr { + json_expr(&self) + } + + fn by_ref(&self) -> Expr { + json_expr(self) + } +} + +impl super::assignment::Assign for JsonValue { + fn into_assignment(self) -> super::assignment::Assignment { + super::set(>::into_expr(self)) + } +} + +impl fmt::Display for Json { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +impl serde_core::Serialize for Json +where + T: serde_core::Serialize, +{ + fn serialize(&self, serializer: S) -> Result + where + S: serde_core::Serializer, + { + self.0.serialize(serializer) + } +} + +impl<'de, T> serde_core::Deserialize<'de> for Json +where + T: serde_core::Deserialize<'de>, +{ + fn deserialize(deserializer: D) -> Result + where + D: serde_core::Deserializer<'de>, + { + T::deserialize(deserializer).map(Json) + } +} diff --git a/packages/toasty/src/stmt/list.rs b/packages/toasty/src/stmt/list.rs new file mode 100644 index 00000000000..406def6cfa9 --- /dev/null +++ b/packages/toasty/src/stmt/list.rs @@ -0,0 +1,9 @@ +use std::marker::PhantomData; + +/// A sized marker type representing "list of `M`". +/// +/// Used as a type parameter to [`Statement`], [`Load`](crate::schema::Load), and other +/// types to encode that the result is a collection of `M` values. Unlike `[M]` +/// (which is unsized), `List` is always `Sized`, so it composes cleanly in +/// tuples: `(List, List)` is valid. +pub struct List(PhantomData); diff --git a/packages/toasty/src/stmt/numeric.rs b/packages/toasty/src/stmt/numeric.rs new file mode 100644 index 00000000000..654c36328ea --- /dev/null +++ b/packages/toasty/src/stmt/numeric.rs @@ -0,0 +1,33 @@ +use toasty_core::stmt; + +/// A numeric type that has a representable "one" value. +/// +/// Used as a bound on [`crate::stmt::increment`] and [`crate::stmt::decrement`] +/// so the literal `1` is encoded as a [`stmt::Value`] variant matching the +/// field type — both backend binding and driver-side arithmetic require it. +/// +/// Implemented for every primitive numeric type ([`i8`]–[`i64`], [`u8`]–[`u64`], +/// [`f32`], [`f64`]). Implement for custom newtypes around supported numeric +/// values (e.g. wrapping `rust_decimal::Decimal`) to use them with +/// [`increment`] / [`decrement`]. +/// +/// [`increment`]: crate::stmt::increment +/// [`decrement`]: crate::stmt::decrement +pub trait Numeric { + /// Return the value `1` for this type as a [`stmt::Value`]. + fn one() -> stmt::Value; +} + +macro_rules! impl_numeric { + ($($t:ty),* $(,)?) => { + $( + impl Numeric for $t { + fn one() -> stmt::Value { + stmt::Value::from(1 as $t) + } + } + )* + }; +} + +impl_numeric!(i8, i16, i32, i64, u8, u16, u32, u64, f32, f64); diff --git a/packages/toasty/src/stmt/page.rs b/packages/toasty/src/stmt/page.rs new file mode 100644 index 00000000000..ecf70c4fbaf --- /dev/null +++ b/packages/toasty/src/stmt/page.rs @@ -0,0 +1,133 @@ +use std::ops::Deref; + +use super::{List, Paginate, Query}; +use crate::schema::Load; +use crate::{Executor, Result}; +use toasty_core::stmt; + +/// A page of results from a cursor-based paginated query. +/// +/// Obtained by calling [`Paginate::exec`]. The +/// page contains up to `per_page` items and optional cursors for fetching the +/// next or previous page. `per_page` is an upper bound, not a guarantee: +/// Toasty applies the page size at the database query and may filter the +/// returned rows further, so a page can have fewer items than `per_page` even +/// when more results exist. Use [`has_next`](Page::has_next) (or check +/// `next_cursor`) to detect the end of the result set rather than relying on +/// page size. +/// +/// `Page` dereferences to `[M]`, so it can be used anywhere a slice is +/// expected. +/// +/// # Navigation +/// +/// Call [`next`](Page::next) or [`prev`](Page::prev) to fetch adjacent pages. +/// Use [`has_next`](Page::has_next) and [`has_prev`](Page::has_prev) to check +/// availability without fetching. +#[derive(Debug)] +pub struct Page { + /// Items in this page. + pub items: Vec, + + /// Base query (without cursors/offsets). + query: Query>, + + /// Opaque cursor for fetching the next page. + pub next_cursor: Option, + + /// Opaque cursor for fetching the previous page. + pub prev_cursor: Option, +} + +impl Page { + pub(crate) fn new( + items: Vec, + query: Query>, + next_cursor: Option, + prev_cursor: Option, + ) -> Self { + Self { + items, + query, + next_cursor, + prev_cursor, + } + } + + /// Returns true if there is a next page available + pub fn has_next(&self) -> bool { + self.next_cursor.is_some() + } + + /// Returns true if there is a previous page available + pub fn has_prev(&self) -> bool { + self.prev_cursor.is_some() + } +} + +impl Page { + /// Fetches the next page of results. + /// + /// Returns `None` if there are no more pages available. Uses the cursor from + /// the last item in the current page to fetch the next set of results. + /// + /// # Examples + /// + /// ```no_run + /// # use toasty::Db; + /// # async fn example(db: &mut Db, page: toasty::stmt::Page) -> toasty::Result<()> { + /// if let Some(next_page) = page.next(db).await? { + /// println!("Found {} items in next page", next_page.items.len()); + /// } + /// # Ok(()) + /// # } + /// ``` + pub async fn next(&self, executor: &mut dyn Executor) -> Result>> { + match &self.next_cursor { + Some(cursor) => Ok(Some( + Paginate::from(self.query.clone()) + .after(cursor.clone()) + .exec(executor) + .await?, + )), + None => Ok(None), + } + } + + /// Fetches the previous page of results. + /// + /// Returns `None` if there are no previous pages available. Uses the cursor from + /// the first item in the current page to fetch the previous set of results. + /// + /// # Examples + /// + /// ```no_run + /// # use toasty::Db; + /// # async fn example(db: &mut Db, page: toasty::stmt::Page) -> toasty::Result<()> { + /// if let Some(prev_page) = page.prev(db).await? { + /// println!("Found {} items in previous page", prev_page.items.len()); + /// } + /// # Ok(()) + /// # } + /// ``` + pub async fn prev(&self, executor: &mut dyn Executor) -> Result>> { + match &self.prev_cursor { + Some(cursor) => Ok(Some( + Paginate::from(self.query.clone()) + .before(cursor.clone()) + .exec(executor) + .await?, + )), + None => Ok(None), + } + } +} + +// Allow using pages like a regular slice for ergonomics. +impl Deref for Page { + type Target = [M]; + + fn deref(&self) -> &Self::Target { + &self.items + } +} diff --git a/packages/toasty/src/stmt/paginate.rs b/packages/toasty/src/stmt/paginate.rs new file mode 100644 index 00000000000..dbc2a239fd4 --- /dev/null +++ b/packages/toasty/src/stmt/paginate.rs @@ -0,0 +1,276 @@ +use super::{List, Query}; + +use crate::{Executor, Result, schema::Load}; + +use toasty_core::stmt; + +/// Cursor-based pagination over a [`Query`]. +/// +/// `Paginate` wraps a query with a fixed page size and provides +/// [`after`](Paginate::after) / [`before`](Paginate::before) methods for +/// forward and backward navigation using opaque cursors. +/// +/// # Construction +/// +/// Create a `Paginate` from a query via [`Paginate::new`] or by calling +/// `.into()` on a query that already has `limit` and `order_by` set. +/// +/// ``` +/// # tokio::runtime::Runtime::new().unwrap().block_on(async { +/// # #[derive(Debug, toasty::Model)] +/// # struct User { +/// # #[key] +/// # id: i64, +/// # name: String, +/// # } +/// # let driver = toasty_driver_sqlite::Sqlite::in_memory(); +/// # let mut db = toasty::Db::builder().models(toasty::models!(User)).build(driver).await.unwrap(); +/// # db.push_schema().await.unwrap(); +/// use toasty::stmt::{List, Paginate, Query}; +/// +/// let q = Query::>::all() +/// .order_by(User::fields().name().asc()); +/// let page = Paginate::new(q, 20) +/// .exec(&mut db) +/// .await +/// .unwrap(); +/// # }); +/// ``` +/// +/// # Requirements +/// +/// The underlying query **must** have an `order_by` clause. [`Paginate::new`] +/// sets the limit for you; [`From>`] requires both `limit` and +/// `order_by` to be present already. +#[derive(Debug)] +pub struct Paginate { + query: Query>, + reverse: bool, +} + +impl Paginate { + /// Create a paginator from `query` with the given page size. + /// + /// The query must **not** already have a `limit` clause (this method sets + /// it) and **must** have an `order_by` clause. + /// + /// # Panics + /// + /// Panics if `query` already has a `limit` or is missing `order_by`. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// use toasty::stmt::{List, Paginate, Query}; + /// + /// let q = Query::>::all() + /// .order_by(User::fields().name().asc()); + /// let _paginator = Paginate::new(q, 20); + /// ``` + pub fn new(mut query: Query>, per_page: usize) -> Self { + assert!( + query.untyped.limit.is_none(), + "pagination requires no limit clause" + ); + + let per_page = i64::try_from(per_page).expect("per_page exceeds i64::MAX"); + query.untyped.limit = Some(stmt::Limit::Cursor(stmt::LimitCursor { + page_size: stmt::Value::from(per_page).into(), + after: None, + })); + + Self { + query, + reverse: false, + } + } + + /// Set the cursor for forward pagination. + /// + /// Records returned will come **after** `key` in the current sort order. + /// Obtain `key` from [`Page::next_cursor`](super::Page::next_cursor) of a + /// previous page. Toasty-generated cursors include any internal + /// primary-key fields needed to distinguish rows with equal ordering + /// values. + /// + /// A caller-provided cursor may contain only a prefix of the ordering + /// fields. Such a cursor cannot select a position within rows that tie on + /// that prefix. Pages fetched from it produce complete cursors for + /// subsequent navigation. + /// + /// # Panics + /// + /// Panics if the query has no `limit` clause. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// use toasty::stmt::{List, Paginate, Query}; + /// + /// let q = Query::>::all() + /// .order_by(User::fields().id().asc()); + /// let paginator = Paginate::new(q, 10) + /// .after(toasty_core::stmt::Value::from(42_i64)); + /// ``` + pub fn after(mut self, key: impl Into) -> Self { + let Some(stmt::Limit::Cursor(cursor)) = self.query.untyped.limit.as_mut() else { + panic!("pagination requires a cursor limit clause"); + }; + cursor.after = Some(key.into()); + self.reverse = false; + self + } + + /// Set the cursor for backward pagination. + /// + /// Records returned will come **before** `key` in the current sort order. + /// The result set is still returned in the original sort order (not + /// reversed). Obtain `key` from + /// [`Page::prev_cursor`](super::Page::prev_cursor) of a previous page. + /// Toasty-generated cursors include any internal primary-key fields needed + /// to distinguish rows with equal ordering values. + /// + /// A caller-provided cursor may contain only a prefix of the ordering + /// fields. Such a cursor cannot select a position within rows that tie on + /// that prefix. Pages fetched from it produce complete cursors for + /// subsequent navigation. + /// + /// # Panics + /// + /// Panics if the query has no `limit` clause. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// use toasty::stmt::{List, Paginate, Query}; + /// + /// let q = Query::>::all() + /// .order_by(User::fields().id().asc()); + /// let paginator = Paginate::new(q, 10) + /// .before(toasty_core::stmt::Value::from(100_i64)); + /// ``` + pub fn before(mut self, key: impl Into) -> Self { + let Some(stmt::Limit::Cursor(cursor)) = self.query.untyped.limit.as_mut() else { + panic!("pagination requires a cursor limit clause"); + }; + cursor.after = Some(key.into()); + self.reverse = true; + self + } +} + +impl Paginate { + /// Execute the paginated query and return a [`Page`](super::Page). + /// + /// The returned page contains up to `per_page` items along with optional + /// cursors for the next and previous pages. `per_page` is an upper bound, + /// not a guarantee. The page size is applied to the database query, but + /// Toasty may filter the returned rows further, so a page can have fewer + /// than `per_page` items even when more results exist. Continue paging + /// until `next_cursor` is `None` to consume the full result set. + /// + /// # Examples + /// + /// ``` + /// # tokio::runtime::Runtime::new().unwrap().block_on(async { + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// # let driver = toasty_driver_sqlite::Sqlite::in_memory(); + /// # let mut db = toasty::Db::builder().models(toasty::models!(User)).build(driver).await.unwrap(); + /// # db.push_schema().await.unwrap(); + /// use toasty::stmt::{List, Paginate, Query}; + /// + /// let q = Query::>::all() + /// .order_by(User::fields().name().asc()); + /// let page = Paginate::new(q, 20) + /// .exec(&mut db) + /// .await + /// .unwrap(); + /// # }); + /// ``` + pub async fn exec(mut self, executor: &mut dyn Executor) -> Result> { + let original_query = self.query.untyped.clone(); + + // Reverse ORDER BY for backward pagination + if self.reverse { + let Some(order_by) = self.query.untyped.order_by.as_mut() else { + panic!("pagination requires order by clause"); + }; + order_by.reverse(); + } + + // Execute with pagination - engine handles cursor extraction + let response = executor + .exec_untyped(stmt::Statement::Query(self.query.untyped.clone())) + .await?; + + // Collect values from response + let stmt::Value::List(mut items) = response.values.collect_as_value().await? else { + return Err(crate::Error::invalid_result( + "paginated query expected a list of rows", + )); + }; + + // Reverse result set if paginating backward + if self.reverse { + items.reverse(); + } + + // Load the raw values into model instances + let loaded_items: Vec = items.into_iter().map(M::load).collect::>()?; + + // For backward pagination, swap cursors (next becomes prev) + let (next_cursor, prev_cursor) = if self.reverse { + (response.prev_cursor, response.next_cursor) + } else { + (response.next_cursor, response.prev_cursor) + }; + let next_cursor = next_cursor.map(|cursor| *cursor); + let prev_cursor = prev_cursor.map(|cursor| *cursor); + + // Store the original query (not the reversed one) in the Page so that + // subsequent .next() and .prev() calls use the correct ORDER BY direction + Ok(super::Page::new( + loaded_items, + Query::from_untyped(original_query), + next_cursor, + prev_cursor, + )) + } +} + +impl From>> for Paginate { + fn from(value: Query>) -> Self { + assert!( + value.untyped.limit.is_some(), + "pagination requires a limit clause" + ); + + Paginate { + query: value, + reverse: false, + } + } +} diff --git a/packages/toasty/src/stmt/path.rs b/packages/toasty/src/stmt/path.rs new file mode 100644 index 00000000000..334ecda0dad --- /dev/null +++ b/packages/toasty/src/stmt/path.rs @@ -0,0 +1,771 @@ +use super::{Expr, IntoExpr, IntoStatement, List}; +use crate::schema::Field; +use std::{fmt, marker::PhantomData}; +use toasty_core::{ + schema::app::{ModelId, VariantId}, + stmt::{self, Direction, OrderByExpr}, +}; + +/// A typed path from a root model `T` to a field of type `U`. +/// +/// `Path` represents a traversal through a model's fields and relations. The +/// type parameter `T` is the root model the path starts from, and `U` is the +/// type of the value at the end of the path. +/// +/// Paths are the primary way to reference model fields in queries. Generated +/// code provides accessor methods (e.g., `User::fields().name()`) that return +/// paths, which you then use with comparison methods to build filter +/// expressions: +/// +/// ``` +/// # #[derive(Debug, toasty::Model)] +/// # struct User { +/// # #[key] +/// # id: i64, +/// # name: String, +/// # } +/// // Path — the "name" field on User +/// let path = User::fields().name(); +/// +/// // Expr — a filter expression +/// let filter = path.eq("Alice"); +/// ``` +/// +/// Paths can also be used to construct order-by clauses via +/// [`asc`](Path::asc) and [`desc`](Path::desc), and can be chained to +/// navigate through relations with [`chain`](Path::chain). +pub struct Path { + pub(super) untyped: stmt::Path, + _p: PhantomData<(T, U)>, +} + +impl Path { + /// Create a path rooted at the model (or embedded type) identified by + /// `model`. + /// + /// This is the low-level constructor behind the `path_root` and + /// `path_model_list` helpers on the [`Model`](crate::schema::Model) and + /// [`Embed`](crate::schema::Embed) traits, which supply the [`ModelId`] + /// from their own `id()`. + pub(crate) fn from_model_id(model: ModelId) -> Self { + Self { + untyped: stmt::Path::model(model), + _p: PhantomData, + } + } + + /// Create a path to the field at `index` on the model (or embedded type) + /// identified by `model`. + /// + /// Low-level constructor behind the `path_field` helpers on the + /// [`Model`](crate::schema::Model) and [`Embed`](crate::schema::Embed) + /// traits. + pub(crate) fn field_at(model: ModelId, index: usize) -> Self { + Self { + untyped: stmt::Path::from_index(model, index), + _p: PhantomData, + } + } + + /// Converts this path into a variant-rooted path for use in `.matches()` + /// closures on embedded enum fields. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// # use toasty::schema::Model; + /// # use toasty_core::schema::app::{ModelId, VariantId}; + /// let path = User::path_field::(1); + /// let variant_id = VariantId { model: ModelId(0), index: 0 }; + /// let _variant_path = path.into_variant(variant_id); + /// ``` + pub fn into_variant(self, variant_id: VariantId) -> Self { + Self { + untyped: stmt::Path::from_variant(self.untyped, variant_id), + _p: PhantomData, + } + } + + /// Append `other` to this path, producing a new path from `T` to `V`. + /// + /// The origin of `other` is left unconstrained because list field structs + /// store `Path>` while chaining segments rooted at `M` + /// (not `List`). The untyped path concatenation is always correct; the + /// generic parameters serve only as compile-time markers. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// use toasty::stmt::Path; + /// use toasty::schema::Model; + /// + /// let user_path = User::path_root(); + /// let name_path = User::path_field::(1); + /// let _chained: Path = user_path.chain(name_path); + /// ``` + pub fn chain(mut self, other: impl Into>) -> Path { + let other = other.into(); + self.untyped.chain(&other.untyped); + + Path { + untyped: self.untyped, + _p: PhantomData, + } + } + + /// Build a filter `Expr` from this path's expression. + /// + /// The filter is the bare predicate. For a variant-rooted path, the + /// engine's statement normalization adds the `is_variant` checks the + /// predicate requires. + fn build_filter(self, build_body: F) -> Expr + where + F: FnOnce(stmt::Expr) -> stmt::Expr, + { + Expr::from_untyped(build_body(self.untyped.into_stmt())) + } + + /// Test whether this field equals `rhs`. + /// + /// For a variant-rooted path (e.g. `contact().email().address()`), the + /// resulting filter implicitly requires the variant to match — the + /// engine expands it to `is_email(contact) AND address == rhs`. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// let filter = User::fields().name().eq("Alice"); + /// ``` + pub fn eq(self, rhs: impl IntoExpr) -> Expr { + let rhs = rhs.into_expr().untyped; + self.build_filter(move |path| stmt::Expr::eq(path, rhs)) + } + + /// Test whether this field does not equal `rhs`. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// let filter = User::fields().name().ne("Alice"); + /// ``` + pub fn ne(self, rhs: impl IntoExpr) -> Expr { + let rhs = rhs.into_expr().untyped; + self.build_filter(move |path| stmt::Expr::ne(path, rhs)) + } + + /// Test whether this field is greater than `rhs`. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// let filter = User::fields().id().gt(10); + /// ``` + pub fn gt(self, rhs: impl IntoExpr) -> Expr { + let rhs = rhs.into_expr().untyped; + self.build_filter(move |path| stmt::Expr::gt(path, rhs)) + } + + /// Test whether this field is greater than or equal to `rhs`. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// let filter = User::fields().id().ge(1); + /// ``` + pub fn ge(self, rhs: impl IntoExpr) -> Expr { + let rhs = rhs.into_expr().untyped; + self.build_filter(move |path| stmt::Expr::ge(path, rhs)) + } + + /// Test whether this field is less than `rhs`. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// let filter = User::fields().id().lt(100); + /// ``` + pub fn lt(self, rhs: impl IntoExpr) -> Expr { + let rhs = rhs.into_expr().untyped; + self.build_filter(move |path| stmt::Expr::lt(path, rhs)) + } + + /// Test whether this field is less than or equal to `rhs`. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// let filter = User::fields().id().le(100); + /// ``` + pub fn le(self, rhs: impl IntoExpr) -> Expr { + let rhs = rhs.into_expr().untyped; + self.build_filter(move |path| stmt::Expr::le(path, rhs)) + } + + /// Test whether this field's value is in the inclusive range `[low, high]`. + /// + /// Generates a `BETWEEN low AND high` condition in SQL and a native + /// `BETWEEN` condition expression in DynamoDB. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// let filter = User::fields().id().between(18_i64, 65_i64); + /// ``` + pub fn between(self, low: impl IntoExpr, high: impl IntoExpr) -> Expr { + let low = low.into_expr().untyped; + let high = high.into_expr().untyped; + self.build_filter(move |path| stmt::Expr::between(path, low, high)) + } + + /// Test whether this field's value is in `rhs`. + /// + /// `rhs` can be any collection that implements `IntoExpr>`, such + /// as a `Vec`, array, or slice. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// let filter = User::fields().id().in_list([1_i64, 2, 3]); + /// ``` + pub fn in_list(self, rhs: impl IntoExpr>) -> Expr { + let rhs = rhs.into_expr().untyped; + self.build_filter(move |path| stmt::Expr::in_list(path, rhs)) + } + + /// Test whether this field's value appears in the result set of a + /// subquery. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// use toasty::stmt::{List, Query}; + /// use toasty::schema::Model; + /// + /// // A path targeting User values + /// let path = User::path_root(); + /// // A subquery returning List + /// let subquery = Query::>::all().filter(User::fields().name().eq("Alice")); + /// let filter = path.in_query(subquery); + /// ``` + pub fn in_query(self, rhs: Q) -> Expr + where + Q: IntoStatement>, + { + let query = rhs.into_statement().into_untyped_query(); + self.build_filter(move |path| stmt::Expr::in_subquery(path, query)) + } + + /// Produce an ascending [`OrderByExpr`] for this path. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// let q = User::all() + /// .order_by(User::fields().name().asc()); + /// ``` + pub fn asc(self) -> OrderByExpr { + OrderByExpr { + expr: self.untyped.into_stmt(), + order: Some(Direction::Asc), + } + } + + /// Produce a descending [`OrderByExpr`] for this path. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// let q = User::all() + /// .order_by(User::fields().name().desc()); + /// ``` + pub fn desc(self) -> OrderByExpr { + OrderByExpr { + expr: self.untyped.into_stmt(), + order: Some(Direction::Desc), + } + } +} + +impl Path> { + /// Build an `IN subquery` expression that tests whether **any** associated + /// record satisfies `filter`. + /// + /// The path must point to a `HasMany` (or similar collection) field on the + /// parent model. The returned expression can be used as a filter on the + /// parent query. + /// + /// This also works on model-terminal `#[has_many(via = ...)]` fields. In a + /// join-model many-to-many relation, call `any` on the derived field to + /// filter by the opposite endpoint, or on the direct join-model field to + /// filter by data stored on the connection. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// # #[derive(Debug, toasty::Model)] + /// # struct Todo { + /// # #[key] + /// # id: i64, + /// # title: String, + /// # } + /// use toasty::stmt::List; + /// use toasty::schema::Model; + /// + /// // Find users that have at least one todo with "urgent" in the title + /// let todos_path = User::path_field::>(2); + /// let filter = todos_path.any(Todo::fields().title().eq("urgent")); + /// ``` + pub fn any(self, filter: Expr) -> Expr + where + U: crate::schema::Model, + { + // Build a query on the child model filtered by `filter` + let child_query = super::Query::>::all().filter(filter); + self.build_filter(move |path| stmt::Expr::in_subquery(path, child_query.untyped)) + } + + /// Build a `NOT IN subquery` expression that tests whether **all** associated + /// records satisfy `filter`. + /// + /// The path must point to a `HasMany` (or similar collection) field on the + /// parent model. Returns `true` when every associated record matches + /// `filter`, including the vacuous case where the parent has no associated + /// records (matching Rust's `[].iter().all()` semantics). + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// # #[derive(Debug, toasty::Model)] + /// # struct Todo { + /// # #[key] + /// # id: i64, + /// # complete: bool, + /// # } + /// use toasty::stmt::List; + /// use toasty::schema::Model; + /// + /// // Find users whose todos are all complete + /// let todos_path = User::path_field::>(2); + /// let filter = todos_path.all(Todo::fields().complete().eq(true)); + /// ``` + pub fn all(self, filter: Expr) -> Expr + where + U: crate::schema::Model, + { + // parent NOT IN (SELECT child_fk FROM child WHERE NOT filter) + let child_query = super::Query::>::all().filter(filter.not()); + self.build_filter(move |path| stmt::Expr::not_in_subquery(path, child_query.untyped)) + } +} + +/// Container-style predicates on a `Vec` model field. The path target +/// is the [`List`] marker (matching `Field::Path` for `Vec`), +/// and the element type `U` is constrained to a path-target scalar so the +/// same `IntoExpr` infrastructure that powers `eq`/`in_list` covers the +/// right-hand side. +impl Path> +where + U: crate::schema::Scalar, +{ + /// Test whether the array contains `value`. + /// + /// Mirrors [`Vec::contains`]. Lowers to `value = ANY(col)` on PostgreSQL. + /// + /// # Examples + /// + /// ```ignore + /// let filter = User::fields().tags().contains("admin"); + /// ``` + pub fn contains(self, value: impl IntoExpr) -> Expr { + let value = value.into_expr().untyped; + self.build_filter(move |path| stmt::Expr::any_op(value, stmt::BinaryOp::Eq, path)) + } + + /// Test whether the array contains every element of `values`. + /// + /// Mirrors [`HashSet::is_superset`](std::collections::HashSet::is_superset). + /// Lowers to `col @> values` (PostgreSQL `@>` operator). + pub fn is_superset(self, values: impl IntoExpr>) -> Expr { + let values = values.into_expr().untyped; + self.build_filter(move |path| stmt::Expr::array_is_superset(path, values)) + } + + /// Test whether the array shares at least one element with `values`. + /// + /// Negation of [`HashSet::is_disjoint`](std::collections::HashSet::is_disjoint). + /// Lowers to `col && values` (PostgreSQL `&&` operator). + pub fn intersects(self, values: impl IntoExpr>) -> Expr { + let values = values.into_expr().untyped; + self.build_filter(move |path| stmt::Expr::array_intersects(path, values)) + } + + /// Returns the array length. + /// + /// Mirrors [`Vec::len`]. Lowers to `cardinality(col)` on PostgreSQL. + pub fn len(self) -> Expr { + Expr::from_untyped(stmt::Expr::array_length(self.untyped.into_stmt())) + } + + /// Returns `true` if the array is empty. + /// + /// Equivalent to `.len().eq(0)`. Mirrors [`Vec::is_empty`]. + pub fn is_empty(self) -> Expr { + self.build_filter(|path| { + stmt::Expr::eq( + stmt::Expr::array_length(path), + stmt::Expr::Value(stmt::Value::I64(0)), + ) + }) + } +} + +impl Path> { + /// Test whether this optional field is `NULL`. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # bio: Option, + /// # } + /// let filter = User::fields().bio().is_none(); + /// ``` + pub fn is_none(self) -> Expr { + self.build_filter(stmt::Expr::is_null) + } + + /// Test whether this optional field is not `NULL`. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # bio: Option, + /// # } + /// let filter = User::fields().bio().is_some(); + /// ``` + pub fn is_some(self) -> Expr { + self.build_filter(stmt::Expr::is_not_null) + } +} + +impl Path +where + U: Field, +{ + /// Test whether this string field starts with `prefix`. + /// + /// Available on any string-valued field, including `String`, + /// `Option`, and other wrappers whose `Field::Inner` is `String`. + /// For DynamoDB, this maps to `begins_with` in a `KeyConditionExpression` + /// (sort key) or `FilterExpression` (non-key attribute). + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # nickname: Option, + /// # } + /// let filter = User::fields().name().starts_with("Al".to_string()); + /// let filter = User::fields().nickname().starts_with("Al".to_string()); + /// ``` + pub fn starts_with(self, prefix: impl IntoExpr) -> Expr { + let prefix = prefix.into_expr().untyped; + self.build_filter(move |path| stmt::Expr::starts_with(path, prefix)) + } + + /// Test whether this string field matches a SQL `LIKE` pattern. + /// + /// Available on any string-valued field, including `String`, + /// `Option`, and other wrappers whose `Field::Inner` is `String`. + /// The caller is responsible for including any `%` or `_` wildcard + /// characters in `pattern`. Not supported by the DynamoDB driver — use + /// [`starts_with`](Self::starts_with) instead. + /// + /// `.like()` is a pass-through to the database's own `LIKE`, whose case + /// sensitivity differs by backend: case-sensitive on PostgreSQL, + /// case-insensitive for ASCII on SQLite, and collation-dependent on MySQL. + /// For a case-insensitive match on PostgreSQL, use [`ilike`](Self::ilike). + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # nickname: Option, + /// # } + /// let filter = User::fields().name().like("Al%".to_string()); + /// let filter = User::fields().nickname().like("Al%".to_string()); + /// ``` + pub fn like(self, pattern: impl IntoExpr) -> Expr { + let pattern = pattern.into_expr().untyped; + self.build_filter(move |path| stmt::Expr::like(path, pattern)) + } + + /// Case-insensitive variant of [`like`](Self::like), mapping to + /// PostgreSQL's `ILIKE` operator. + /// + /// PostgreSQL is the only supported backend with a native `ILIKE`, so + /// `.ilike()` works only there. Toasty does not emulate it: on MySQL, + /// SQLite, and DynamoDB the query is rejected with an unsupported-feature + /// error. PostgreSQL's `LIKE` is case-sensitive and so provides `ILIKE` as + /// its case-insensitive companion; the other backends fold ASCII case in + /// `LIKE` (SQLite) or set it through the column collation (MySQL), so there + /// is no operator with matching semantics for `.ilike()` to pass through to. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// // Matches "Alice", "ALICIA", and "alfred". + /// let filter = User::fields().name().ilike("al%".to_string()); + /// ``` + pub fn ilike(self, pattern: impl IntoExpr) -> Expr { + let pattern = pattern.into_expr().untyped; + self.build_filter(move |path| stmt::Expr::ilike(path, pattern)) + } + + /// Test whether this string field matches a SQL `LIKE` pattern using an + /// explicit escape character. + /// + /// Available on any string-valued field, including `String`, + /// `Option`, and other wrappers whose `Field::Inner` is `String`. + /// + /// The caller is responsible for constructing the `LIKE` pattern. Toasty does + /// not escape `pattern` automatically. Any `%` or `_` characters that are not + /// escaped by `escape` are treated as SQL `LIKE` wildcards: + /// + /// - `%` matches any sequence of characters. + /// - `_` matches any single character. + /// - `escape` followed by `%`, `_`, or `escape` matches that character + /// literally. + /// + /// This is useful when the pattern needs to match literal `%` or `_` + /// characters. For example, with `escape` set to `'\\'`, the pattern + /// `"Alice\\%%"` matches strings beginning with the literal text `"Alice%"`. + /// The first `%` is escaped and matched literally; the second `%` remains a + /// wildcard. + /// + /// Not supported by the DynamoDB driver. Use [`starts_with`](Self::starts_with) + /// instead when targeting DynamoDB. + /// + /// `.like_with_escape()` is a pass-through to the database's own `LIKE` + /// operator with an `ESCAPE` clause. Case sensitivity differs by backend: + /// case-sensitive on PostgreSQL, case-insensitive for ASCII on SQLite, and + /// collation-dependent on MySQL. For a case-insensitive match on PostgreSQL, + /// use [`ilike`](Self::ilike). + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # nickname: Option, + /// # } + /// // Match names starting with "Al". + /// let filter = User::fields().name().like_with_escape("Al%".to_string(), '\\'); + /// + /// // Match names starting with the literal text "Alice%". + /// let filter = User::fields().name().like_with_escape("Alice\\%%".to_string(), '\\'); + /// + /// // Also works on nullable string fields. + /// let filter = User::fields().nickname().like_with_escape("Al%".to_string(), '\\'); + /// ``` + pub fn like_with_escape(self, pattern: impl IntoExpr, escape: char) -> Expr { + let pattern = pattern.into_expr().untyped; + self.build_filter(move |path| stmt::Expr::like_with_escape(path, pattern, escape)) + } + + /// Test whether this string field matches a case-insensitive SQL `LIKE` + /// pattern using an explicit escape character. + /// + /// This is the escaped variant of [`ilike`](Self::ilike). It maps to + /// PostgreSQL's `ILIKE ... ESCAPE ...` syntax. + /// + /// PostgreSQL is the only supported backend with a native `ILIKE`, so + /// `.ilike_with_escape()` works only there. Toasty does not emulate it: on + /// MySQL, SQLite, and DynamoDB the query is rejected with an + /// unsupported-feature error. + /// + /// The caller is responsible for constructing the `ILIKE` pattern. Toasty does + /// not escape `pattern` automatically. Any `%` or `_` characters that are not + /// escaped by `escape` are treated as SQL pattern wildcards: + /// + /// - `%` matches any sequence of characters. + /// - `_` matches any single character. + /// - `escape` followed by `%`, `_`, or `escape` matches that character + /// literally. + /// + /// This is useful when the pattern needs to match literal `%` or `_` + /// characters while still matching case-insensitively. For example, with + /// `escape` set to `'\\'`, the pattern `"Alice\\%%"` matches strings beginning + /// with the literal text `"Alice%"`, ignoring case. It can match `"Alice%1"`, + /// `"ALICE%1"`, or `"alice%foo"`, but not `"AliceA1"`. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// // Match names starting with "al", ignoring case. + /// let filter = User::fields().name().ilike_with_escape("al%".to_string(), '\\'); + /// + /// // Match names starting with the literal text "alice%", ignoring case. + /// let filter = User::fields().name().ilike_with_escape("alice\\%%".to_string(), '\\'); + /// ``` + pub fn ilike_with_escape(self, pattern: impl IntoExpr, escape: char) -> Expr { + let pattern = pattern.into_expr().untyped; + self.build_filter(move |path| stmt::Expr::ilike_with_escape(path, pattern, escape)) + } +} + +impl Clone for Path { + fn clone(&self) -> Self { + Self { + untyped: self.untyped.clone(), + _p: PhantomData, + } + } +} + +impl IntoExpr for Path { + fn into_expr(self) -> Expr { + Expr { + untyped: self.untyped.into_stmt(), + _p: PhantomData, + } + } + + fn by_ref(&self) -> Expr { + Self::into_expr(self.clone()) + } +} + +impl From> for stmt::Path { + fn from(value: Path) -> Self { + value.untyped + } +} + +impl fmt::Debug for Path { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{:?}", self.untyped) + } +} diff --git a/packages/toasty/src/stmt/query.rs b/packages/toasty/src/stmt/query.rs new file mode 100644 index 00000000000..a339c3f0845 --- /dev/null +++ b/packages/toasty/src/stmt/query.rs @@ -0,0 +1,671 @@ +use super::{ + Delete, Expr, IntoExpr, IntoStatement, List, RelationInsert, RelationRemove, Statement, Value, +}; +use crate::{ + Executor, Result, + schema::{Load, Model}, + stmt::Path, +}; +use std::{fmt, marker::PhantomData}; +use toasty_core::stmt::{self, Returning}; + +/// A typed query that selects records from the database. +/// +/// The type parameter `T` is the **returning type** — it encodes what +/// `exec()` produces, not just which model is being queried. A `Query` starts +/// as `Query>` (returns `Vec`) and can be narrowed: +/// +/// | Type | `exec()` produces | Created by | +/// |---|---|---| +/// | `Query>` | `Vec` | [`Query::all`], [`Query::filter`] | +/// | `Query` | `M` (errors if missing) | [`.one()`](Query::one) | +/// | `Query>` | `Option` | [`.first()`](Query::first) | +/// +/// # Building queries +/// +/// Start with a generated finder (e.g., `User::filter_by_name("Alice")`) or +/// use [`Query::all`] and chain [`filter`](Query::filter): +/// +/// ``` +/// # #[derive(Debug, toasty::Model)] +/// # struct User { +/// # #[key] +/// # id: i64, +/// # name: String, +/// # age: i64, +/// # } +/// use toasty::stmt::{List, Query}; +/// +/// // All users +/// let q = Query::>::all(); +/// +/// // Filtered +/// let q = Query::>::all().filter(User::fields().age().gt(18)); +/// +/// // Chained +/// let q = Query::>::all() +/// .filter(User::fields().name().eq("Alice")) +/// .limit(10); +/// ``` +/// +/// # Execution +/// +/// Pass the query to [`Db::exec`](crate::Db::exec) or convert it with +/// [`IntoStatement`] for batch use. +pub struct Query { + pub(crate) untyped: stmt::Query, + _p: PhantomData, +} + +// Methods available on all Query +impl Query { + /// Create an empty unit query that returns no records. + /// + /// # Examples + /// + /// ``` + /// # use toasty::stmt::Query; + /// let q = Query::<()>::unit(); + /// ``` + pub fn unit() -> Self { + Self { + untyped: stmt::Query::unit(), + _p: PhantomData, + } + } + + pub(crate) const fn from_untyped(untyped: stmt::Query) -> Self { + Self { + untyped, + _p: PhantomData, + } + } + + /// Take the via association out of this query's source, if any. + /// + /// Returns `Some(_)` when the query was scoped from a relation traversal + /// (e.g. built via [`Association::many`](crate::stmt::Association::many)). + /// After the call the query no longer carries the association on its source. + pub(crate) fn take_via_assoc(&mut self) -> Option { + let stmt::ExprSet::Select(select) = &mut self.untyped.body else { + return None; + }; + let stmt::Source::Model(model) = &mut select.source else { + return None; + }; + model.via.take() + } + + /// Convert a model expression to a query. + /// + /// # Examples + /// + /// ``` + /// # use toasty::stmt::{Query, Expr}; + /// # use toasty_core::stmt as core_stmt; + /// let expr = Expr::::from_untyped(core_stmt::Expr::Value( + /// core_stmt::Value::from(42_i64), + /// )); + /// let _q = Query::from_expr(expr); + /// ``` + pub fn from_expr(expr: Expr) -> Self { + match expr.untyped { + stmt::Expr::Stmt(expr) => match *expr.stmt { + stmt::Statement::Query(stmt) => Self::from_untyped(stmt), + stmt => todo!("stmt={stmt:#?}"), + }, + expr => Self::from_untyped(stmt::Query::values(expr)), + } + } + + /// Add an additional filter, combined with AND, to this query. + /// + /// Returns `self` for method chaining. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// use toasty::stmt::{List, Query}; + /// + /// let q = Query::>::all() + /// .filter(User::fields().name().eq("Alice")); + /// ``` + pub fn filter(mut self, filter: Expr) -> Self { + self.untyped.add_filter(filter.untyped); + self + } + + /// Sets the filter, combined with AND, for this query overwriting existing ones. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// use toasty::stmt::{List, Query}; + /// + /// let mut q = Query::>::all(); + /// q.set_filter(User::fields().name().eq("Alice")); + /// ``` + pub fn set_filter(&mut self, filter: Expr) -> &mut Self { + self.untyped.set_filter(filter); + self + } + + /// Eagerly load a related association when this query executes. + /// + /// `path` identifies the relation to include (e.g., a has-many or + /// belongs-to field). The related records are loaded in the same + /// round-trip and attached to the parent model. + /// + /// A multi-step (`via`) relation can also be included. Its targets are + /// reached through the relation path and grouped under each parent, with + /// duplicate targets collapsed so each one appears once. Including a `via` + /// relation is supported on SQL backends (SQLite, PostgreSQL, MySQL); it + /// is not yet available on DynamoDB. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// use toasty::stmt::{List, Query}; + /// use toasty::schema::Model; + /// + /// // Include the field at index 1 (name) + /// let q = Query::>::all().include(User::path_field::(1)); + /// ``` + pub fn include(mut self, include: impl Into) -> Self { + self.untyped.include(include.into()); + self + } + + /// Add a sort order to this query. + /// + /// Pass an [`OrderByExpr`](toasty_core::stmt::OrderByExpr) obtained from + /// [`Path::asc`] or [`Path::desc`], or a tuple of them to sort by several + /// fields at once. Calling `order_by` multiple times appends each + /// expression to the existing order, so later calls act as tie-breakers + /// for earlier ones. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # age: i64, + /// # } + /// use toasty::stmt::{List, Query}; + /// + /// let q = Query::>::all() + /// .order_by((User::fields().age().desc(), User::fields().name().asc())); + /// ``` + pub fn order_by(mut self, order_by: impl Into) -> Self { + let order_by = order_by.into(); + match &mut self.untyped.order_by { + Some(existing) => existing.exprs.extend(order_by.exprs), + None => self.untyped.order_by = Some(order_by), + } + self + } + + /// Sets the sort order for this query overwriting existing ones. + /// + /// Pass an [`OrderByExpr`](toasty_core::stmt::OrderByExpr) obtained from + /// [`Path::asc`] or [`Path::desc`], or a tuple of them to sort by several + /// fields at once. Calling `order_by` multiple times appends each + /// expression to the existing order, so later calls act as tie-breakers + /// for earlier ones. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # age: i64 + /// # } + /// use toasty::stmt::{List, Query}; + /// + /// let mut q = Query::>::all(); + /// q.set_order_by(User::fields().name().desc()); + /// ``` + pub fn set_order_by(&mut self, order_by: impl Into) -> &mut Self { + let order_by = order_by.into(); + self.untyped.order_by = Some(order_by); + self + } + + /// Limit the number of records returned. + /// + /// `n` is an upper bound, not a guarantee. The limit is applied to the + /// database query, but Toasty may apply additional filtering to the rows + /// the database returns. When that happens, the final result can have + /// fewer than `n` records even if more than `n` rows match the filter + /// expression. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// use toasty::stmt::{List, Query}; + /// + /// let q = Query::>::all().limit(10); + /// ``` + pub fn limit(mut self, n: usize) -> Self { + let n = i64::try_from(n).expect("limit exceeds i64::MAX"); + self.untyped.limit = Some(stmt::Limit::Offset(stmt::LimitOffset { + limit: stmt::Value::from(n).into(), + offset: None, + })); + self + } + + /// Skip the first `n` records. Requires a prior call to [`limit`](Query::limit). + /// + /// # Panics + /// + /// Panics if no `limit` has been set on this query. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// use toasty::stmt::{List, Query}; + /// + /// let q = Query::>::all().limit(10).offset(20); + /// ``` + pub fn offset(mut self, n: usize) -> Self { + let n = i64::try_from(n).expect("offset exceeds i64::MAX"); + self.untyped.limit = match self.untyped.limit.take() { + Some(stmt::Limit::Offset(limit_offset)) => { + Some(stmt::Limit::Offset(stmt::LimitOffset { + limit: limit_offset.limit, + offset: Some(stmt::Value::from(n).into()), + })) + } + Some(stmt::Limit::Cursor(_)) => { + panic!("cannot use offset with cursor-based pagination") + } + None => panic!("limit required for offset"), + }; + self + } + + /// Convert this query into a [`Delete`] statement that removes all matching + /// records. + /// + /// The returned `Delete<()>` does not return the removed records. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// use toasty::stmt::{List, Query}; + /// + /// let delete = Query::>::all().filter(User::fields().name().eq("Alice")) + /// .delete(); + /// ``` + pub fn delete(self) -> Delete<()> { + Delete::from_untyped(self.untyped.delete()) + } + + /// Widen a single-row query back into a list query. + /// + /// This is the inverse of [`first`](Query::first) or [`one`](Query::one). + /// Panics if the query is not currently in single-row mode. + /// + /// # Panics + /// + /// Panics if this query is not a single-row query. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// use toasty::stmt::{List, Query}; + /// + /// let q: Query = Query::>::all().one(); + /// let _list_q: Query> = q.to_list(); + /// ``` + pub fn to_list(mut self) -> Query> { + assert!(self.untyped.single, "not a single query"); + self.untyped.single = false; + + Query { + untyped: self.untyped, + _p: PhantomData, + } + } +} + +impl Query> { + /// Narrow this list query to return at most one record, wrapped in + /// `Option`. + /// + /// The resulting `Query>` returns `Some(record)` if at least one + /// row matches, or `None` if no rows match. + /// + /// This applies `LIMIT 1` to the database query. Toasty may then filter + /// the returned row, so `None` does not always mean that no rows in the + /// table match the filter expression. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// use toasty::stmt::{List, Query}; + /// + /// let q: Query> = Query::>::all().first(); + /// ``` + pub fn first(mut self) -> Query> { + set_first(&mut self.untyped); + + Query { + untyped: self.untyped, + _p: PhantomData, + } + } + + /// Narrow this list query to return exactly one record. + /// + /// The resulting `Query` returns the record directly. If no rows match, + /// execution returns an error. + /// + /// This applies `LIMIT 1` to the database query. If Toasty filters the + /// returned row out, execution returns the same error as when the database + /// returns no rows. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// use toasty::stmt::{List, Query}; + /// + /// let q: Query = Query::>::all().one(); + /// ``` + pub fn one(mut self) -> Query { + set_first(&mut self.untyped); + + Query { + untyped: self.untyped, + _p: PhantomData, + } + } +} + +impl Query> { + /// Convert this query into a [`RelationInsert`] statement that inserts `item` into the + /// relation that produced this list query. + pub fn insert(self, item: impl IntoExpr) -> RelationInsert { + RelationInsert { + query: self, + item: item.into_expr(), + } + } + + /// Convert this query into a [`RelationRemove`] statement that removes `item` from the + /// relation that produced this list query. + pub fn remove(self, item: impl IntoExpr) -> RelationRemove { + RelationRemove { + query: self, + item: item.into_expr(), + } + } +} + +fn set_first(query: &mut stmt::Query) { + assert!(!query.single, "query is single"); + query.single = true; + let offset = match &query.limit { + Some(stmt::Limit::Offset(limit_offset)) => limit_offset.offset.clone(), + _ => None, + }; + query.limit = Some(stmt::Limit::Offset(stmt::LimitOffset { + limit: stmt::Expr::Static(stmt::Value::I64(1)), + offset, + })); +} + +impl Query { + /// Execute this query against the given executor and return the result. + /// + /// The return type depends on the query's type parameter `T`: + /// - `Query>` returns `Vec`. + /// - `Query` returns `M` (errors if no row matches). + /// - `Query>` returns `Option`. + /// + /// # Examples + /// + /// ``` + /// # tokio::runtime::Runtime::new().unwrap().block_on(async { + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// # let driver = toasty_driver_sqlite::Sqlite::in_memory(); + /// # let mut db = toasty::Db::builder().models(toasty::models!(User)).build(driver).await.unwrap(); + /// # db.push_schema().await.unwrap(); + /// use toasty::stmt::{List, Query}; + /// + /// let users: Vec = Query::>::all() + /// .exec(&mut db) + /// .await + /// .unwrap(); + /// # }); + /// ``` + pub async fn exec(self, executor: &mut dyn Executor) -> Result { + executor.exec(self.into_statement()).await + } +} + +/// Methods for list queries: `Query>` +impl Query> { + /// Convert this list query into a count query that returns the number of + /// matching records as a `u64`. + /// + /// The returned `Query` wraps a `SELECT COUNT(*)` query. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// use toasty::stmt::{List, Query}; + /// + /// let q: Query = Query::>::all().count(); + /// ``` + pub fn count(mut self) -> Query { + // Set the returning clause to COUNT(*) + *self.untyped.returning_mut_unwrap() = Returning::Project(stmt::Expr::count_star()); + self.untyped.single = true; + + Query::from_untyped(self.untyped) + } + + /// Project this list query onto an expression, narrowing the returned + /// shape from `M` to `T`. + /// + /// `projection` can be any expression source: a single field path + /// (returning `Vec` for the field's Rust type), a tuple of field paths + /// (returning `Vec` of a tuple), or any other type that implements + /// `IntoExpr`. The default model projection is replaced wholesale by + /// the columns the projection expression references. + /// + /// A multi-step (`via`) relation can be projected as well: a `has_many` + /// `via` yields a `Vec` of the distinct targets reached through the path + /// per row, and a single (`has_one`) `via` yields one target (or `None`). + /// This is supported on SQL backends (SQLite, PostgreSQL, MySQL); it is + /// not yet available on DynamoDB. + /// + /// # Examples + /// + /// ``` + /// # tokio::runtime::Runtime::new().unwrap().block_on(async { + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// # let driver = toasty_driver_sqlite::Sqlite::in_memory(); + /// # let mut db = toasty::Db::builder().models(toasty::models!(User)).build(driver).await.unwrap(); + /// # db.push_schema().await.unwrap(); + /// use toasty::stmt::{List, Query}; + /// + /// // Single-field projection: returns `Vec`. + /// let names: Vec = Query::>::all() + /// .select(User::fields().name()) + /// .exec(&mut db) + /// .await + /// .unwrap(); + /// + /// // Tuple projection: returns `Vec<(i64, String)>`. + /// let pairs: Vec<(i64, String)> = Query::>::all() + /// .select((User::fields().id(), User::fields().name())) + /// .exec(&mut db) + /// .await + /// .unwrap(); + /// # }); + /// ``` + pub fn select(mut self, projection: E) -> Query> + where + E: IntoExpr, + T: Load, + { + *self.untyped.returning_mut_unwrap() = Returning::Project(projection.into_expr().untyped); + + Query::from_untyped(self.untyped) + } + + /// Create a query that selects all records of `M`. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// use toasty::stmt::{List, Query}; + /// + /// let q = Query::>::all(); + /// ``` + pub fn all() -> Self { + let filter = stmt::Expr::Value(Value::from_bool(true)); + let mut query = stmt::Query::new_select(M::id(), filter); + query.single = false; + Self::from_untyped(query) + } + + /// Sort this query so the most recently inserted records on `field` + /// appear first. + /// + /// Convenience for [`order_by`](Self::order_by) with [`Path::desc`]. + /// `field` must be a path rooted at this query's model `M`. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// use toasty::stmt::{List, Query}; + /// + /// let q = Query::>::all().latest_by(User::fields().id()); + /// ``` + pub fn latest_by(self, field: Path) -> Self { + self.order_by(field.desc()) + } +} + +impl IntoStatement for Query { + type Returning = T; + + fn into_statement(self) -> Statement { + Statement::from_untyped_stmt(self.untyped.into()) + } +} + +impl IntoStatement for &Query { + type Returning = T; + + fn into_statement(self) -> Statement { + Statement::from_untyped_stmt(self.clone().untyped.into()) + } +} + +impl Clone for Query { + fn clone(&self) -> Self { + Self { + untyped: self.untyped.clone(), + _p: PhantomData, + } + } +} + +impl fmt::Debug for Query { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + self.untyped.fmt(fmt) + } +} diff --git a/packages/toasty/src/stmt/relation_insert.rs b/packages/toasty/src/stmt/relation_insert.rs new file mode 100644 index 00000000000..439de300e28 --- /dev/null +++ b/packages/toasty/src/stmt/relation_insert.rs @@ -0,0 +1,74 @@ +use crate::{ + Error, Executor, Result, + schema::Model, + stmt::{Association, Expr, List, Query}, +}; + +/// Deferred insert into a has-many relation. Returned by the generated +/// `insert()` method on a relation-scoped `Query>`. On execution +/// this will add an item to the relation this query was scoped from. +/// +/// # Execution +/// +/// Call [`exec`](RelationInsert::exec) to run the insert. +pub struct RelationInsert { + pub(crate) query: Query>, + pub(crate) item: Expr, +} + +impl RelationInsert { + /// Execute this insert statement of a relation against the given executor. + /// + /// Returns `Ok(())` on success. The given item will be inserted as a relation of the parent + /// object into the database. + /// + /// # Examples + /// + /// ``` + /// # tokio::runtime::Runtime::new().unwrap().block_on(async { + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # #[auto] + /// # id: i64, + /// # name: String, + /// # #[has_many] + /// # todos: toasty::Deferred>, + /// # } + /// # #[derive(Debug, toasty::Model)] + /// # struct Todo { + /// # #[key] + /// # #[auto] + /// # id: i64, + /// # title: String, + /// # #[index] + /// # user_id: Option, + /// # #[belongs_to(key = user_id, references = id)] + /// # user: toasty::Deferred>, + /// # } + /// # let driver = toasty_driver_sqlite::Sqlite::in_memory(); + /// # let mut db = toasty::Db::builder().models(toasty::models!(User)).build(driver).await.unwrap(); + /// # db.push_schema().await.unwrap(); + /// let user = User::create() + /// .name("John") + /// .exec(&mut db) + /// .await.unwrap(); + /// let todo = Todo::create().title("dummy").exec(&mut db).await.unwrap(); + /// user.todos().insert(&todo).exec(&mut db).await.unwrap(); + /// # }); + /// ``` + pub async fn exec(mut self, executor: &mut dyn Executor) -> Result<()> { + match self.query.take_via_assoc() { + Some(untyped) if untyped.path.projection.as_slice().len() == 1 => { + let assoc = Association::>::from_untyped(untyped); + executor.exec(assoc.insert(self.item)).await + } + Some(_) => Err(Error::unsupported_feature( + "insert is not supported on multi-step relation traversals", + )), + None => Err(Error::unsupported_feature( + "insert requires a relation-scoped query", + )), + } + } +} diff --git a/packages/toasty/src/stmt/relation_remove.rs b/packages/toasty/src/stmt/relation_remove.rs new file mode 100644 index 00000000000..cdcfb0909ee --- /dev/null +++ b/packages/toasty/src/stmt/relation_remove.rs @@ -0,0 +1,77 @@ +use crate::{ + Error, Executor, Result, + schema::Model, + stmt::{Association, Expr, List, Query}, +}; + +/// Deferred remove of a has-many relation. Returned by the generated +/// `insert()` method on a relation-scoped `Query>`. On execution +/// this will remove an item from the relation this query was scoped from. +/// +/// # Execution +/// +/// Call [`exec`](RelationRemove::exec) to run the removal. +pub struct RelationRemove { + pub(crate) query: Query>, + pub(crate) item: Expr, +} + +impl RelationRemove { + /// Execute this remove statement of a relation against the given executor. + /// + /// Returns `Ok(())` on success. The given item will be removed as a relation fromm the parent + /// object in the database. + /// + /// # Examples + /// + /// ``` + /// # tokio::runtime::Runtime::new().unwrap().block_on(async { + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # #[auto] + /// # id: i64, + /// # name: String, + /// # #[has_many] + /// # todos: toasty::Deferred>, + /// # } + /// # #[derive(Debug, toasty::Model)] + /// # struct Todo { + /// # #[key] + /// # #[auto] + /// # id: i64, + /// # title: String, + /// # #[index] + /// # user_id: Option, + /// # #[belongs_to(key = user_id, references = id)] + /// # user: toasty::Deferred>, + /// # } + /// # let driver = toasty_driver_sqlite::Sqlite::in_memory(); + /// # let mut db = toasty::Db::builder().models(toasty::models!(User)).build(driver).await.unwrap(); + /// # db.push_schema().await.unwrap(); + /// let user = User::create() + /// .name("John") + /// .todos([Todo::create().title("dummy one")]) + /// .todos([Todo::create().title("dummy two")]) + /// .exec(&mut db) + /// .await + /// .unwrap(); + /// let todos: Vec<_> = user.todos().exec(&mut db).await.unwrap(); + /// user.todos().remove(&todos[0]).exec(&mut db).await.unwrap(); + /// # }); + /// ``` + pub async fn exec(mut self, executor: &mut dyn Executor) -> Result<()> { + match self.query.take_via_assoc() { + Some(untyped) if untyped.path.projection.as_slice().len() == 1 => { + let assoc = Association::>::from_untyped(untyped); + executor.exec(assoc.remove(self.item)).await + } + Some(_) => Err(Error::unsupported_feature( + "remove is not supported on multi-step relation traversals", + )), + None => Err(Error::unsupported_feature( + "remove requires a relation-scoped query", + )), + } + } +} diff --git a/packages/toasty/src/stmt/scope.rs b/packages/toasty/src/stmt/scope.rs new file mode 100644 index 00000000000..2c3597f4cd0 --- /dev/null +++ b/packages/toasty/src/stmt/scope.rs @@ -0,0 +1,52 @@ +use super::{List, Query, Statement}; + +/// Convert a query into a scope statement over model `M`. +/// +/// `IntoScope` captures "this is a query targeting model `M`, regardless of +/// cardinality." It admits the three forms a relation accessor can produce — +/// `Query>`, `Query>`, and `Query` — and lowers each to a +/// `Statement>` suitable for use as an insert scope (see +/// [`Insert::set_scope`](crate::stmt::Insert::set_scope)). +/// +/// Compare to [`IntoStatement`](crate::stmt::IntoStatement), which preserves the +/// query's returning type and so distinguishes the three forms above. `IntoScope` +/// erases that distinction, since the scope of an insert only cares about which +/// rows the query addresses, not how many it would return on execution. +/// +/// For `Query>` and `Query` — produced by [`Query::first`] and +/// [`Query::one`] respectively — the `single` flag and the paired `LIMIT 1` +/// added by those narrowing methods are cleared, since `Statement>` +/// would otherwise be self-contradictory ("a list that returns one row"). +/// +/// [`Query::first`]: crate::stmt::Query::first +/// [`Query::one`]: crate::stmt::Query::one +pub trait IntoScope { + /// Lower `self` into a `Statement>` for use as an insert scope. + fn into_scope(self) -> Statement>; +} + +impl IntoScope for Query> { + fn into_scope(self) -> Statement> { + Statement::from_untyped_stmt(self.untyped.into()) + } +} + +impl IntoScope for Query> { + fn into_scope(self) -> Statement> { + Statement::from_untyped_stmt(widen(self.untyped).into()) + } +} + +impl IntoScope for Query { + fn into_scope(self) -> Statement> { + Statement::from_untyped_stmt(widen(self.untyped).into()) + } +} + +fn widen(mut query: toasty_core::stmt::Query) -> toasty_core::stmt::Query { + if query.single { + query.single = false; + query.limit = None; + } + query +} diff --git a/packages/toasty/src/stmt/update.rs b/packages/toasty/src/stmt/update.rs new file mode 100644 index 00000000000..61aa1709139 --- /dev/null +++ b/packages/toasty/src/stmt/update.rs @@ -0,0 +1,210 @@ +use super::Query; +use crate::{ + Executor, Result, + schema::{Load, Model}, +}; +use std::{fmt, marker::PhantomData}; +use toasty_core::stmt; + +/// A typed update statement. +/// +/// `Update` modifies records matching a selection, typically derived from a +/// [`Query`]. Field assignments can be added with [`set`](Update::set) or +/// populated by generated update builders. +/// +/// The type parameter `T` is the returning type—it determines what `exec()` +/// produces, not which model is being updated. For example, `Update` +/// returns one updated `User`, while `Update>` returns the updated +/// records as `Vec`. +/// +/// Generated update-builders wrap this type and expose typed setter methods. +/// You rarely construct `Update` by hand. +pub struct Update { + pub(crate) untyped: stmt::Update, + _p: PhantomData, +} + +// Methods available on all Update regardless of M +impl Update { + /// Create a new update statement from the given query selection. + /// + /// All records matched by `selection` will be updated. By default the + /// update returns the changed records (`Returning::Changed`). + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// use toasty::stmt::{List, Query, Update}; + /// + /// let update = Update::>::new(Query::>::all()); + /// ``` + pub fn new(selection: Query) -> Self { + let mut stmt = selection.untyped.update(); + stmt.returning = Some(stmt::Returning::Changed); + + Self { + untyped: stmt, + _p: PhantomData, + } + } + + /// Get a mutable reference to the underlying untyped update. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// use toasty::stmt::{List, Query, Update}; + /// + /// let mut update = Update::>::new(Query::>::all()); + /// let raw = update.as_untyped_mut(); + /// // Inspect or modify the raw update + /// assert!(raw.returning.is_some()); + /// ``` + pub fn as_untyped_mut(&mut self) -> &mut stmt::Update { + &mut self.untyped + } + + /// Replace all field assignments with `assignments`. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// use toasty::stmt::{List, Query, Update}; + /// + /// let mut update = Update::>::new(Query::>::all()); + /// update.set_assignments(toasty_core::stmt::Assignments::default()); + /// ``` + pub fn set_assignments(&mut self, assignments: stmt::Assignments) { + self.untyped.assignments = assignments; + } + + /// Assign a value to a field. + /// + /// `field` identifies which field to update and `expr` is the new value. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// use toasty::stmt::{List, Query, Update}; + /// + /// let mut update = Update::>::new(Query::>::all()); + /// update.set(1, toasty_core::stmt::Value::from("Bob")); + /// ``` + pub fn set(&mut self, field: impl Into, expr: impl Into) { + self.untyped.assignments.set(field, expr); + } + + /// Consume this typed update and return the untyped core statement. + /// + /// # Examples + /// + /// ``` + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// use toasty::stmt::{List, Query, Update}; + /// + /// let update = Update::>::new(Query::>::all()); + /// let _raw = update.into_untyped(); + /// ``` + pub fn into_untyped(self) -> stmt::Statement { + self.untyped.into() + } +} + +impl Update { + /// Execute this update statement against the given executor. + /// + /// # Examples + /// + /// ``` + /// # tokio::runtime::Runtime::new().unwrap().block_on(async { + /// # #[derive(Debug, toasty::Model)] + /// # struct User { + /// # #[key] + /// # id: i64, + /// # name: String, + /// # } + /// # let driver = toasty_driver_sqlite::Sqlite::in_memory(); + /// # let mut db = toasty::Db::builder().models(toasty::models!(User)).build(driver).await.unwrap(); + /// # db.push_schema().await.unwrap(); + /// use toasty::stmt::{List, Query, Update}; + /// + /// let mut update = Update::>::new(Query::>::all()); + /// update.set(1, toasty_core::stmt::Value::from("Bob")); + /// let _users: Vec = update.exec(&mut db).await.unwrap(); + /// # }); + /// ``` + pub async fn exec(self, executor: &mut dyn Executor) -> Result { + executor.exec(self.into()).await + } +} + +impl Clone for Update { + fn clone(&self) -> Self { + Self { + untyped: self.untyped.clone(), + _p: PhantomData, + } + } +} + +impl Default for Update { + fn default() -> Self { + Self { + untyped: stmt::Update { + target: stmt::UpdateTarget::Model(M::id()), + assignments: stmt::Assignments::default(), + filter: stmt::Filter::new(stmt::Expr::from(false)), + condition: stmt::Condition::default(), + returning: Some(stmt::Returning::Changed), + }, + _p: PhantomData, + } + } +} + +impl AsRef for Update { + fn as_ref(&self) -> &stmt::Update { + &self.untyped + } +} + +impl AsMut for Update { + fn as_mut(&mut self) -> &mut stmt::Update { + &mut self.untyped + } +} + +impl fmt::Debug for Update { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.untyped.fmt(f) + } +} diff --git a/packages/toasty/src/stmt/upsert.rs b/packages/toasty/src/stmt/upsert.rs new file mode 100644 index 00000000000..0d05bde90a3 --- /dev/null +++ b/packages/toasty/src/stmt/upsert.rs @@ -0,0 +1,122 @@ +use super::{Insert, Statement}; +use crate::schema::Model; +use std::{fmt, marker::PhantomData}; +use toasty_core::stmt; + +/// A typed single-row upsert statement for model `M`. +/// +/// `#[derive(Model)]` generates an `upsert_by_*` constructor for each primary +/// key and unique constraint. The generated builder owns an `Upsert` and +/// exposes typed field setters, `on_create`, `on_update`, `or_ignore`, and +/// `exec`. Users normally work through that builder instead of constructing +/// this type directly. +/// +/// An ordinary replacement setter supplies a value for both branches. A shared +/// mutation applies to the field's `#[default]` value when the row is absent +/// and to the stored value when the selected constraint matches. The +/// branch-specific closures override that behavior for fields that need +/// different create and update values. A regular execution returns the record +/// stored by the database; `or_ignore` returns `Some(M)` after an insert and +/// `None` after a conflict. +/// +/// The database executes the conflict check and mutation atomically. Backend +/// support differs: PostgreSQL, SQLite, and Turso accept primary-key and unique +/// targets plus branch-specific assignments; DynamoDB accepts primary-key +/// targets and create-only assignments for required fields, but not +/// update-only assignments; MySQL does not support this targeted upsert API. +/// +/// # Examples +/// +/// ``` +/// # tokio::runtime::Runtime::new().unwrap().block_on(async { +/// # #[derive(Debug, toasty::Model)] +/// # struct User { +/// # #[key] +/// # #[auto] +/// # id: i64, +/// # #[unique] +/// # email: String, +/// # name: String, +/// # } +/// # let driver = toasty_driver_sqlite::Sqlite::in_memory(); +/// # let mut db = toasty::Db::builder().models(toasty::models!(User)).build(driver).await.unwrap(); +/// # db.push_schema().await.unwrap(); +/// let created = User::upsert_by_email("alice@example.com") +/// .name("Alice") +/// .exec(&mut db) +/// .await +/// .unwrap(); +/// +/// let updated = User::upsert_by_email("alice@example.com") +/// .name("Alicia") +/// .exec(&mut db) +/// .await +/// .unwrap(); +/// +/// assert_eq!(updated.id, created.id); +/// assert_eq!(updated.name, "Alicia"); +/// # }); +/// ``` +pub struct Upsert { + pub(crate) untyped: stmt::Insert, + _p: PhantomData, +} + +impl Upsert { + /// Creates a blank upsert targeting the specified model fields. + #[doc(hidden)] + pub fn blank(target: impl IntoIterator) -> Self { + let mut insert = Insert::::blank_single().untyped; + insert.upsert = Some(Box::new(stmt::Upsert { + target: stmt::UpsertTarget::Fields( + target + .into_iter() + .map(stmt::Projection::from_index) + .collect(), + ), + shared: stmt::Assignments::new(), + defaults: stmt::Assignments::new(), + update_defaults: stmt::Assignments::new(), + create: stmt::Assignments::new(), + update: stmt::Assignments::new(), + action: stmt::UpsertAction::Update, + })); + Self { + untyped: insert, + _p: PhantomData, + } + } + + /// Returns the untyped insert AST owned by this wrapper. + #[doc(hidden)] + pub fn untyped_mut(&mut self) -> &mut stmt::Insert { + &mut self.untyped + } + + /// Consumes the wrapper and returns its untyped statement. + #[doc(hidden)] + pub fn into_untyped(self) -> stmt::Statement { + self.untyped.into() + } +} + +impl From> for Statement { + fn from(value: Upsert) -> Self { + Statement::from_untyped_stmt(value.untyped.into()) + } +} + +impl Clone for Upsert { + fn clone(&self) -> Self { + Self { + untyped: self.untyped.clone(), + _p: PhantomData, + } + } +} + +impl fmt::Debug for Upsert { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.untyped.fmt(f) + } +} diff --git a/packages/toasty/src/update_target.rs b/packages/toasty/src/update_target.rs new file mode 100644 index 00000000000..4f09a0f2698 --- /dev/null +++ b/packages/toasty/src/update_target.rs @@ -0,0 +1,33 @@ +use crate::{Result, schema::Load}; +use toasty_core::stmt::{self, Value}; + +/// Trait for types that can serve as the target of an update operation. +/// +/// This trait is implemented by types that represent different update targets: +/// - Generated query struct: builds the update from its inner query, producing +/// `Update>` for multi-row updates +/// - `&mut Model`: builds the update from the model's primary key, producing +/// `Update` for single-row updates +/// +/// The associated type `Returning` determines the statement return type. +pub trait UpdateTarget { + /// The type parameter for the typed `Update` statement. + type Returning: Load; + + /// Build the update statement by combining this target's selection with the + /// provided assignments. + /// + /// For query-based targets, this takes the inner query (replacing it with a + /// default) and wraps it in an `Update>`. + /// For `&mut Model`, this builds an `Update` from the model's PK. + fn to_update_stmt( + &mut self, + assignments: stmt::Assignments, + ) -> crate::stmt::Update; + + /// Apply the result of an update operation. + /// + /// For query-based updates, this discards the values. + /// For instance updates, this reloads the model from the value. + fn apply_result(self, value: Value) -> Result<()>; +} diff --git a/packages/toasty/tests/embedded_migrations.rs b/packages/toasty/tests/embedded_migrations.rs new file mode 100644 index 00000000000..3b98db7a8e8 --- /dev/null +++ b/packages/toasty/tests/embedded_migrations.rs @@ -0,0 +1,170 @@ +#![cfg(feature = "migration")] + +use std::{ + borrow::Cow, + sync::{Arc, Mutex}, +}; + +use async_trait::async_trait; +use toasty::codegen_support::core::{ + Result, Schema, + driver::{Capability, ConnectContext, Connection, Driver, ExecResponse, Operation}, + schema::{ + db::{AppliedMigration, Migration}, + diff, + }, +}; + +#[derive(Debug, toasty::Model)] +struct TestModel { + #[key] + id: u64, +} + +static BASE_MIGRATIONS: toasty::migration::MigrationSet = + toasty::embed_migrations!("tests/fixtures/embedded_migrations/base"); +static DEFAULT_MIGRATIONS: toasty::migration::MigrationSet = toasty::embed_migrations!(); + +#[derive(Debug, Default)] +struct MigrationState { + applied: Mutex>, + sql: Mutex>, +} + +#[derive(Debug)] +struct MigrationDriver { + states: Arc>>>, +} + +impl MigrationDriver { + fn new() -> (Self, Arc>>>) { + let states = Arc::new(Mutex::new(Vec::new())); + ( + Self { + states: states.clone(), + }, + states, + ) + } +} + +#[async_trait] +impl Driver for MigrationDriver { + fn url(&self) -> Cow<'_, str> { + Cow::Borrowed("test:embedded-migrations") + } + + fn capability(&self) -> &'static Capability { + &Capability::SQLITE + } + + async fn connect(&self, _cx: &ConnectContext) -> Result> { + let state = Arc::new(MigrationState::default()); + self.states.lock().unwrap().push(state.clone()); + Ok(Box::new(MigrationConnection { state })) + } + + fn max_connections(&self) -> Option { + Some(1) + } + + fn generate_migration(&self, _schema_diff: &diff::Schema<'_>) -> Migration { + Migration::new_sql(String::new()) + } + + async fn reset_db(&self) -> Result<()> { + for state in self.states.lock().unwrap().iter() { + state.applied.lock().unwrap().clear(); + state.sql.lock().unwrap().clear(); + } + Ok(()) + } +} + +#[derive(Debug)] +struct MigrationConnection { + state: Arc, +} + +#[async_trait] +impl Connection for MigrationConnection { + async fn exec(&mut self, _schema: &Arc, _op: Operation) -> Result { + unreachable!("migration test does not execute query plans") + } + + async fn push_schema(&mut self, _schema: &Schema) -> Result<()> { + Ok(()) + } + + async fn applied_migrations(&mut self) -> Result> { + Ok(self + .state + .applied + .lock() + .unwrap() + .iter() + .copied() + .map(AppliedMigration::new) + .collect()) + } + + async fn apply_migration(&mut self, id: u64, _name: &str, migration: &Migration) -> Result<()> { + self.state.applied.lock().unwrap().push(id); + self.state + .sql + .lock() + .unwrap() + .push(migration.statements().join("\n")); + Ok(()) + } +} + +async fn setup_db() -> (toasty::Db, Arc) { + let (driver, states) = MigrationDriver::new(); + let db = toasty::Db::builder() + .models(toasty::models!(TestModel)) + .build(driver) + .await + .unwrap(); + let states = states.lock().unwrap(); + assert_eq!(states.len(), 1); + let state = states[0].clone(); + (db, state) +} + +#[test] +fn embedded_migrations_can_initialize_static_sets_and_default_to_toasty_directory() { + let base = BASE_MIGRATIONS.migrations(); + assert_eq!(base.len(), 1); + assert_eq!(base[0].id(), 101); + assert_eq!(base[0].name(), "0000_base.sql"); + + let default = DEFAULT_MIGRATIONS.migrations(); + assert_eq!(default.len(), 1); + assert_eq!(default[0].id(), 303); + assert_eq!(default[0].name(), "0000_default.sql"); + assert!(default[0].sql().contains("default_embedded_items")); +} + +#[tokio::test] +async fn embedded_migrations_apply_per_database_and_skip_applied_ids() { + let (base_db, base_state) = setup_db().await; + let (log_db, log_state) = setup_db().await; + let log = toasty::embed_migrations!("tests/fixtures/embedded_migrations/log"); + + let base_report = BASE_MIGRATIONS.apply(&base_db).await.unwrap(); + let log_report = log.apply(&log_db).await.unwrap(); + + assert_eq!(base_report.applied(), 1); + assert_eq!(base_report.skipped(), 0); + assert_eq!(log_report.applied(), 1); + assert_eq!(log_report.skipped(), 0); + assert_eq!(*base_state.applied.lock().unwrap(), vec![101]); + assert_eq!(*log_state.applied.lock().unwrap(), vec![202]); + assert!(base_state.sql.lock().unwrap()[0].contains("base_embedded_items")); + assert!(log_state.sql.lock().unwrap()[0].contains("log_embedded_items")); + + let repeated = BASE_MIGRATIONS.apply(&base_db).await.unwrap(); + assert_eq!(repeated.applied(), 0); + assert_eq!(repeated.skipped(), 1); +} diff --git a/packages/toasty/tests/fixtures/embedded_migrations/base/history.toml b/packages/toasty/tests/fixtures/embedded_migrations/base/history.toml new file mode 100644 index 00000000000..4770b1997db --- /dev/null +++ b/packages/toasty/tests/fixtures/embedded_migrations/base/history.toml @@ -0,0 +1,6 @@ +version = 1 + +[[migrations]] +id = 101 +name = "0000_base.sql" +snapshot_name = "0000_snapshot.toml" diff --git a/packages/toasty/tests/fixtures/embedded_migrations/base/migrations/0000_base.sql b/packages/toasty/tests/fixtures/embedded_migrations/base/migrations/0000_base.sql new file mode 100644 index 00000000000..8422dfa507a --- /dev/null +++ b/packages/toasty/tests/fixtures/embedded_migrations/base/migrations/0000_base.sql @@ -0,0 +1,3 @@ +CREATE TABLE base_embedded_items ( + id INTEGER PRIMARY KEY +); diff --git a/packages/toasty/tests/fixtures/embedded_migrations/log/history.toml b/packages/toasty/tests/fixtures/embedded_migrations/log/history.toml new file mode 100644 index 00000000000..7d257af24a3 --- /dev/null +++ b/packages/toasty/tests/fixtures/embedded_migrations/log/history.toml @@ -0,0 +1,6 @@ +version = 1 + +[[migrations]] +id = 202 +name = "0000_log.sql" +snapshot_name = "0000_snapshot.toml" diff --git a/packages/toasty/tests/fixtures/embedded_migrations/log/migrations/0000_log.sql b/packages/toasty/tests/fixtures/embedded_migrations/log/migrations/0000_log.sql new file mode 100644 index 00000000000..bb889bd592a --- /dev/null +++ b/packages/toasty/tests/fixtures/embedded_migrations/log/migrations/0000_log.sql @@ -0,0 +1,3 @@ +CREATE TABLE log_embedded_items ( + id INTEGER PRIMARY KEY +); diff --git a/packages/toasty/tests/generate_unique_id.rs b/packages/toasty/tests/generate_unique_id.rs new file mode 100644 index 00000000000..82ebddcfa0d --- /dev/null +++ b/packages/toasty/tests/generate_unique_id.rs @@ -0,0 +1,46 @@ +use toasty::codegen_support::generate_unique_id; + +#[test] +fn test_generate_unique_id() { + let id1 = generate_unique_id(); + let id2 = generate_unique_id(); + let id3 = generate_unique_id(); + + // IDs should be unique and sequential + assert_ne!(id1, id2); + assert_ne!(id2, id3); + assert_ne!(id1, id3); + + // Should be sequential (though we can't guarantee exact values due to other tests) + assert!(id1.0 < id2.0); + assert!(id2.0 < id3.0); +} + +#[test] +fn test_generate_unique_id_thread_safety() { + use std::thread; + + let handles: Vec<_> = (0..10) + .map(|_| { + thread::spawn(|| { + let mut ids = Vec::new(); + for _ in 0..100 { + ids.push(generate_unique_id()); + } + ids + }) + }) + .collect(); + + let mut all_ids = Vec::new(); + for handle in handles { + let ids = handle.join().unwrap(); + all_ids.extend(ids); + } + + // All IDs should be unique + all_ids.sort_by_key(|id| id.0); + for window in all_ids.windows(2) { + assert_ne!(window[0], window[1], "Found duplicate ID: {:?}", window[0]); + } +} diff --git a/packages/toasty/tests/migration_generate.rs b/packages/toasty/tests/migration_generate.rs new file mode 100644 index 00000000000..7897186e427 --- /dev/null +++ b/packages/toasty/tests/migration_generate.rs @@ -0,0 +1,125 @@ +#![cfg(feature = "migration")] + +use toasty::migration; +use toasty::schema::{db, diff}; +use toasty_core::stmt; +use toasty_driver_sqlite::Sqlite; + +fn round_trip_snapshot(schema: db::Schema) -> (String, migration::Snapshot) { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("snapshot.toml"); + + migration::Snapshot::new(schema).save(&path).unwrap(); + + let contents = std::fs::read_to_string(&path).unwrap(); + let snapshot = migration::Snapshot::load(path).unwrap(); + (contents, snapshot) +} + +fn users_schema() -> db::Schema { + let table_id = db::TableId(0); + let id = db::ColumnId { + table: table_id, + index: 0, + }; + + db::Schema { + tables: vec![db::Table { + id: table_id, + name: "users".to_string(), + columns: vec![db::Column { + id, + name: "id".to_string(), + ty: stmt::Type::I64, + storage_ty: db::Type::Integer(8), + nullable: false, + primary_key: true, + auto_increment: false, + versionable: false, + }], + primary_key: db::PrimaryKey { + columns: vec![id], + index: db::IndexId { + table: table_id, + index: 0, + }, + }, + indices: vec![], + }], + } +} + +#[test] +fn generate_returns_none_for_empty_diff() { + let schema = users_schema(); + let hints = diff::RenameHints::new(); + let driver = Sqlite::in_memory(); + + let generated = migration::generate(&driver, &schema, &schema, &hints); + + assert!(generated.is_none()); +} + +#[test] +fn generate_returns_migration_and_next_snapshot() { + let previous = db::Schema::default(); + let next = users_schema(); + let hints = diff::RenameHints::new(); + let driver = Sqlite::in_memory(); + + let generated = migration::generate(&driver, &previous, &next, &hints).unwrap(); + + assert_eq!(generated.snapshot.schema.tables[0].name, "users"); + let db::Migration::Sql(sql) = generated.migration; + assert!(sql.contains("CREATE TABLE")); + assert!(sql.contains("users")); +} + +#[test] +fn snapshot_round_trips_empty_indices() { + let (_, snapshot) = round_trip_snapshot(users_schema()); + + assert!(snapshot.schema.tables[0].indices.is_empty()); +} + +#[test] +fn snapshot_round_trips_unconstrained_numeric() { + let mut schema = users_schema(); + schema.tables[0].columns[0].storage_ty = db::Type::Numeric(None); + + let (contents, snapshot) = round_trip_snapshot(schema); + + assert!(contents.contains("storage_ty = { Numeric = [] }")); + assert_eq!( + snapshot.schema.tables[0].columns[0].storage_ty, + db::Type::Numeric(None) + ); +} + +#[test] +fn snapshot_round_trips_bounded_numeric() { + let mut schema = users_schema(); + schema.tables[0].columns[0].storage_ty = db::Type::Numeric(Some((28, 10))); + + let (contents, snapshot) = round_trip_snapshot(schema); + + assert!(contents.contains("storage_ty = { Numeric = [28, 10] }")); + assert_eq!( + snapshot.schema.tables[0].columns[0].storage_ty, + db::Type::Numeric(Some((28, 10))) + ); +} + +#[test] +fn snapshot_round_trips_list_of_unconstrained_numeric() { + let mut schema = users_schema(); + schema.tables[0].columns[0].storage_ty = db::Type::List(Box::new(db::Type::Numeric(None))); + + let (contents, snapshot) = round_trip_snapshot(schema); + + assert!(contents.contains("storage_ty = { List = { Numeric = [] } }")); + assert_eq!( + snapshot.schema.tables[0].columns[0].storage_ty, + db::Type::List(Box::new(db::Type::Numeric(None))) + ); +} diff --git a/packages/toasty/tests/pool_sweep.rs b/packages/toasty/tests/pool_sweep.rs new file mode 100644 index 00000000000..d6590c5a98b --- /dev/null +++ b/packages/toasty/tests/pool_sweep.rs @@ -0,0 +1,553 @@ +//! Deterministic tests for the pool's background sweep and recycle +//! behavior. +//! +//! Each test runs on paused tokio time against a fault-injecting +//! `Driver` that wraps a file-backed SQLite. The wrapper lets the +//! test arm a queue of "next N pings fail" / "next N execs fail" +//! tokens; everything else passes through to SQLite so the engine +//! can run real schema pushes and inserts. No real-time deadlines. + +use std::{ + borrow::Cow, + sync::{ + Arc, + atomic::{AtomicU32, Ordering}, + }, + time::Duration, +}; + +use async_trait::async_trait; +use tempfile::TempDir; +use toasty_core::{ + Result, Schema, + driver::{Capability, ConnectContext, Connection, Driver, ExecResponse, Operation}, + schema::{ + db::{AppliedMigration, Migration}, + diff, + }, +}; + +#[derive(Debug, toasty::Model)] +struct User { + #[key] + #[auto] + id: u64, + name: String, + age: i64, +} + +#[derive(Debug, Default)] +struct MockState { + pings: AtomicU32, + ping_fail_tokens: AtomicU32, + exec_fail_tokens: AtomicU32, + connects: AtomicU32, +} + +#[derive(Debug)] +struct MockDriver { + inner: toasty_driver_sqlite::Sqlite, + state: Arc, + // Keeps the on-disk SQLite file alive for the lifetime of the + // driver — the tempdir self-deletes on drop. + _tempdir: TempDir, +} + +impl MockDriver { + fn new() -> Self { + let tempdir = TempDir::new().expect("create tempdir"); + let path = tempdir.path().join("pool_sweep.db"); + Self { + inner: toasty_driver_sqlite::Sqlite::open(&path), + state: Arc::new(MockState::default()), + _tempdir: tempdir, + } + } + + fn state(&self) -> Arc { + self.state.clone() + } +} + +#[async_trait] +impl Driver for MockDriver { + fn url(&self) -> Cow<'_, str> { + self.inner.url() + } + + fn capability(&self) -> &'static Capability { + self.inner.capability() + } + + async fn connect(&self, cx: &ConnectContext) -> Result> { + let inner = self.inner.connect(cx).await?; + self.state.connects.fetch_add(1, Ordering::Relaxed); + Ok(Box::new(MockConnection { + inner, + state: self.state.clone(), + valid: true, + })) + } + + fn generate_migration(&self, diff: &diff::Schema<'_>) -> Migration { + self.inner.generate_migration(diff) + } + + async fn reset_db(&self) -> Result<()> { + self.inner.reset_db().await + } +} + +#[derive(Debug)] +struct MockConnection { + inner: Box, + state: Arc, + valid: bool, +} + +#[async_trait] +impl Connection for MockConnection { + async fn exec(&mut self, schema: &Arc, op: Operation) -> Result { + if self.state.exec_fail_tokens.load(Ordering::Relaxed) > 0 { + self.state.exec_fail_tokens.fetch_sub(1, Ordering::Relaxed); + self.valid = false; + return Err(toasty_core::Error::connection_lost(std::io::Error::other( + "mock exec failure", + ))); + } + self.inner.exec(schema, op).await + } + + fn is_valid(&self) -> bool { + self.valid && self.inner.is_valid() + } + + async fn ping(&mut self) -> Result<()> { + self.state.pings.fetch_add(1, Ordering::Relaxed); + if self.state.ping_fail_tokens.load(Ordering::Relaxed) > 0 { + self.state.ping_fail_tokens.fetch_sub(1, Ordering::Relaxed); + self.valid = false; + return Err(toasty_core::Error::connection_lost(std::io::Error::other( + "mock ping failure", + ))); + } + self.inner.ping().await + } + + async fn push_schema(&mut self, schema: &Schema) -> Result<()> { + self.inner.push_schema(schema).await + } + + async fn applied_migrations(&mut self) -> Result> { + self.inner.applied_migrations().await + } + + async fn apply_migration(&mut self, id: u64, name: &str, migration: &Migration) -> Result<()> { + self.inner.apply_migration(id, name, migration).await + } +} + +/// Build a `Db` with the given pool configuration. Returns the `Db` +/// and a handle to the mock's fault-injection / counter state. +async fn build_db( + max_pool_size: usize, + health_check_interval: Option, + pre_ping: bool, +) -> (toasty::Db, Arc) { + let driver = MockDriver::new(); + let state = driver.state(); + + let db = toasty::Db::builder() + .models(toasty::models!(User)) + .max_pool_size(max_pool_size) + .pool_health_check_interval(health_check_interval) + .pool_pre_ping(pre_ping) + .build(driver) + .await + .unwrap(); + db.push_schema().await.unwrap(); + (db, state) +} + +/// Passive recovery: with the sweep disabled, a user-observed +/// `connection_lost` must still drain the dead slot so the next +/// call opens a fresh connection. Exercises the `Manager::recycle` +/// path in isolation. +#[tokio::test(start_paused = true)] +async fn pool_recovers_after_connection_lost() { + let (mut db, state) = build_db(1, None, false).await; + + toasty::create!(User { + name: "alice", + age: 30 + }) + .exec(&mut db) + .await + .unwrap(); + + state.exec_fail_tokens.store(1, Ordering::Relaxed); + + let err = toasty::create!(User { + name: "bob", + age: 30 + }) + .exec(&mut db) + .await + .unwrap_err(); + assert!( + err.is_connection_lost(), + "expected connection_lost, got {err}" + ); + + toasty::create!(User { + name: "carol", + age: 30 + }) + .exec(&mut db) + .await + .unwrap(); +} + +/// Periodic sweep evicts a silently-broken idle connection without +/// needing a user query to trip it. +#[tokio::test(start_paused = true)] +async fn sweep_evicts_dead_idle_connection() { + let (mut db, state) = build_db(1, Some(Duration::from_millis(50)), false).await; + + // Force the pool to open its one connection. + toasty::create!(User { + name: "alice", + age: 30 + }) + .exec(&mut db) + .await + .unwrap(); + assert_eq!(db.pool().status().size, 1); + + state.ping_fail_tokens.store(1, Ordering::Relaxed); + + // Advance past the first sweep tick. + tokio::time::sleep(Duration::from_millis(100)).await; + + assert_eq!( + db.pool().status().size, + 0, + "sweep did not evict the dead idle connection", + ); + + // Next user query opens a fresh connection. + toasty::create!(User { + name: "bob", + age: 30 + }) + .exec(&mut db) + .await + .unwrap(); +} + +/// One user query observing `connection_lost` must trigger an eager +/// sweep that pings every remaining idle connection. Otherwise each +/// queued fault would surface as a separate user-query failure. +#[tokio::test(start_paused = true)] +async fn eager_escalation_after_observed_loss() { + // 60-second interval so the periodic tick cannot fire during the + // test — only the notify-driven escalation path can drain the + // queued faults here. + let (mut db, state) = build_db(3, Some(Duration::from_secs(60)), false).await; + + let c1 = db.connection().await.unwrap(); + let c2 = db.connection().await.unwrap(); + let c3 = db.connection().await.unwrap(); + drop((c1, c2, c3)); + assert_eq!(db.pool().status().size, 3); + + // One fault for the user query, two for the sweep's escalation + // pings over the remaining idle connections. + state.exec_fail_tokens.store(1, Ordering::Relaxed); + state.ping_fail_tokens.store(2, Ordering::Relaxed); + + let err = toasty::create!(User { + name: "alice", + age: 30 + }) + .exec(&mut db) + .await + .unwrap_err(); + assert!( + err.is_connection_lost(), + "expected connection_lost, got {err}" + ); + + // Yield long enough on the virtual clock for the sweep to + // observe the wake() and run its escalate. + tokio::time::sleep(Duration::from_millis(50)).await; + + // Sweep must have pinged both remaining idles. With a 60s + // periodic interval, nothing else explains this — only the + // notify-driven escalation path can have fired. + assert_eq!(state.pings.load(Ordering::Relaxed), 2); + // Both queued ping faults were consumed by the escalation pass. + assert_eq!(state.ping_fail_tokens.load(Ordering::Relaxed), 0); + + // And the system recovers: the next queries succeed against + // fresh connections (would surface connection_lost if the bad + // idles were still around). + toasty::create!(User { + name: "bob", + age: 30 + }) + .exec(&mut db) + .await + .unwrap(); + toasty::create!(User { + name: "carol", + age: 30 + }) + .exec(&mut db) + .await + .unwrap(); +} + +/// A failing periodic ping self-wakes via `ConnectionTask::respond`, +/// which queues a sweep-notify permit before the same iteration +/// calls `escalate()`. The escalate snapshot includes that bump, so +/// on the next loop pass the queued permit must be deduped against +/// `last_serviced`. Without the dedup, every periodic-detected +/// failure produces a second escalate that re-pings every surviving +/// idle connection. +#[tokio::test(start_paused = true)] +async fn periodic_failure_does_not_redundantly_escalate() { + let (db, state) = build_db(3, Some(Duration::from_secs(1)), false).await; + + let c1 = db.connection().await.unwrap(); + let c2 = db.connection().await.unwrap(); + let c3 = db.connection().await.unwrap(); + drop((c1, c2, c3)); + assert_eq!(db.pool().status().size, 3); + + state.ping_fail_tokens.store(1, Ordering::Relaxed); + + // Past the first tick (1s), well before the second (2s). + tokio::time::sleep(Duration::from_millis(1_500)).await; + + // 1 failing periodic ping + 2 healthy escalate pings = 3. Without + // dedup, the queued notify permit would drive a second escalate + // over the two remaining idles, raising the count to 5. + assert_eq!( + state.pings.load(Ordering::Relaxed), + 3, + "expected one escalation pass (1 fail + 2 healthy); redundant escalate would have raised this to 5", + ); +} + +/// Pre-ping catches a silently-broken idle connection on checkout +/// and the caller never sees the failure. Sweep is disabled so the +/// only path that can drain the dead slot before the user's query +/// goes out is `Manager::recycle`'s active ping. +#[tokio::test(start_paused = true)] +async fn pre_ping_evicts_silently_broken_idle_connection() { + let (mut db, state) = build_db(1, None, true).await; + + toasty::create!(User { + name: "alice", + age: 30 + }) + .exec(&mut db) + .await + .unwrap(); + assert_eq!(db.pool().status().size, 1); + + let pings_before = state.pings.load(Ordering::Relaxed); + state.ping_fail_tokens.store(1, Ordering::Relaxed); + + // Recycle pings, the ping fails, deadpool drops the slot and + // opens a fresh one before the user's exec runs. + toasty::create!(User { + name: "bob", + age: 30 + }) + .exec(&mut db) + .await + .unwrap(); + + assert_eq!(state.pings.load(Ordering::Relaxed) - pings_before, 1); + assert_eq!(state.ping_fail_tokens.load(Ordering::Relaxed), 0); + assert_eq!(db.pool().status().size, 1); +} + +/// Pre-ping issues a round-trip on every checkout. The first query +/// after build may or may not recycle (depending on whether the +/// build-time probe left a slot warm), so we measure pings against a +/// snapshot taken once the pool is steady-state. +#[tokio::test(start_paused = true)] +async fn pre_ping_runs_on_every_checkout() { + let (mut db, state) = build_db(1, None, true).await; + + // Warm the pool past any setup-driven acquires. + toasty::create!(User { + name: "alice", + age: 30 + }) + .exec(&mut db) + .await + .unwrap(); + + let pings_before = state.pings.load(Ordering::Relaxed); + + // Two more checkouts — each must recycle the lone idle slot and + // therefore fire exactly one pre-ping. + toasty::create!(User { + name: "bob", + age: 30 + }) + .exec(&mut db) + .await + .unwrap(); + toasty::create!(User { + name: "carol", + age: 30 + }) + .exec(&mut db) + .await + .unwrap(); + + assert_eq!(state.pings.load(Ordering::Relaxed) - pings_before, 2); +} + +/// Lifetime cap: a connection older than `pool_max_connection_lifetime` +/// is evicted by `recycle` and replaced with a fresh one. Uses real +/// time because deadpool's `Metrics::age` reads `std::time::Instant`, +/// which `tokio::time::pause` does not advance. +#[tokio::test] +async fn lifetime_cap_evicts_aged_connection() { + let driver = MockDriver::new(); + let state = driver.state(); + + let mut db = toasty::Db::builder() + .models(toasty::models!(User)) + .max_pool_size(1) + .pool_health_check_interval(None) + .pool_max_connection_lifetime(Some(Duration::from_millis(50))) + .build(driver) + .await + .unwrap(); + db.push_schema().await.unwrap(); + + toasty::create!(User { + name: "alice", + age: 30 + }) + .exec(&mut db) + .await + .unwrap(); + + let connects_before = state.connects.load(Ordering::Relaxed); + + // Real-time wait past the lifetime cap. + tokio::time::sleep(Duration::from_millis(150)).await; + + toasty::create!(User { + name: "bob", + age: 30 + }) + .exec(&mut db) + .await + .unwrap(); + + assert_eq!( + state.connects.load(Ordering::Relaxed) - connects_before, + 1, + "recycle did not replace the over-lifetime connection", + ); + assert_eq!(db.pool().status().size, 1); +} + +/// Idle-time cap: a connection that has been sitting unused for longer +/// than `pool_max_connection_idle_time` is evicted by `recycle` and +/// replaced on the next checkout. +#[tokio::test] +async fn idle_time_cap_evicts_long_idle_connection() { + let driver = MockDriver::new(); + let state = driver.state(); + + let mut db = toasty::Db::builder() + .models(toasty::models!(User)) + .max_pool_size(1) + .pool_health_check_interval(None) + .pool_max_connection_idle_time(Some(Duration::from_millis(50))) + .build(driver) + .await + .unwrap(); + db.push_schema().await.unwrap(); + + toasty::create!(User { + name: "alice", + age: 30 + }) + .exec(&mut db) + .await + .unwrap(); + + let connects_before = state.connects.load(Ordering::Relaxed); + + // Real-time wait past the idle cap. + tokio::time::sleep(Duration::from_millis(150)).await; + + toasty::create!(User { + name: "bob", + age: 30 + }) + .exec(&mut db) + .await + .unwrap(); + + assert_eq!( + state.connects.load(Ordering::Relaxed) - connects_before, + 1, + "recycle did not replace the over-idle connection", + ); + assert_eq!(db.pool().status().size, 1); +} + +/// A connection within both caps is reused — `recycle` must not evict +/// a healthy young connection. +#[tokio::test] +async fn lifetime_and_idle_caps_do_not_evict_fresh_connection() { + let driver = MockDriver::new(); + let state = driver.state(); + + let mut db = toasty::Db::builder() + .models(toasty::models!(User)) + .max_pool_size(1) + .pool_health_check_interval(None) + .pool_max_connection_lifetime(Some(Duration::from_secs(60))) + .pool_max_connection_idle_time(Some(Duration::from_secs(60))) + .build(driver) + .await + .unwrap(); + db.push_schema().await.unwrap(); + + toasty::create!(User { + name: "alice", + age: 30 + }) + .exec(&mut db) + .await + .unwrap(); + + let connects_before = state.connects.load(Ordering::Relaxed); + + toasty::create!(User { + name: "bob", + age: 30 + }) + .exec(&mut db) + .await + .unwrap(); + + assert_eq!( + state.connects.load(Ordering::Relaxed), + connects_before, + "recycle replaced a connection within both caps", + ); +} diff --git a/packages/toasty/tests/query_macro.rs b/packages/toasty/tests/query_macro.rs new file mode 100644 index 00000000000..1355c0915cc --- /dev/null +++ b/packages/toasty/tests/query_macro.rs @@ -0,0 +1,515 @@ +use assert_struct::assert_struct; +use toasty::schema::Model; +use toasty::stmt::IntoStatement; +use toasty_core::stmt::{self as core_stmt, ExprReference}; + +// --------------------------------------------------------------------------- +// Test model — field indices: id=0, name=1, age=2, active=3 +// --------------------------------------------------------------------------- + +#[derive(Debug, toasty::Model)] +struct User { + #[key] + id: i64, + name: String, + age: i64, + active: bool, +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/// Extract the core `Select` from anything implementing IntoStatement. +fn select(q: impl IntoStatement) -> core_stmt::Select { + q.into_statement() + .into_untyped() + .into_query_unwrap() + .into_select() +} + +/// Extract just the filter expression. +fn filter_expr(q: impl IntoStatement) -> core_stmt::Expr { + select(q).filter.into_expr() +} + +/// Shorthand for a field reference expr. +fn field_ref(index: usize) -> core_stmt::Expr { + core_stmt::Expr::Reference(ExprReference::Field { nesting: 0, index }) +} + +/// Shorthand for a string value expr. +fn str_val(s: &str) -> core_stmt::Expr { + core_stmt::Expr::Value(core_stmt::Value::String(s.into())) +} + +/// Shorthand for an i64 value expr. +fn i64_val(n: i64) -> core_stmt::Expr { + core_stmt::Expr::Value(core_stmt::Value::I64(n)) +} + +/// Shorthand for a bool value expr. +fn bool_val(b: bool) -> core_stmt::Expr { + core_stmt::Expr::Value(core_stmt::Value::Bool(b)) +} + +// --------------------------------------------------------------------------- +// No filter +// --------------------------------------------------------------------------- + +#[test] +fn query_all() { + let sel = select(toasty::query!(User)); + + assert_struct!(sel, { + source: core_stmt::Source::Model({ via: None }), + filter: { + expr: Some(core_stmt::Expr::Value(core_stmt::Value::Bool(true))), + }, + returning: core_stmt::Returning::Model { include: [] }, + }); +} + +// --------------------------------------------------------------------------- +// Comparison operators +// --------------------------------------------------------------------------- + +#[test] +fn filter_eq_string() { + let expr = filter_expr(toasty::query!(User filter .name == "Alice")); + assert_eq!(expr, core_stmt::Expr::eq(field_ref(1), str_val("Alice"))); +} + +#[test] +fn filter_ne_string() { + let expr = filter_expr(toasty::query!(User filter .name != "Bob")); + assert_eq!(expr, core_stmt::Expr::ne(field_ref(1), str_val("Bob"))); +} + +#[test] +fn filter_gt() { + let expr = filter_expr(toasty::query!(User filter .age > 18)); + assert_eq!(expr, core_stmt::Expr::gt(field_ref(2), i64_val(18))); +} + +#[test] +fn filter_ge() { + let expr = filter_expr(toasty::query!(User filter .age >= 21)); + assert_eq!(expr, core_stmt::Expr::ge(field_ref(2), i64_val(21))); +} + +#[test] +fn filter_lt() { + let expr = filter_expr(toasty::query!(User filter .age < 65)); + assert_eq!(expr, core_stmt::Expr::lt(field_ref(2), i64_val(65))); +} + +#[test] +fn filter_le() { + let expr = filter_expr(toasty::query!(User filter .age <= 99)); + assert_eq!(expr, core_stmt::Expr::le(field_ref(2), i64_val(99))); +} + +// --------------------------------------------------------------------------- +// Boolean operators +// --------------------------------------------------------------------------- + +#[test] +fn filter_and() { + let expr = filter_expr(toasty::query!(User filter .name == "Alice" and .age > 18)); + + assert_eq!( + expr, + core_stmt::Expr::and( + core_stmt::Expr::eq(field_ref(1), str_val("Alice")), + core_stmt::Expr::gt(field_ref(2), i64_val(18)), + ) + ); +} + +#[test] +fn filter_or() { + let expr = filter_expr(toasty::query!(User filter .name == "Alice" or .name == "Bob")); + + assert_eq!( + expr, + core_stmt::Expr::or( + core_stmt::Expr::eq(field_ref(1), str_val("Alice")), + core_stmt::Expr::eq(field_ref(1), str_val("Bob")), + ) + ); +} + +#[test] +fn filter_not() { + let expr = filter_expr(toasty::query!(User filter not .active == true)); + + assert_eq!( + expr, + core_stmt::Expr::not(core_stmt::Expr::eq(field_ref(3), bool_val(true))) + ); +} + +// --------------------------------------------------------------------------- +// Parenthesized expressions +// --------------------------------------------------------------------------- + +#[test] +fn filter_parens_override_precedence() { + // Without parens: .name == "A" or (.name == "B" and .age > 18) + // With parens: (.name == "A" or .name == "B") and .age > 18 + let expr = + filter_expr(toasty::query!(User filter (.name == "Alice" or .name == "Bob") and .age > 18)); + + // The outer node should be AND (parens made OR bind tighter) + assert_eq!( + expr, + core_stmt::Expr::and( + core_stmt::Expr::or( + core_stmt::Expr::eq(field_ref(1), str_val("Alice")), + core_stmt::Expr::eq(field_ref(1), str_val("Bob")), + ), + core_stmt::Expr::gt(field_ref(2), i64_val(18)), + ) + ); +} + +// --------------------------------------------------------------------------- +// Boolean literals +// --------------------------------------------------------------------------- + +#[test] +fn filter_bool_true() { + let expr = filter_expr(toasty::query!(User filter .active == true)); + assert_eq!(expr, core_stmt::Expr::eq(field_ref(3), bool_val(true))); +} + +#[test] +fn filter_bool_false() { + let expr = filter_expr(toasty::query!(User filter .active == false)); + assert_eq!(expr, core_stmt::Expr::eq(field_ref(3), bool_val(false))); +} + +// --------------------------------------------------------------------------- +// External references +// --------------------------------------------------------------------------- + +#[test] +fn filter_external_variable() { + let name = String::from("Carl"); + let expr = filter_expr(toasty::query!(User filter .name == #name)); + assert_eq!(expr, core_stmt::Expr::eq(field_ref(1), str_val("Carl"))); +} + +#[test] +fn filter_external_expression() { + fn make_name() -> String { + String::from("Computed") + } + + let expr = filter_expr(toasty::query!(User filter .name == #(make_name()))); + assert_eq!(expr, core_stmt::Expr::eq(field_ref(1), str_val("Computed"))); +} + +// --------------------------------------------------------------------------- +// Case-insensitive keywords +// --------------------------------------------------------------------------- + +#[test] +fn filter_keyword_uppercase() { + let expr = filter_expr(toasty::query!(User FILTER .name == "test")); + assert_eq!(expr, core_stmt::Expr::eq(field_ref(1), str_val("test"))); +} + +#[test] +fn filter_keyword_mixed_case() { + let expr = filter_expr(toasty::query!(User Filter .name == "test")); + assert_eq!(expr, core_stmt::Expr::eq(field_ref(1), str_val("test"))); +} + +#[test] +fn and_keyword_uppercase() { + let expr = filter_expr(toasty::query!(User filter .name == "a" AND .age > 1)); + + assert_struct!(expr, core_stmt::Expr::And({ operands.len(): 2 })); +} + +#[test] +fn or_keyword_uppercase() { + let expr = filter_expr(toasty::query!(User filter .name == "a" OR .name == "b")); + + assert_struct!(expr, core_stmt::Expr::Or({ operands.len(): 2 })); +} + +#[test] +fn not_keyword_uppercase() { + let expr = filter_expr(toasty::query!(User filter NOT .active == true)); + + assert_struct!(expr, core_stmt::Expr::Not(_)); +} + +// --------------------------------------------------------------------------- +// Complex compound expressions +// --------------------------------------------------------------------------- + +#[test] +fn complex_and_or_not_with_parens() { + // Precedence: NOT > AND > OR + // not .active == true and (.name == "Alice" or .age >= 21) + // → (NOT(.active == true)) AND ((.name == "Alice") OR (.age >= 21)) + let expr = filter_expr( + toasty::query!(User filter not .active == true and (.name == "Alice" or .age >= 21)), + ); + + assert_eq!( + expr, + core_stmt::Expr::and( + core_stmt::Expr::not(core_stmt::Expr::eq(field_ref(3), bool_val(true))), + core_stmt::Expr::or( + core_stmt::Expr::eq(field_ref(1), str_val("Alice")), + core_stmt::Expr::ge(field_ref(2), i64_val(21)), + ), + ) + ); +} + +#[test] +fn triple_and() { + let expr = + filter_expr(toasty::query!(User filter .name == "A" and .age > 0 and .active == true)); + + // AND flattens: and(and(a, b), c) → and(a, b, c) + assert_struct!(expr, core_stmt::Expr::And({ operands.len(): 3 })); +} + +#[test] +fn triple_or() { + let expr = + filter_expr(toasty::query!(User filter .name == "A" or .name == "B" or .name == "C")); + + // OR flattens: or(or(a, b), c) → or(a, b, c) + assert_struct!(expr, core_stmt::Expr::Or({ operands.len(): 3 })); +} + +#[test] +fn or_precedence_lower_than_and() { + // a and b or c → (a and b) or c + let expr = + filter_expr(toasty::query!(User filter .name == "A" and .age > 0 or .active == false)); + + assert_eq!( + expr, + core_stmt::Expr::or( + core_stmt::Expr::and( + core_stmt::Expr::eq(field_ref(1), str_val("A")), + core_stmt::Expr::gt(field_ref(2), i64_val(0)), + ), + core_stmt::Expr::eq(field_ref(3), bool_val(false)), + ) + ); +} + +#[test] +fn double_not() { + let expr = filter_expr(toasty::query!(User filter not not .active == true)); + + assert_eq!( + expr, + core_stmt::Expr::not(core_stmt::Expr::not(core_stmt::Expr::eq( + field_ref(3), + bool_val(true) + ))) + ); +} + +// --------------------------------------------------------------------------- +// Query-level properties +// --------------------------------------------------------------------------- + +#[test] +fn query_all_is_not_single() { + let query = toasty::query!(User) + .into_statement() + .into_untyped() + .into_query_unwrap(); + + assert_struct!(query, { + single: false, + order_by: None, + limit: None, + with: None, + body: core_stmt::ExprSet::Select(_), + }); +} + +#[test] +fn filter_query_is_not_single() { + let query = toasty::query!(User filter .age > 0) + .into_statement() + .into_untyped() + .into_query_unwrap(); + + assert_struct!(query, { + single: false, + body: core_stmt::ExprSet::Select({ + source: core_stmt::Source::Model({ via: None }), + }), + }); +} + +// --------------------------------------------------------------------------- +// Integer literals +// --------------------------------------------------------------------------- + +#[test] +fn filter_integer_literal() { + let expr = filter_expr(toasty::query!(User filter .age == 42)); + assert_eq!(expr, core_stmt::Expr::eq(field_ref(2), i64_val(42))); +} + +// --------------------------------------------------------------------------- +// Multiple fields in same expression +// --------------------------------------------------------------------------- + +#[test] +fn filter_different_fields() { + let expr = filter_expr(toasty::query!(User filter .id == 1 and .name == "X" and .age > 0)); + + assert_eq!( + expr, + core_stmt::Expr::and( + core_stmt::Expr::and( + core_stmt::Expr::eq(field_ref(0), i64_val(1)), + core_stmt::Expr::eq(field_ref(1), str_val("X")), + ), + core_stmt::Expr::gt(field_ref(2), i64_val(0)), + ) + ); +} + +// --------------------------------------------------------------------------- +// Nested parentheses +// --------------------------------------------------------------------------- + +#[test] +fn nested_parens() { + let expr = + filter_expr(toasty::query!(User filter ((.name == "A" or .name == "B") and .age > 0))); + + // Outer parens are transparent; inner parens group the OR + assert_eq!( + expr, + core_stmt::Expr::and( + core_stmt::Expr::or( + core_stmt::Expr::eq(field_ref(1), str_val("A")), + core_stmt::Expr::eq(field_ref(1), str_val("B")), + ), + core_stmt::Expr::gt(field_ref(2), i64_val(0)), + ) + ); +} + +// --------------------------------------------------------------------------- +// Source model identity +// --------------------------------------------------------------------------- + +#[test] +fn source_model_matches_user_id() { + let sel = select(toasty::query!(User)); + + // Verify the source references the User model specifically + let source_model = sel.source.as_model_unwrap(); + assert_eq!(source_model.id, ::id()); + assert!(source_model.via.is_none()); +} + +#[test] +fn filter_source_model_matches_user_id() { + let sel = select(toasty::query!(User filter .name == "test")); + + let source_model = sel.source.as_model_unwrap(); + assert_eq!(source_model.id, ::id()); +} + +// --------------------------------------------------------------------------- +// IN list +// --------------------------------------------------------------------------- + +#[test] +fn filter_in_list_integers() { + let expr = filter_expr(toasty::query!(User filter .id IN [1, 2, 3])); + + assert_eq!( + expr, + core_stmt::Expr::in_list( + field_ref(0), + core_stmt::Expr::List(core_stmt::ExprList { + items: vec![i64_val(1), i64_val(2), i64_val(3)], + }), + ) + ); +} + +#[test] +fn filter_in_list_strings() { + let expr = filter_expr(toasty::query!(User filter .name IN ["Alice", "Bob"])); + + assert_eq!( + expr, + core_stmt::Expr::in_list( + field_ref(1), + core_stmt::Expr::List(core_stmt::ExprList { + items: vec![str_val("Alice"), str_val("Bob")], + }), + ) + ); +} + +#[test] +fn filter_in_list_external_variable() { + let ids = vec![1_i64, 2, 3]; + let expr = filter_expr(toasty::query!(User filter .id IN #ids)); + + assert_eq!( + expr, + core_stmt::Expr::in_list( + field_ref(0), + core_stmt::Expr::List(core_stmt::ExprList { + items: vec![i64_val(1), i64_val(2), i64_val(3)], + }), + ) + ); +} + +#[test] +fn filter_in_list_with_and() { + let expr = filter_expr(toasty::query!(User filter .id IN [1, 2] and .active == true)); + + assert_eq!( + expr, + core_stmt::Expr::and( + core_stmt::Expr::in_list( + field_ref(0), + core_stmt::Expr::List(core_stmt::ExprList { + items: vec![i64_val(1), i64_val(2)], + }), + ), + core_stmt::Expr::eq(field_ref(3), bool_val(true)), + ) + ); +} + +#[test] +fn filter_in_list_case_insensitive() { + let expr = filter_expr(toasty::query!(User filter .id in [1, 2, 3])); + + assert_eq!( + expr, + core_stmt::Expr::in_list( + field_ref(0), + core_stmt::Expr::List(core_stmt::ExprList { + items: vec![i64_val(1), i64_val(2), i64_val(3)], + }), + ) + ); +} diff --git a/packages/toasty/tests/relation_load.rs b/packages/toasty/tests/relation_load.rs new file mode 100644 index 00000000000..2b28e9343a0 --- /dev/null +++ b/packages/toasty/tests/relation_load.rs @@ -0,0 +1,143 @@ +use toasty::Deferred; +use toasty::schema::{self, Load, Model, ModelSet, RelationManyField, RelationOneField}; +use toasty::stmt::{Expr, Insert, IntoExpr, IntoInsert, Path}; +use toasty_core::stmt::{self, Value}; + +#[derive(Debug, PartialEq)] +struct Dummy { + id: i64, +} + +#[derive(Default)] +struct DummyCreate; + +impl Load for Dummy { + type Output = Self; + + fn ty() -> stmt::Type { + stmt::Type::Model(Self::id()) + } + + fn load(value: Value) -> toasty::Result { + let Value::Record(mut record) = value else { + return Err(toasty_core::Error::type_conversion(value, "Dummy")); + }; + + let id = record[0].take(); + let Value::I64(id) = id else { + return Err(toasty_core::Error::type_conversion(id, "Dummy::id")); + }; + + Ok(Self { id }) + } +} + +impl Model for Dummy { + type Query = (); + type Create = DummyCreate; + type Update<'a> = (); + type UpdateQuery = (); + type Path = Path; + type PrimaryKey = i64; + type ManyField = (); + type OneField = (); + + fn id() -> schema::app::ModelId { + schema::app::ModelId(usize::MAX) + } + + fn schema() -> schema::app::Model { + panic!("not needed for relation lazy-slot decode tests") + } + + fn register(_model_set: &mut ModelSet) {} + + fn new_path(path: Path) -> Self::Path { + path + } + + fn new_many_field(_path: Path>) {} + + fn field_name_to_id(_name: &str) -> schema::app::FieldId { + panic!("not needed for relation lazy-slot decode tests") + } + + fn find_by_primary_key(_id: Expr) -> Self::Query> {} + + fn wrap_query(_stmt: toasty::stmt::Query) -> Self::Query {} + + fn query_one(_query: Self::Query>) -> Self::Query {} + + fn query_first(_query: Self::Query>) -> Self::Query> {} +} + +impl IntoInsert for DummyCreate { + type Model = Dummy; + + fn into_insert(self) -> Insert { + panic!("not needed for relation lazy-slot decode tests") + } +} + +impl IntoExpr for DummyCreate { + fn into_expr(self) -> Expr { + panic!("not needed for relation lazy-slot decode tests") + } + + fn by_ref(&self) -> Expr { + panic!("not needed for relation lazy-slot decode tests") + } +} + +fn assert_has_many_field>() {} +fn assert_has_one_field>() {} +fn assert_belongs_to_field>() {} + +#[test] +fn deferred_relation_field_shapes_are_supported() { + assert_has_many_field::>(); + assert_has_many_field::>>(); + + assert_has_one_field::(); + assert_has_one_field::>(); + assert_has_one_field::>(); + assert_has_one_field::>>(); + + assert_belongs_to_field::(); + assert_belongs_to_field::>(); + assert_belongs_to_field::>(); + assert_belongs_to_field::>>(); +} + +#[test] +fn deferred_load_distinguishes_unloaded_and_loaded_null() { + assert!( + Deferred::>::load(Value::Null) + .unwrap() + .is_unloaded() + ); + for value in [None, Some(42)] { + let record = Value::Record(stmt::ValueRecord::from_vec(vec![value.into()])); + let loaded = Deferred::>::load(record).unwrap(); + assert!(!loaded.is_unloaded()); + assert_eq!(loaded.into_inner(), value); + } +} + +#[test] +fn deferred_load_rejects_malformed_values() { + for value in [ + Value::I64(1), + Value::Record(stmt::ValueRecord::from_vec(vec![])), + Value::Record(stmt::ValueRecord::from_vec(vec![Value::Null, Value::Null])), + ] { + let error = Deferred::>::load(value).unwrap_err(); + assert!( + error + .to_string() + .contains("deferred field decoder expected Null or single-field Record") + ); + } + let invalid_inner = Value::Record(stmt::ValueRecord::from_vec(vec![Value::Bool(true)])); + assert!(Deferred::::load(invalid_inner).is_err()); +} diff --git a/packages/toasty/tests/stmt_into_expr.rs b/packages/toasty/tests/stmt_into_expr.rs new file mode 100644 index 00000000000..8a867768e86 --- /dev/null +++ b/packages/toasty/tests/stmt_into_expr.rs @@ -0,0 +1,363 @@ +use toasty::stmt::{Expr, IntoExpr, List}; +use toasty_core::stmt as core_stmt; + +// --------------------------------------------------------------------------- +// Helper: unwrap the core expression from a typed Expr +// --------------------------------------------------------------------------- + +fn untyped(expr: Expr) -> core_stmt::Expr { + core_stmt::Expr::from(expr) +} + +// --------------------------------------------------------------------------- +// Primitive scalar impls +// --------------------------------------------------------------------------- + +#[test] +fn into_expr_bool() { + let expr: Expr = true.into_expr(); + assert_eq!( + untyped(expr), + core_stmt::Expr::Value(core_stmt::Value::Bool(true)) + ); +} + +#[test] +fn into_expr_i64() { + let expr: Expr = 42i64.into_expr(); + assert_eq!( + untyped(expr), + core_stmt::Expr::Value(core_stmt::Value::I64(42)) + ); +} + +#[test] +fn into_expr_string() { + let expr: Expr = String::from("hello").into_expr(); + assert_eq!( + untyped(expr), + core_stmt::Expr::Value(core_stmt::Value::String("hello".into())) + ); +} + +#[test] +fn into_expr_str_ref() { + let expr: Expr = "world".into_expr(); + assert_eq!( + untyped(expr), + core_stmt::Expr::Value(core_stmt::Value::String("world".into())) + ); +} + +#[test] +fn into_expr_uuid() { + let id = uuid::Uuid::nil(); + let expr: Expr = id.into_expr(); + assert_eq!( + untyped(expr), + core_stmt::Expr::Value(core_stmt::Value::Uuid(uuid::Uuid::nil())) + ); +} + +// --------------------------------------------------------------------------- +// by_ref for scalars +// --------------------------------------------------------------------------- + +#[test] +fn by_ref_i64() { + let val = 7i64; + let expr: Expr = val.by_ref(); + assert_eq!( + untyped(expr), + core_stmt::Expr::Value(core_stmt::Value::I64(7)) + ); +} + +#[test] +fn by_ref_string() { + let val = String::from("ref"); + let expr: Expr = val.by_ref(); + assert_eq!( + untyped(expr), + core_stmt::Expr::Value(core_stmt::Value::String("ref".into())) + ); +} + +// --------------------------------------------------------------------------- +// Expr identity impl +// --------------------------------------------------------------------------- + +#[test] +fn into_expr_identity() { + let original: Expr = 99i64.into_expr(); + let cloned_untyped = untyped(original.clone()); + let roundtripped: Expr = original.into_expr(); + assert_eq!(untyped(roundtripped), cloned_untyped); +} + +#[test] +fn by_ref_identity() { + let original: Expr = 99i64.into_expr(); + let cloned_untyped = untyped(original.clone()); + let by_ref_expr: Expr = original.by_ref(); + assert_eq!(untyped(by_ref_expr), cloned_untyped); +} + +// --------------------------------------------------------------------------- +// &T delegates to T::by_ref +// --------------------------------------------------------------------------- + +#[test] +fn into_expr_ref_delegates() { + let val = 5i64; + let from_ref: Expr = (&val).into_expr(); + let from_by_ref: Expr = val.by_ref(); + assert_eq!(untyped(from_ref), untyped(from_by_ref)); +} + +// --------------------------------------------------------------------------- +// Option +// --------------------------------------------------------------------------- + +#[test] +fn into_expr_some() { + let expr: Expr> = Some(10i64).into_expr(); + // Some wraps via cast, so the inner value should be an i64 + assert_eq!( + untyped(expr), + core_stmt::Expr::Value(core_stmt::Value::I64(10)) + ); +} + +#[test] +fn into_expr_none() { + let expr: Expr> = None::.into_expr(); + assert_eq!( + untyped(expr), + core_stmt::Expr::Value(core_stmt::Value::Null) + ); +} + +#[test] +fn by_ref_some() { + let val: Option = Some(10); + let expr: Expr> = val.by_ref(); + assert_eq!( + untyped(expr), + core_stmt::Expr::Value(core_stmt::Value::I64(10)) + ); +} + +#[test] +fn by_ref_none() { + let val: Option = None; + let expr: Expr> = val.by_ref(); + assert_eq!( + untyped(expr), + core_stmt::Expr::Value(core_stmt::Value::Null) + ); +} + +// --------------------------------------------------------------------------- +// T -> Option +// --------------------------------------------------------------------------- + +#[test] +fn into_expr_value_as_option() { + let expr: Expr> = 42i64.into_expr(); + assert_eq!( + untyped(expr), + core_stmt::Expr::Value(core_stmt::Value::I64(42)) + ); +} + +// --------------------------------------------------------------------------- +// &Option -> T +// --------------------------------------------------------------------------- + +// --------------------------------------------------------------------------- +// Arrays [U; N] -> List +// --------------------------------------------------------------------------- + +#[test] +fn into_expr_array_to_list() { + let arr = [1i64, 2, 3]; + let expr: Expr> = arr.into_expr(); + let expected = core_stmt::Expr::list([ + core_stmt::Expr::Value(core_stmt::Value::I64(1)), + core_stmt::Expr::Value(core_stmt::Value::I64(2)), + core_stmt::Expr::Value(core_stmt::Value::I64(3)), + ]); + assert_eq!(untyped(expr), expected); +} + +#[test] +fn into_expr_empty_array_to_list() { + let arr: [i64; 0] = []; + let expr: Expr> = arr.into_expr(); + let expected = core_stmt::Expr::list(std::iter::empty::()); + assert_eq!(untyped(expr), expected); +} + +#[test] +fn by_ref_array_to_list() { + let arr = [1i64, 2]; + let expr: Expr> = arr.by_ref(); + let expected = core_stmt::Expr::list([ + core_stmt::Expr::Value(core_stmt::Value::I64(1)), + core_stmt::Expr::Value(core_stmt::Value::I64(2)), + ]); + assert_eq!(untyped(expr), expected); +} + +// --------------------------------------------------------------------------- +// &[U; N] -> List +// --------------------------------------------------------------------------- + +#[test] +fn into_expr_ref_array_to_list() { + let arr = [10i64, 20]; + let expr: Expr> = (&arr).into_expr(); + let expected = core_stmt::Expr::list([ + core_stmt::Expr::Value(core_stmt::Value::I64(10)), + core_stmt::Expr::Value(core_stmt::Value::I64(20)), + ]); + assert_eq!(untyped(expr), expected); +} + +// --------------------------------------------------------------------------- +// Vec -> List +// --------------------------------------------------------------------------- + +#[test] +fn into_expr_vec_to_list() { + let v = vec![4i64, 5, 6]; + let expr: Expr> = v.into_expr(); + let expected = core_stmt::Expr::list([ + core_stmt::Expr::Value(core_stmt::Value::I64(4)), + core_stmt::Expr::Value(core_stmt::Value::I64(5)), + core_stmt::Expr::Value(core_stmt::Value::I64(6)), + ]); + assert_eq!(untyped(expr), expected); +} + +#[test] +fn into_expr_empty_vec_to_list() { + let v: Vec = vec![]; + let expr: Expr> = v.into_expr(); + let expected = core_stmt::Expr::list(std::iter::empty::()); + assert_eq!(untyped(expr), expected); +} + +#[test] +fn by_ref_vec_to_list() { + let v = vec![7i64, 8]; + let expr: Expr> = v.by_ref(); + let expected = core_stmt::Expr::list([ + core_stmt::Expr::Value(core_stmt::Value::I64(7)), + core_stmt::Expr::Value(core_stmt::Value::I64(8)), + ]); + assert_eq!(untyped(expr), expected); +} + +// --------------------------------------------------------------------------- +// &[E] -> List +// --------------------------------------------------------------------------- + +#[test] +fn into_expr_slice_to_list() { + let v = vec![100i64, 200]; + let slice: &[i64] = &v; + let expr: Expr> = slice.into_expr(); + let expected = core_stmt::Expr::list([ + core_stmt::Expr::Value(core_stmt::Value::I64(100)), + core_stmt::Expr::Value(core_stmt::Value::I64(200)), + ]); + assert_eq!(untyped(expr), expected); +} + +// --------------------------------------------------------------------------- +// Tuple impls +// --------------------------------------------------------------------------- + +#[test] +fn into_expr_pair() { + let pair: (i64, bool) = (1, true); + let expr: Expr<(i64, bool)> = pair.into_expr(); + let core_expr = untyped(expr); + let expected = core_stmt::Expr::Record(core_stmt::ExprRecord::from_vec(vec![ + core_stmt::Expr::Value(core_stmt::Value::I64(1)), + core_stmt::Expr::Value(core_stmt::Value::Bool(true)), + ])); + assert_eq!(core_expr, expected); +} + +#[test] +fn into_expr_triple() { + let triple = (1i64, "hi", true); + let expr: Expr<(i64, String, bool)> = triple.into_expr(); + let core_expr = untyped(expr); + let expected = core_stmt::Expr::Record(core_stmt::ExprRecord::from_vec(vec![ + core_stmt::Expr::Value(core_stmt::Value::I64(1)), + core_stmt::Expr::Value(core_stmt::Value::String("hi".into())), + core_stmt::Expr::Value(core_stmt::Value::Bool(true)), + ])); + assert_eq!(core_expr, expected); +} + +#[test] +fn by_ref_pair() { + let pair = (10i64, false); + let expr: Expr<(i64, bool)> = pair.by_ref(); + let core_expr = untyped(expr); + let expected = core_stmt::Expr::Record(core_stmt::ExprRecord::from_vec(vec![ + core_stmt::Expr::Value(core_stmt::Value::I64(10)), + core_stmt::Expr::Value(core_stmt::Value::Bool(false)), + ])); + assert_eq!(core_expr, expected); +} + +// --------------------------------------------------------------------------- +// Batch -> IntoExpr +// --------------------------------------------------------------------------- + +#[test] +fn batch_into_expr_wraps_statement() { + let query = core_stmt::Query::new_single(core_stmt::Values::from(core_stmt::Expr::Value( + core_stmt::Value::I64(1), + ))); + let stmt = toasty::Statement::::from_untyped_stmt(query.into()); + let batch = toasty::stmt::Batch::from(stmt); + let expr: Expr = batch.into_expr(); + let core_expr = untyped(expr); + + // Should be an Expr::Stmt wrapping the original statement + assert!(matches!(core_expr, core_stmt::Expr::Stmt(_))); +} + +#[test] +fn batch_by_ref_wraps_statement() { + let query = core_stmt::Query::new_single(core_stmt::Values::from(core_stmt::Expr::Value( + core_stmt::Value::I64(1), + ))); + let stmt = toasty::Statement::::from_untyped_stmt(query.into()); + let batch = toasty::stmt::Batch::from(stmt); + let expr: Expr = batch.by_ref(); + let core_expr = untyped(expr); + + assert!(matches!(core_expr, core_stmt::Expr::Stmt(_))); +} + +#[test] +fn batch_into_expr_and_by_ref_agree() { + let query = core_stmt::Query::new_single(core_stmt::Values::from(core_stmt::Expr::Value( + core_stmt::Value::Bool(true), + ))); + let stmt = toasty::Statement::::from_untyped_stmt(query.into()); + let batch = toasty::stmt::Batch::from(stmt); + + let by_ref_expr: Expr = batch.by_ref(); + let owned_expr: Expr = batch.into_expr(); + + assert_eq!(untyped(by_ref_expr), untyped(owned_expr)); +} diff --git a/packages/toasty/tests/stmt_into_scope.rs b/packages/toasty/tests/stmt_into_scope.rs new file mode 100644 index 00000000000..c11bf32681d --- /dev/null +++ b/packages/toasty/tests/stmt_into_scope.rs @@ -0,0 +1,69 @@ +use toasty::stmt::{IntoScope, IntoStatement, List, Query}; +use toasty_core::stmt as core_stmt; + +#[derive(Debug, toasty::Model)] +struct User { + #[key] + id: i64, + name: String, +} + +fn untyped_query(stmt: toasty::Statement) -> core_stmt::Query { + stmt.into_untyped().into_query_unwrap() +} + +#[test] +fn into_scope_from_list_preserves_query() { + let q: Query> = Query::all(); + let original = untyped_query(q.clone().into_statement()); + + let widened = untyped_query(IntoScope::::into_scope(q)); + + assert_eq!(widened, original); + assert!(!widened.single); + assert!(widened.limit.is_none()); +} + +#[test] +fn into_scope_from_option_clears_single_and_limit() { + let q: Query> = Query::>::all().first(); + + // Sanity: .first() applied both `single = true` and `LIMIT 1`. + let pre = untyped_query(q.clone().into_statement()); + assert!(pre.single); + assert!(pre.limit.is_some()); + + let widened = untyped_query(IntoScope::::into_scope(q)); + + assert!(!widened.single); + assert!(widened.limit.is_none()); +} + +#[test] +fn into_scope_from_one_clears_single_and_limit() { + let q: Query = Query::>::all().one(); + + let pre = untyped_query(q.clone().into_statement()); + assert!(pre.single); + assert!(pre.limit.is_some()); + + let widened = untyped_query(IntoScope::::into_scope(q)); + + assert!(!widened.single); + assert!(widened.limit.is_none()); +} + +#[test] +fn into_scope_preserves_filter_body() { + // The body (source + filter) should round-trip identically through + // widening — only `single` / `limit` get reset. + let list_q: Query> = + Query::>::all().filter(User::fields().name().eq("alice")); + let from_list = untyped_query(IntoScope::::into_scope(list_q.clone())); + + let from_option = untyped_query(IntoScope::::into_scope(list_q.clone().first())); + let from_one = untyped_query(IntoScope::::into_scope(list_q.one())); + + assert_eq!(from_list.body, from_option.body); + assert_eq!(from_list.body, from_one.body); +} diff --git a/packages/toasty/toasty/history.toml b/packages/toasty/toasty/history.toml new file mode 100644 index 00000000000..fdd07c7f31a --- /dev/null +++ b/packages/toasty/toasty/history.toml @@ -0,0 +1,6 @@ +version = 1 + +[[migrations]] +id = 303 +name = "0000_default.sql" +snapshot_name = "0000_snapshot.toml" diff --git a/packages/toasty/toasty/migrations/0000_default.sql b/packages/toasty/toasty/migrations/0000_default.sql new file mode 100644 index 00000000000..23ae9c3a861 --- /dev/null +++ b/packages/toasty/toasty/migrations/0000_default.sql @@ -0,0 +1,3 @@ +CREATE TABLE default_embedded_items ( + id INTEGER PRIMARY KEY +);